ob-metaflow-extensions 1.1.161rc1__py2.py3-none-any.whl → 1.1.162__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/ollama/status_card.py

@@ -0,0 +1,292 @@
+from metaflow.cards import Markdown, Table, VegaChart
+from metaflow.metaflow_current import current
+from datetime import datetime
+import threading
+import time
+
+from ..card_utilities.async_cards import CardRefresher
+
+
+class OllamaStatusCard(CardRefresher):
+    """
+    Real-time status card for Ollama system monitoring.
+    Shows circuit breaker state, server health, model status, and recent events.
+    """
+
+    CARD_ID = "ollama_status"
+
+    def __init__(self, refresh_interval=10):
+        self.refresh_interval = refresh_interval
+        self.status_data = {
+            "circuit_breaker": {
+                "state": "CLOSED",
+                "failure_count": 0,
+                "last_failure_time": None,
+                "last_open_time": None,
+            },
+            "server": {
+                "status": "Starting",
+                "uptime_start": None,
+                "restart_attempts": 0,
+                "last_health_check": None,
+                "health_status": "Unknown",
+            },
+            "models": {},  # model_name -> {status, pull_time, load_time, etc}
+            "performance": {
+                "install_time": None,
+                "server_startup_time": None,
+                "total_initialization_time": None,
+            },
+            "versions": {
+                "ollama_system": "Detecting...",
+                "ollama_python": "Detecting...",
+            },
+            "cache": {
+                "policy": "auto",
+                "model_status": {},  # model_name -> cache status
+            },
+            "events": [],  # Recent events log
+        }
+        self._lock = threading.Lock()
+        self._already_rendered = False
+
+    def update_status(self, category, data):
+        """Thread-safe method to update status data"""
+        with self._lock:
+            if category in self.status_data:
+                self.status_data[category].update(data)
+
+    def add_event(self, event_type, message, timestamp=None):
+        """Add an event to the timeline"""
+        if timestamp is None:
+            timestamp = datetime.now()
+
+        with self._lock:
+            self.status_data["events"].insert(
+                0,
+                {
+                    "type": event_type,  # 'info', 'warning', 'error', 'success'
+                    "message": message,
+                    "timestamp": timestamp,
+                },
+            )
+            # Keep only last 10 events
+            self.status_data["events"] = self.status_data["events"][:10]
+
+    def get_circuit_breaker_emoji(self, state):
+        """Get status emoji for circuit breaker state"""
+        emoji_map = {"CLOSED": "🟢", "OPEN": "🔴", "HALF_OPEN": "🟡"}
+        return emoji_map.get(state, "⚪")
+
+    def get_uptime_string(self, start_time):
+        """Calculate uptime string"""
+        if not start_time:
+            return "Not started"
+
+        uptime = datetime.now() - start_time
+        hours, remainder = divmod(int(uptime.total_seconds()), 3600)
+        minutes, seconds = divmod(remainder, 60)
+
+        if hours > 0:
+            return f"{hours}h {minutes}m {seconds}s"
+        elif minutes > 0:
+            return f"{minutes}m {seconds}s"
+        else:
+            return f"{seconds}s"
+
+    def on_startup(self, current_card):
+        """Initialize the card when monitoring starts"""
+        current_card.append(Markdown("# 🦙 `@ollama` Status Dashboard"))
+        current_card.append(Markdown("_Initializing Ollama system..._"))
+        current_card.refresh()
+
+    def render_card_fresh(self, current_card, data):
+        """Render the complete card with all status information"""
+        self._already_rendered = True
+        current_card.clear()
+
+        # Header with version information
+        current_card.append(Markdown("# 🦙 `@ollama` Status Dashboard"))
+
+        # Version information in header
+        versions = data.get("versions", {})
+        system_version = versions.get("ollama_system", "Unknown")
+        python_version = versions.get("ollama_python", "Unknown")
+        current_card.append(
+            Markdown(
+                f"**System:** `{system_version}` | **Python Client:** `{python_version}`"
+            )
+        )
+
+        # Cache policy information
+        cache_info = data.get("cache", {})
+        cache_policy = cache_info.get("policy", "auto")
+        current_card.append(Markdown(f"**Cache Policy:** `{cache_policy}`"))
+
+        current_card.append(
+            Markdown(f"_Last updated: {datetime.now().strftime('%H:%M:%S')}_")
+        )
+
+        # Circuit Breaker Status
+        cb_data = data["circuit_breaker"]
+        cb_emoji = self.get_circuit_breaker_emoji(cb_data["state"])
+        cb_status = f"{cb_emoji} **{cb_data['state']}**"
+        if cb_data["failure_count"] > 0:
+            cb_status += f" (failures: {cb_data['failure_count']})"
+
+        # Server Status
+        server_data = data["server"]
+        uptime = self.get_uptime_string(server_data.get("uptime_start"))
+        server_status = f"**{server_data['status']}**"
+        if server_data["restart_attempts"] > 0:
+            server_status += f" (restarts: {server_data['restart_attempts']})"
+
+        # Status Overview Table
+        status_table = [
+            ["Circuit Breaker", Markdown(cb_status)],
+            ["Server Status", Markdown(server_status)],
+            ["Server Uptime", Markdown(uptime)],
+            [
+                "Last Health Check",
+                Markdown(server_data.get("health_status", "Unknown")),
+            ],
+        ]
+
+        current_card.append(Markdown("## System Status"))
+        current_card.append(Table(status_table, headers=["Component", "Status"]))
+
+        # Models Status
+        if data["models"]:
+            current_card.append(Markdown("## Models"))
+            model_table = []
+            cache_model_status = cache_info.get("model_status", {})
+
+            for model_name, model_info in data["models"].items():
+                status = model_info.get("status", "Unknown")
+                pull_time = model_info.get("pull_time", "N/A")
+                if isinstance(pull_time, (int, float)):
+                    pull_time = f"{pull_time:.1f}s"
+
+                # Add cache status indicator
+                cache_status = cache_model_status.get(model_name, "unknown")
+                cache_emoji = {
+                    "exists": "💾",
+                    "missing": "❌",
+                    "error": "⚠️",
+                    "unknown": "❓",
+                }.get(cache_status, "❓")
+
+                # Get model metadata
+                size_formatted = model_info.get("size_formatted", "Unknown")
+                blob_count = model_info.get("blob_count", "Unknown")
+                if blob_count == 0:
+                    blob_count = "Unknown"
+
+                model_table.append(
+                    [
+                        f"{model_name} {cache_emoji}",
+                        status,
+                        pull_time,
+                        size_formatted,
+                        str(blob_count),
+                    ]
+                )
+
+            current_card.append(
+                Table(
+                    model_table,
+                    headers=["Model (Cache)", "Status", "Pull Time", "Size", "Blobs"],
+                )
+            )
+
+        # Performance Metrics
+        perf_data = data["performance"]
+        if any(v is not None for v in perf_data.values()):
+            current_card.append(Markdown("## Performance"))
+
+            # Separate initialization and shutdown metrics
+            init_metrics = []
+            shutdown_metrics = []
+            other_metrics = []
+
+            for metric, value in perf_data.items():
+                if value is not None:
+                    display_value = value
+                    if isinstance(value, (int, float)):
+                        display_value = f"{value:.1f}s"
+
+                    metric_display = metric.replace("_", " ").title()
+
+                    if "shutdown" in metric.lower():
+                        shutdown_metrics.append([metric_display, display_value])
+                    elif metric in [
+                        "install_time",
+                        "server_startup_time",
+                        "total_initialization_time",
+                    ]:
+                        init_metrics.append([metric_display, display_value])
+                    else:
+                        other_metrics.append([metric_display, display_value])
+
+            # Display metrics in organized sections
+            if init_metrics:
+                current_card.append(Markdown("### Initialization"))
+                current_card.append(Table(init_metrics, headers=["Metric", "Duration"]))
+
+            if shutdown_metrics:
+                current_card.append(Markdown("### Shutdown"))
+                current_card.append(
+                    Table(shutdown_metrics, headers=["Metric", "Value"])
+                )
+
+            if other_metrics:
+                current_card.append(Markdown("### Other"))
+                current_card.append(Table(other_metrics, headers=["Metric", "Value"]))
+
+        # Recent Events
+        if data["events"]:
+            current_card.append(Markdown("## Recent Events"))
+            events_table = []
+            for event in data["events"][:5]:  # Show last 5 events
+                timestamp = event["timestamp"].strftime("%H:%M:%S")
+                event_type = event["type"]
+                message = event["message"]
+
+                # Add emoji based on event type
+                type_emoji = {
+                    "info": "ℹ️",
+                    "success": "✅",
+                    "warning": "⚠️",
+                    "error": "❌",
+                }.get(event_type, "ℹ️")
+
+                events_table.append([timestamp, f"{type_emoji} {message}"])
+
+            current_card.append(Table(events_table, headers=["Time", "Event"]))
+
+        current_card.refresh()
+
+    def on_error(self, current_card, error_message):
+        """Handle errors in card rendering"""
+        if not self._already_rendered:
+            current_card.clear()
+            current_card.append(Markdown("# 🦙 `@ollama` Status Dashboard"))
+            current_card.append(Markdown(f"## ❌ Error: {str(error_message)}"))
+            current_card.refresh()
+
+    def on_update(self, current_card, data_object):
+        """Update the card with new data"""
+        with self._lock:
+            current_data = self.status_data.copy()
+
+        if not self._already_rendered:
+            self.render_card_fresh(current_card, current_data)
+        else:
+            # For frequent updates, we could implement incremental updates here
+            # For now, just re-render the whole card
+            self.render_card_fresh(current_card, current_data)
+
+    def sqlite_fetch_func(self, conn):
+        """Required by CardRefresher (which needs a refactor), but we use in-memory data instead"""
+        with self._lock:
+            return {"status": self.status_data}

metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py

@@ -53,4 +53,3 @@ def S3(*args, **kwargs):
 from .. import profilers
 from ..plugins.snowflake import Snowflake
 from ..plugins.checkpoint_datastores import nebius_checkpoints, coreweave_checkpoints
-from . import ob_internal

{ob_metaflow_extensions-1.1.161rc1.dist-info → ob_metaflow_extensions-1.1.162.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.161rc1
+Version: 1.1.162
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.1.161rc1.dist-info → ob_metaflow_extensions-1.1.162.dist-info}/RECORD

@@ -17,7 +17,6 @@ metaflow_extensions/outerbounds/plugins/checkpoint_datastores/__init__.py,sha256
 metaflow_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.py,sha256=_WzoOROFjoFa8TzsMNFp-r_1Zz7NUp-5ljn_kKlczXA,4534
 metaflow_extensions/outerbounds/plugins/checkpoint_datastores/nebius.py,sha256=zgqDLFewCeF5jqh-hUNKmC_OAjld09ln0bb8Lkeqapc,4659
 metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py,sha256=ShE5omFBr83wkvEhL_ptRFvDNMs6wefg4BjaafQjTcM,3602
 metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=Tl520HdBteg-aDOM7mnnJJpdDCZc49BmFFmLUc_vTi8,15018
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=PE81ZB54OAMXkMGSB7JqgvgMg7N9kvoVclrWL-6jc2U,5626
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kqFyu2bJSnc9_9aYfBpz5xK6L6luWFZK_NMuh8f1eVk,1494
@@ -44,10 +43,11 @@ metaflow_extensions/outerbounds/plugins/nvct/nvct_cli.py,sha256=bB9AURhRep9PV_-b
 metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py,sha256=HKCvYn1Jh8uwLXeUqPNhxgBatq3mXNG5YIUl-zjNlHE,9429
 metaflow_extensions/outerbounds/plugins/nvct/nvct_runner.py,sha256=8IPkdvuTZNIqgAAt75gVNn-ydr-Zz2sKC8UX_6pNEKI,7091
 metaflow_extensions/outerbounds/plugins/nvct/utils.py,sha256=U4_Fu8H94j_Bbox7mmMhNnlRhlYHqnK28R5w_TMWEFM,1029
-metaflow_extensions/outerbounds/plugins/ollama/__init__.py,sha256=vzh8sQEfwKRdx0fsGFJ-km4mwfi0vm2q1_vsZv-EMcc,3034
+metaflow_extensions/outerbounds/plugins/ollama/__init__.py,sha256=4T8LQqAuh8flSMvYztw6-OPoDoAorcBWhC-vPuuQPbc,9234
 metaflow_extensions/outerbounds/plugins/ollama/constants.py,sha256=hxkTpWEJp1pKHwUcG4EE3-17M6x2CyeMfbeqgUzF9TA,28
 metaflow_extensions/outerbounds/plugins/ollama/exceptions.py,sha256=8Ss296_MGZl1wXAoDNwpH-hsPe6iYLe90Ji1pczNocU,668
-metaflow_extensions/outerbounds/plugins/ollama/ollama.py,sha256=oe-k1ISSMtUF2y3YpfmJhU_3yR7SP31PVilN5NPgKv0,31450
+metaflow_extensions/outerbounds/plugins/ollama/ollama.py,sha256=C-6Hz8OxsJiB14AAxmunq3P4k7DrmVHsSOxE0xsP-nY,79780
+metaflow_extensions/outerbounds/plugins/ollama/status_card.py,sha256=F5e4McDl28lhtjeUyInkl03bqjr1lgLxWoau8Q9xwBE,10994
 metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py,sha256=oI_C3c64XBm7n88FILqHwn-Nnc5DeT_68I67lM9rXaI,2434
 metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py,sha256=gDHQ2sMIp4NuZSzUspbSd8RGdFAoO5mgZAyFcZ2a51Y,2619
 metaflow_extensions/outerbounds/plugins/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -67,15 +67,14 @@ metaflow_extensions/outerbounds/plugins/torchtune/__init__.py,sha256=TOXNeyhcgd8
 metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
 metaflow_extensions/outerbounds/profilers/gpu.py,sha256=3Er8uKQzfm_082uadg4yn_D4Y-iSCgzUfFmguYxZsz4,27485
 metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
-metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=505fLAY4NkBCGtjpXlE0RgaKVpJ0jOUq1-Fq-EKzPew,2035
-metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=53xM6d_UYT3uGFFA59UzxN23H5QMO5_F39pALpmGy04,51
+metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=oCRFfwoLODDju2-6JU2h74x6O-GCyO7PRRKTZsj-73k,2009
 metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
 metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py,sha256=GRSz2zwqkvlmFS6bcfYD_CX6CMko9DHQokMaH1iBshA,47
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
 metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py,sha256=uTVkdSk3xZ7hEKYfdlyVteWj5KeDwaM1hU9WT-_YKfI,50
-ob_metaflow_extensions-1.1.161rc1.dist-info/METADATA,sha256=QZ_Y0zbT95Qr57WQ4XnvhZUSXoOQy3x2Pd-KT73mnxo,524
-ob_metaflow_extensions-1.1.161rc1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.161rc1.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.161rc1.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.162.dist-info/METADATA,sha256=Mp8oyzEml9MbPiZMSwCBUHsB-Tfq8cMbraHKfBbfgjY,521
+ob_metaflow_extensions-1.1.162.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.162.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.162.dist-info/RECORD,,

metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py

@@ -1,110 +0,0 @@
-import threading
-import time
-import sys
-from typing import Dict, Optional, Any, Callable
-from functools import partial
-from metaflow.exception import MetaflowException
-from metaflow.metaflow_config import FAST_BAKERY_URL
-
-from .fast_bakery import FastBakery, FastBakeryApiResponse, FastBakeryException
-from .docker_environment import cache_request
-
-BAKERY_METAFILE = ".imagebakery-cache"
-
-
-class BakerException(MetaflowException):
-    headline = "Ran into an error while baking image"
-
-    def __init__(self, msg):
-        super(BakerException, self).__init__(msg)
-
-
-def bake_image(
-    cache_file_path: str,
-    ref: Optional[str] = None,
-    python: Optional[str] = None,
-    pypi_packages: Optional[Dict[str, str]] = None,
-    conda_packages: Optional[Dict[str, str]] = None,
-    base_image: Optional[str] = None,
-    logger: Optional[Callable[[str], Any]] = None,
-) -> FastBakeryApiResponse:
-    """
-    Bakes a Docker image with the specified dependencies.
-
-    Args:
-        cache_file_path: Path to the cache file
-        ref: Reference identifier for this bake (for logging purposes)
-        python: Python version to use
-        pypi_packages: Dictionary of PyPI packages and versions
-        conda_packages: Dictionary of Conda packages and versions
-        base_image: Base Docker image to use
-        logger: Optional logger function to output progress
-
-    Returns:
-        FastBakeryApiResponse: The response from the bakery service
-
-    Raises:
-        BakerException: If the baking process fails
-    """
-    # Default logger if none provided
-    if logger is None:
-        logger = partial(print, file=sys.stderr)
-
-    # Thread lock for logging
-    logger_lock = threading.Lock()
-    images_baked = 0
-
-    @cache_request(cache_file_path)
-    def _cached_bake(
-        ref=None,
-        python=None,
-        pypi_packages=None,
-        conda_packages=None,
-        base_image=None,
-    ):
-        try:
-            bakery = FastBakery(url=FAST_BAKERY_URL)
-            bakery._reset_payload()
-            bakery.python_version(python)
-            bakery.pypi_packages(pypi_packages)
-            bakery.conda_packages(conda_packages)
-            bakery.base_image(base_image)
-            # bakery.ignore_cache()
-
-            with logger_lock:
-                logger(f"🍳 Baking [{ref}] ...")
-                logger(f"     🐍 Python: {python}")
-
-                if pypi_packages:
-                    logger(f"     📦 PyPI packages:")
-                    for package, version in pypi_packages.items():
-                        logger(f"        🔧 {package}: {version}")
-
-                if conda_packages:
-                    logger(f"     📦 Conda packages:")
-                    for package, version in conda_packages.items():
-                        logger(f"        🔧 {package}: {version}")
-
-                logger(f"     🏗️  Base image: {base_image}")
-
-            start_time = time.time()
-            res = bakery.bake()
-            # TODO: Get actual bake time from bakery
-            bake_time = time.time() - start_time
-
-            with logger_lock:
-                logger(f"🏁 Baked [{ref}] in {bake_time:.2f} seconds!")
-            nonlocal images_baked
-            images_baked += 1
-            return res
-        except FastBakeryException as ex:
-            raise BakerException(f"Bake [{ref}] failed: {str(ex)}")
-
-    # Call the cached bake function with the provided parameters
-    return _cached_bake(
-        ref=ref,
-        python=python,
-        pypi_packages=pypi_packages,
-        conda_packages=conda_packages,
-        base_image=base_image,
-    )

metaflow_extensions/outerbounds/toplevel/ob_internal.py

@@ -1 +0,0 @@
-from ..plugins.fast_bakery.baker import bake_image