flux-batch 0.0.12__tar.gz → 0.0.13__tar.gz

{flux_batch-0.0.12/flux_batch.egg-info → flux_batch-0.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flux-batch
-Version: 0.0.12
+Version: 0.0.13
 Summary: Python SDK for flux batch jobs and services
 Home-page: https://github.com/converged-computing/flux-batch
 Author: Vanessa Sochat
@@ -47,6 +47,8 @@ Requires-Dist: rich; extra == "scribe"
 - [fractale-mcp](https://github.com/compspec/fractale-mcp): (fractale) MCP orchestration (agents, databases, ui interfaces).
 - [hpc-mcp](https://github.com/converged-computing/hpc-mcp): HPC tools for a larger set of HPC and converged computing use cases.
 
+If you are looking for `flux batch` please see the documentation [here](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/man1/flux-batch.html). This library supports Flux Framework and is experimental.
+
 ## Services
 
  - **flux-scribe**: Write job events to a local sqlite database via the JournalConsumer (not added yet, written and needs testing)

{flux_batch-0.0.12 → flux_batch-0.0.13}/README.md

@@ -13,6 +13,8 @@
 - [fractale-mcp](https://github.com/compspec/fractale-mcp): (fractale) MCP orchestration (agents, databases, ui interfaces).
 - [hpc-mcp](https://github.com/converged-computing/hpc-mcp): HPC tools for a larger set of HPC and converged computing use cases.
 
+If you are looking for `flux batch` please see the documentation [here](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/man1/flux-batch.html). This library supports Flux Framework and is experimental.
+
 ## Services
 
  - **flux-scribe**: Write job events to a local sqlite database via the JournalConsumer (not added yet, written and needs testing)

flux_batch-0.0.13/flux_batch/service/scribe/__main__.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+import errno
+import logging
+import os
+import sys
+import time
+
+import flux
+import flux.job
+
+# Not necessary, but it makes it pretty
+from rich import print
+
+# Use the synchronous version of the backend to avoid asyncio-in-thread conflicts
+from flux_batch.service.scribe.database import SQLAlchemyBackend
+
+# Setup logging to stderr (to avoid polluting stdout if run manually)
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr
+)
+logger = logging.getLogger("flux-scribe")
+
+
+class JournalScribe:
+    def __init__(self, db_url: str):
+        """
+        Initializes the Scribe with a synchronous DB backend and a Flux Journal Consumer.
+        """
+        # Get settings from environment
+        self.init_settings()
+
+        # Setup Database
+        logger.info(f"Connecting to Database: {db_url}")
+        self.db = SQLAlchemyBackend(db_url, self.settings["use_ssl"])
+        self.db.initialize()
+
+        try:
+            self.handle = flux.Flux()
+            logger.info("Connected to Flux instance.")
+        except Exception as e:
+            logger.critical(f"Failed to connect to Flux: {e}")
+            sys.exit(1)
+
+        logger.info(f"🍳 Handle: {self.handle.attr_get('local-uri')}")
+
+        # Initialize Journal Consumer
+        # This consumes the global event log for the entire instance
+        self.consumer = flux.job.JournalConsumer(self.handle)
+        self.running = True
+
+    def init_settings(self):
+        """
+        Initialize settings.
+        """
+        self.settings = {
+            "use_ssl": os.environ.get("FLUX_SCRIBE_SSL") not in [None, "no", "false", "off"],
+            "uid": os.environ.get("FLUX_SCRIBE_UID"),
+        }
+
+    def _normalize_event(self, event) -> dict:
+        """
+        Converts a Flux event object into the dictionary format expected by record_event.
+        Matches the logic provided in your EventsEngine reference.
+        """
+        # Convert the SWIG/CFFI event object to a dictionary
+        payload = dict(event)
+
+        return {
+            "id": str(getattr(event, "jobid", "unknown")),
+            "type": getattr(event, "name", "unknown"),
+            "timestamp": getattr(event, "timestamp", time.time()),
+            "payload": payload,
+            "R": getattr(event, "R", None),
+            "jobspec": getattr(event, "jobspec", None),
+        }
+
+    def run(self):
+        """
+        Main execution loop. Polls the journal and writes to the DB.
+        """
+        try:
+            logger.info("🚀 Flux Scribe (Journal Consumer) started.")
+            self.consumer.start()
+
+            while self.running:
+                try:
+                    # Non-blocking poll (100ms timeout)
+                    # This allows the loop to check for shutdown signals regularly
+                    event = self.consumer.poll(timeout=0.1)
+
+                    if event:
+                        print(event)
+                        # We only care about events associated with a job
+                        if hasattr(event, "jobid"):
+                            clean_event = self._normalize_event(event)
+                            self.db.record_event("local", self.settings['uid'], clean_event)
+                    else:
+                        # If no event, yield a tiny bit of CPU
+                        time.sleep(0.01)
+
+                except EnvironmentError as e:
+                    # Ignore timeouts (no data)
+                    if e.errno == errno.ETIMEDOUT:
+                        continue
+                    logger.error(f"Flux connection error: {e}")
+                    time.sleep(1)
+
+                except Exception as e:
+                    logger.error(f"Unexpected error in event loop: {e}")
+                    time.sleep(1)
+
+        except Exception as e:
+            logger.critical(f"EventsEngine crashed: {e}")
+        finally:
+            self.db.close()
+            logger.info("EventsEngine thread exiting.")
+
+
+def main():
+    # Retrieve DB path from environment or use a default
+    db_path = os.environ.get("FLUX_SCRIBE_DATABASE", "sqlite:///server_state.db")
+    scribe = JournalScribe(db_path)
+    scribe.run()
+
+
+if __name__ == "__main__":
+    main()

flux_batch-0.0.13/flux_batch/service/scribe/database.py

@@ -0,0 +1,166 @@
+import time
+from typing import Any, Dict, List, Optional
+
+from sqlalchemy import and_, create_engine, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.orm import sessionmaker
+
+from flux_batch.service.scribe.models import Base, EventModel, EventRecord, JobModel, JobRecord
+
+
+def _record_event_internal(session, cluster: str, uid: str, event: Dict[str, Any]):
+    """
+    Shared synchronous logic for recording events.
+    Used by both Sync and Async backends.
+    """
+    job_id = event.get("id")
+    event_type = event.get("type")
+    data = event.get("payload", {})
+    timestamp = event.get("timestamp", time.time())
+
+    # Add the new event with all metadata
+    new_event = EventModel(
+        job_id=job_id,
+        cluster=cluster,
+        timestamp=timestamp,
+        event_type=event_type,
+        uid=uid,
+        payload=data,
+    )
+    session.add(new_event)
+
+    if event_type == "submit":
+        stmt = select(JobModel).where(and_(JobModel.job_id == job_id, JobModel.cluster == cluster))
+        job = session.execute(stmt).scalar_one_or_none()
+
+        if not job:
+            job = JobModel(
+                job_id=job_id,
+                cluster=cluster,
+                user=str(data.get("userid", "unknown")),
+                state="submitted",
+                workdir=data.get("cwd", ""),
+                submit_time=timestamp,
+                last_updated=timestamp,
+                uid=uid,
+            )
+            session.add(job)
+        else:
+            job.state = "submitted"
+            job.last_updated = timestamp
+
+    # state transitions
+    elif event_type == "state" or (event_type and event_type.endswith(".finish")):
+        state_name = data.get("state_name", event_type)
+        stmt = select(JobModel).where(and_(JobModel.job_id == job_id, JobModel.cluster == cluster))
+        job = session.execute(stmt).scalar_one_or_none()
+        if job:
+            job.state = state_name
+            job.last_updated = time.time()
+            if "status" in data:
+                job.exit_code = data["status"]
+
+
+class AsyncSQLAlchemyBackend:
+    """
+    Asynchronous backend for the MCP Gateway.
+    """
+
+    def __init__(self, db_url: str, use_ssl: bool = False):
+        # Use asyncio connection via asyncmy
+        if db_url.startswith("mysql://") or db_url.startswith("mariadb://"):
+            db_url = db_url.replace("mysql://", "mysql+asyncmy://", 1)
+            db_url = db_url.replace("mariadb://", "mysql+asyncmy://", 1)
+
+        connect_args = {"connect_timeout": 10}
+        if use_ssl:
+            print("SSL is enabled.")
+            connect_args["ssl"] = {"ssl_mode": "REQUIRED", "check_hostname": False}
+
+        self.engine = create_async_engine(db_url, echo=False, connect_args=connect_args)
+        self.SessionLocal = async_sessionmaker(self.engine, expire_on_commit=False)
+
+    async def initialize(self):
+        async with self.engine.begin() as conn:
+            await conn.run_sync(Base.metadata.create_all)
+
+    async def close(self):
+        await self.engine.dispose()
+
+    async def record_event(self, cluster: str, event: Dict[str, Any]):
+        async with self.SessionLocal() as session:
+            # run_sync bridges our shared logic into the async session
+            await session.run_sync(_record_event_internal, cluster, event)
+            await session.commit()
+
+    async def get_job(self, cluster: str, job_id: int) -> Optional[JobRecord]:
+        async with self.SessionLocal() as session:
+            result = await session.execute(
+                select(JobModel).where(and_(JobModel.job_id == job_id, JobModel.cluster == cluster))
+            )
+            job = result.scalar_one_or_none()
+            return job.to_record() if job else None
+
+    async def get_event_history(self, cluster: str, job_id: int) -> List[EventRecord]:
+        async with self.SessionLocal() as session:
+            result = await session.execute(
+                select(EventModel)
+                .where(and_(EventModel.job_id == job_id, EventModel.cluster == cluster))
+                .order_by(EventModel.timestamp.asc())
+            )
+            return [e.to_record() for e in result.scalars().all()]
+
+    async def search_jobs(
+        self, cluster: str = None, state: str = None, limit: int = 10
+    ) -> List[JobRecord]:
+        async with self.SessionLocal() as session:
+            stmt = select(JobModel)
+            if cluster:
+                stmt = stmt.where(JobModel.cluster == cluster)
+            if state:
+                stmt = stmt.where(JobModel.state == state)
+            result = await session.execute(stmt.limit(limit))
+            return [j.to_record() for j in result.scalars().all()]
+
+
+class SQLAlchemyBackend:
+    """
+    Synchronous backend for the standalone Scribe daemon.
+    """
+
+    def __init__(self, db_url: str, use_ssl: bool = False):
+        connect_args = {"connect_timeout": 10}
+        if use_ssl:
+            print("SSL is enabled.")
+            connect_args["ssl"] = {"ssl_mode": "REQUIRED", "check_hostname": False}
+
+        self.engine = create_engine(db_url, echo=False, connect_args=connect_args)
+        self.SessionLocal = sessionmaker(bind=self.engine, expire_on_commit=False)
+
+    def initialize(self):
+        Base.metadata.create_all(self.engine)
+
+    def close(self):
+        self.engine.dispose()
+
+    def record_event(self, cluster: str, uid: str, event: Dict[str, Any]):
+        with self.SessionLocal() as session:
+            with session.begin():
+                _record_event_internal(session, cluster, uid, event)
+
+    def get_unwatched_job_ids(self, cluster: str) -> List[int]:
+        """Specific for Scribe: find jobs that need a watcher."""
+        with self.SessionLocal() as session:
+            stmt = select(JobModel.job_id).where(
+                and_(JobModel.cluster == cluster, JobModel.state == "submitted")
+            )
+            return list(session.execute(stmt).scalars().all())
+
+    def mark_job_as_watched(self, cluster: str, job_id: int):
+        with self.SessionLocal() as session:
+            with session.begin():
+                session.execute(
+                    update(JobModel)
+                    .where(and_(JobModel.job_id == job_id, JobModel.cluster == cluster))
+                    .values(state="watching")
+                )

flux_batch-0.0.13/flux_batch/service/scribe/models.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from sqlalchemy import JSON, Float, Integer, String
+from sqlalchemy.ext.asyncio import AsyncAttrs
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+# DTOs are "Public Data Transfer Objects" and they are used by
+# our interfaces and tools
+
+
+@dataclass
+class JobRecord:
+    """
+    Represents a snapshot of a job state.
+    Returned by get_job() and search_jobs().
+    """
+
+    job_id: str
+    cluster: str
+    state: str
+    user: str
+    workdir: Optional[str] = None
+    exit_code: Optional[int] = None
+    submit_time: float = 0.0
+    last_updated: float = 0.0
+    uid: str = None
+
+
+@dataclass
+class EventRecord:
+    """
+    Represents a single historical event.
+    Returned by get_event_history().
+    """
+
+    timestamp: float
+    event_type: str
+    payload: Dict[str, Any]
+    uid: str = None
+
+
+# Database models for SQLAlchemy ORM
+
+
+class Base(AsyncAttrs, DeclarativeBase):
+    pass
+
+
+class JobModel(Base):
+    __tablename__ = "jobs"
+
+    # Composite Primary Key
+    cluster: Mapped[str] = mapped_column(String(255), primary_key=True)
+    job_id: Mapped[str] = mapped_column(String(255), primary_key=True)
+    uid: Mapped[str] = mapped_column(String(255), nullable=True)
+
+    state: Mapped[str] = mapped_column(String(50))
+    user: Mapped[str] = mapped_column(String(255), nullable=True)
+
+    # Fixed: Added length 1024 for MySQL/MariaDB compatibility
+    workdir: Mapped[Optional[str]] = mapped_column(String(1024), nullable=True)
+
+    exit_code: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    submit_time: Mapped[float] = mapped_column(Float, default=0.0)
+    last_updated: Mapped[float] = mapped_column(Float, default=0.0)
+
+    def to_record(self) -> JobRecord:
+        """
+        Helper to convert ORM model to public DTO
+        """
+        return JobRecord(
+            job_id=self.job_id,
+            cluster=self.cluster,
+            state=self.state,
+            user=self.user,
+            workdir=self.workdir,
+            exit_code=self.exit_code,
+            submit_time=self.submit_time,
+            last_updated=self.last_updated,
+            uid=self.uid,
+        )
+
+
+class EventModel(Base):
+    __tablename__ = "events"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    job_id: Mapped[str] = mapped_column(String(255), primary_key=True)
+    cluster: Mapped[str] = mapped_column(String(255), index=True)
+    timestamp: Mapped[float] = mapped_column(Float)
+    event_type: Mapped[str] = mapped_column(String(50))
+    payload: Mapped[Dict[str, Any]] = mapped_column(JSON)
+    uid: Mapped[str] = mapped_column(String(255), nullable=True)
+
+    def to_record(self) -> EventRecord:
+        """
+        Helper to convert ORM model to public DTO
+        """
+        return EventRecord(
+            timestamp=self.timestamp,
+            event_type=self.event_type,
+            payload=self.payload,
+            cluster=self.cluster,
+            job_id=self.job_id,
+            uid=self.uid,
+        )

flux_batch-0.0.13/flux_batch/service/usernetes/__init__.py

@@ -0,0 +1,2 @@
+MODULE_NAME = "flux_batch.service.usernetes"
+from .template import SERVICE_TEMPLATE, START_MODULE_TEMPLATE, STOP_MODULE_TEMPLATE

flux_batch-0.0.13/flux_batch/service/usernetes/template.py

@@ -0,0 +1,53 @@
+# Template for the Scribe Journal Consumer
+SERVICE_TEMPLATE = """[Unit]
+Description=Flux Scribe Journal Consumer
+After=network.target
+
+[Service]
+ExecStart={python_path} -m flux_batch.service.scribe
+Restart=on-failure
+
+[Install]
+WantedBy=default.target
+"""
+
+START_MODULE_TEMPLATE = """
+from flux.modprobe import task
+import flux.subprocess as subprocess
+
+@task(
+    "start-{service_name}",
+    ranks="0",
+    needs_config=["{service_name}"],
+    after=["resource", "job-list"],
+)
+def start_{service_func}(context):
+    # This triggers the systemd user service provisioned earlier
+    # context.bash("systemctl --user start {service_name}")
+    subprocess.rexec_bg(
+        context.handle,
+        ["{python_bin}", "-m", "{module_name}"],
+        label="{service_name}",
+        nodeid=0
+    )
+"""
+
+STOP_MODULE_TEMPLATE = """
+from flux.modprobe import task
+import flux.subprocess as subprocess
+
+@task(
+    "stop-{service_name}",
+    ranks="0",
+    needs_config=["{service_name}"],
+    before=["resource", "job-list"],
+)
+def stop_{service_func}(context):
+    # context.bash("systemctl --user stop {service_name}")
+    subprocess.kill(context.handle, signum=2, label="{service_name}").get()
+    try:
+        status = subprocess.wait(context.handle, label="{service_name}").get()["status"]
+        print(status)
+    except:
+        pass
+"""

{flux_batch-0.0.12 → flux_batch-0.0.13}/flux_batch/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.12"
+__version__ = "0.0.13"
 AUTHOR = "Vanessa Sochat"
 AUTHOR_EMAIL = "vsoch@users.noreply.github.com"
 NAME = "flux-batch"

{flux_batch-0.0.12 → flux_batch-0.0.13/flux_batch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flux-batch
-Version: 0.0.12
+Version: 0.0.13
 Summary: Python SDK for flux batch jobs and services
 Home-page: https://github.com/converged-computing/flux-batch
 Author: Vanessa Sochat
@@ -47,6 +47,8 @@ Requires-Dist: rich; extra == "scribe"
 - [fractale-mcp](https://github.com/compspec/fractale-mcp): (fractale) MCP orchestration (agents, databases, ui interfaces).
 - [hpc-mcp](https://github.com/converged-computing/hpc-mcp): HPC tools for a larger set of HPC and converged computing use cases.
 
+If you are looking for `flux batch` please see the documentation [here](https://flux-framework.readthedocs.io/projects/flux-core/en/latest/man1/flux-batch.html). This library supports Flux Framework and is experimental.
+
 ## Services
 
  - **flux-scribe**: Write job events to a local sqlite database via the JournalConsumer (not added yet, written and needs testing)

{flux_batch-0.0.12 → flux_batch-0.0.13}/flux_batch.egg-info/SOURCES.txt

@@ -28,6 +28,11 @@ flux_batch/service/scribe/__main__.py
 flux_batch/service/scribe/database.py
 flux_batch/service/scribe/models.py
 flux_batch/service/scribe/template.py
+flux_batch/service/usernetes/__init__.py
+flux_batch/service/usernetes/__main__.py
+flux_batch/service/usernetes/database.py
+flux_batch/service/usernetes/models.py
+flux_batch/service/usernetes/template.py
 flux_batch/utils/__init__.py
 flux_batch/utils/fileio.py
 flux_batch/utils/text.py