londonaicentre-mesa-runner 1.0.0__tar.gz

londonaicentre_mesa_runner-1.0.0/PKG-INFO

@@ -0,0 +1,141 @@
+Metadata-Version: 2.4
+Name: londonaicentre-mesa-runner
+Version: 1.0.0
+Summary: A stateless runner / deployment system for MESA models
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: boto3>=1.42.43
+Requires-Dist: duckdb>=1.4.4
+Requires-Dist: londonaicentre-oncoschema>=2.0.1
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pydantic-settings[yaml]>=2.12.0
+Requires-Dist: snowflake-core>=1.11.0
+Requires-Dist: snowflake-snowpark-python>=1.45.0
+
+# MESA Runner
+
+A stateless runner for deploying MESA registered models onto GSTT Infrastructure. It syncs models from S3, reads unprocessed documents from Snowflake, runs inference, and writes results back.
+
+## Requirements
+
+- Python 3.13+
+- [uv](https://github.com/astral-sh/uv) package manager
+
+## Installation
+
+### Remote Inference (Default)
+
+For remote inference via OpenAI-compatible endpoints:
+
+```bash
+uv sync
+```
+
+### Offline Inference (Optional)
+
+For local GPU inference with vLLM, install the optional dependency:
+
+```bash
+uv sync --group vllm-offline
+```
+
+## Configuration
+
+Create a `config.yaml` file (see example below):
+
+### Remote Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    openai_endpoint: "http://localhost:5000/v1"
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+### Offline Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    max_model_len: 18000
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+## Usage
+
+```bash
+# Run with default config.yaml
+mesa_runner
+
+# Or specify a config file
+mesa_runner --config /path/to/config.yaml
+
+# Dry run mode (uses dummy data, does not read or write real data)
+mesa_runner --dry-run
+```
+
+The dry run mode is useful for testing the runner without accessing real data sources or sinks. It generates 5 dummy documents by default and logs all write operations instead of executing them.
+
+## Docker
+
+```bash
+# Remote inference (default)
+docker build -t mesa-runner .
+docker run mesa-runner
+
+# Offline inference (includes vLLM)
+docker build --target offline -t mesa-runner:offline .
+docker run --gpus all mesa-runner:offline
+```
+
+## Development
+
+```bash
+# Run linting and tests
+make test
+
+# Auto-fix linting issues
+make fix
+
+# Run tests with coverage
+make cov
+```

londonaicentre_mesa_runner-1.0.0/README.md

@@ -0,0 +1,126 @@
+# MESA Runner
+
+A stateless runner for deploying MESA registered models onto GSTT Infrastructure. It syncs models from S3, reads unprocessed documents from Snowflake, runs inference, and writes results back.
+
+## Requirements
+
+- Python 3.13+
+- [uv](https://github.com/astral-sh/uv) package manager
+
+## Installation
+
+### Remote Inference (Default)
+
+For remote inference via OpenAI-compatible endpoints:
+
+```bash
+uv sync
+```
+
+### Offline Inference (Optional)
+
+For local GPU inference with vLLM, install the optional dependency:
+
+```bash
+uv sync --group vllm-offline
+```
+
+## Configuration
+
+Create a `config.yaml` file (see example below):
+
+### Remote Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    openai_endpoint: "http://localhost:5000/v1"
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+### Offline Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    max_model_len: 18000
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+## Usage
+
+```bash
+# Run with default config.yaml
+mesa_runner
+
+# Or specify a config file
+mesa_runner --config /path/to/config.yaml
+
+# Dry run mode (uses dummy data, does not read or write real data)
+mesa_runner --dry-run
+```
+
+The dry run mode is useful for testing the runner without accessing real data sources or sinks. It generates 5 dummy documents by default and logs all write operations instead of executing them.
+
+## Docker
+
+```bash
+# Remote inference (default)
+docker build -t mesa-runner .
+docker run mesa-runner
+
+# Offline inference (includes vLLM)
+docker build --target offline -t mesa-runner:offline .
+docker run --gpus all mesa-runner:offline
+```
+
+## Development
+
+```bash
+# Run linting and tests
+make test
+
+# Auto-fix linting issues
+make fix
+
+# Run tests with coverage
+make cov
+```

londonaicentre_mesa_runner-1.0.0/pyproject.toml

@@ -0,0 +1,61 @@
+[project]
+name = "londonaicentre-mesa-runner"
+version = "1.0.0"
+description = "A stateless runner / deployment system for MESA models"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "boto3>=1.42.43",
+    "duckdb>=1.4.4",
+    "londonaicentre-oncoschema>=2.0.1",
+    "pandas>=2.3.3",
+    "pydantic>=2.12.5",
+    "pydantic-settings[yaml]>=2.12.0",
+    "snowflake-core>=1.11.0",
+    "snowflake-snowpark-python>=1.45.0",
+]
+
+
+[dependency-groups]
+dev = [
+    "fastapi>=0.128.5",
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.0.0",
+    "pytest-mock>=3.15.1",
+    "ruff>=0.15.0",
+    "snowflake-snowpark-python[localtest]>=1.45.0",
+    "tenacity>=9.1.4",
+    "ty>=0.0.15",
+    "types-boto3[s3]>=1.42.44",
+]
+vllm-offline = [
+    "londonaicentre-mesa-local>=2.2.0",
+]
+
+[tool.uv]
+package = true
+
+[project.scripts]
+mesa_runner = "mesa_runnner.main:main"
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle: bad formatting
+    "W",   # pycodestyle: warnings
+    "F",   # pyflakes: common errors
+    "I",   # isort: handles badly setup imports, and orders
+    "B",   # flake8-bugbear: common bugs/footguns
+    "C90", # mccabe cyclomatic complexity: for badly nested conditional statements that makes logic hard to follow
+]
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: marks tests as integration tests (deselect with '-m \"not integration\"')",
+]
+
+[tool.pyrefly]
+project-includes = ["**/*.py*", "**/*.ipynb"]

londonaicentre_mesa_runner-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/PKG-INFO

@@ -0,0 +1,141 @@
+Metadata-Version: 2.4
+Name: londonaicentre-mesa-runner
+Version: 1.0.0
+Summary: A stateless runner / deployment system for MESA models
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: boto3>=1.42.43
+Requires-Dist: duckdb>=1.4.4
+Requires-Dist: londonaicentre-oncoschema>=2.0.1
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pydantic-settings[yaml]>=2.12.0
+Requires-Dist: snowflake-core>=1.11.0
+Requires-Dist: snowflake-snowpark-python>=1.45.0
+
+# MESA Runner
+
+A stateless runner for deploying MESA registered models onto GSTT Infrastructure. It syncs models from S3, reads unprocessed documents from Snowflake, runs inference, and writes results back.
+
+## Requirements
+
+- Python 3.13+
+- [uv](https://github.com/astral-sh/uv) package manager
+
+## Installation
+
+### Remote Inference (Default)
+
+For remote inference via OpenAI-compatible endpoints:
+
+```bash
+uv sync
+```
+
+### Offline Inference (Optional)
+
+For local GPU inference with vLLM, install the optional dependency:
+
+```bash
+uv sync --group vllm-offline
+```
+
+## Configuration
+
+Create a `config.yaml` file (see example below):
+
+### Remote Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    openai_endpoint: "http://localhost:5000/v1"
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+### Offline Inference Example
+
+```yaml
+my_source:
+  model_s3_uri: "s3://aicentre-nlpteam-mesa-build/models/oncoqwen/oncoqwen_1/"
+
+  inference:
+    max_model_len: 18000
+
+  storage:
+    type: snowflake
+    source_database: "str"
+    source_schema: "str"
+    source_table: "str"
+
+    sink_database: "str"
+    sink_schema: "str"
+    sink_table: "str"
+
+    connection_params:
+      account: "str"
+      user: "str"
+      role: "str"
+      password: "str"
+      warehouse: "str"
+      database: "str"
+```
+
+## Usage
+
+```bash
+# Run with default config.yaml
+mesa_runner
+
+# Or specify a config file
+mesa_runner --config /path/to/config.yaml
+
+# Dry run mode (uses dummy data, does not read or write real data)
+mesa_runner --dry-run
+```
+
+The dry run mode is useful for testing the runner without accessing real data sources or sinks. It generates 5 dummy documents by default and logs all write operations instead of executing them.
+
+## Docker
+
+```bash
+# Remote inference (default)
+docker build -t mesa-runner .
+docker run mesa-runner
+
+# Offline inference (includes vLLM)
+docker build --target offline -t mesa-runner:offline .
+docker run --gpus all mesa-runner:offline
+```
+
+## Development
+
+```bash
+# Run linting and tests
+make test
+
+# Auto-fix linting issues
+make fix
+
+# Run tests with coverage
+make cov
+```

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/SOURCES.txt

@@ -0,0 +1,34 @@
+README.md
+pyproject.toml
+src/londonaicentre_mesa_runner.egg-info/PKG-INFO
+src/londonaicentre_mesa_runner.egg-info/SOURCES.txt
+src/londonaicentre_mesa_runner.egg-info/dependency_links.txt
+src/londonaicentre_mesa_runner.egg-info/entry_points.txt
+src/londonaicentre_mesa_runner.egg-info/requires.txt
+src/londonaicentre_mesa_runner.egg-info/top_level.txt
+src/mesa_runner/__init__.py
+src/mesa_runner/inferrer.py
+src/mesa_runner/log_filters.py
+src/mesa_runner/main.py
+src/mesa_runner/runner.py
+src/mesa_runner/adapters/__init__.py
+src/mesa_runner/adapters/dry_run.py
+src/mesa_runner/adapters/filesystem.py
+src/mesa_runner/adapters/io_schemas.py
+src/mesa_runner/adapters/s3.py
+src/mesa_runner/adapters/snowflake.py
+src/mesa_runner/adapters/storage_base.py
+src/mesa_runner/config/__init__.py
+src/mesa_runner/config/config_base.py
+src/mesa_runner/config/filesystem.py
+src/mesa_runner/config/s3.py
+src/mesa_runner/config/snowflake.py
+tests/test_adapter_dry_run.py
+tests/test_adapter_filesystem.py
+tests/test_adapter_s3.py
+tests/test_adapters.py
+tests/test_config.py
+tests/test_inferrer.py
+tests/test_log_filters.py
+tests/test_runtime.py
+tests/test_snowflake_wrapper.py

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mesa_runner = mesa_runnner.main:main

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/requires.txt

@@ -0,0 +1,8 @@
+boto3>=1.42.43
+duckdb>=1.4.4
+londonaicentre-oncoschema>=2.0.1
+pandas>=2.3.3
+pydantic>=2.12.5
+pydantic-settings[yaml]>=2.12.0
+snowflake-core>=1.11.0
+snowflake-snowpark-python>=1.45.0

londonaicentre_mesa_runner-1.0.0/src/londonaicentre_mesa_runner.egg-info/top_level.txt

@@ -0,0 +1 @@
+mesa_runner

londonaicentre_mesa_runner-1.0.0/src/mesa_runner/adapters/__init__.py

@@ -0,0 +1,47 @@
+from mesa_runner.adapters.storage_base import AbstractStorageAdapter
+from mesa_runner.config.config_base import SourceConfig
+from mesa_runner.config.filesystem import FilesystemStorageConfig
+from mesa_runner.config.s3 import S3StorageConfig
+from mesa_runner.config.snowflake import SnowflakeStorageConfig
+
+
+def build_storage_adapter(
+    source_config: SourceConfig, dry_run: bool = False
+) -> AbstractStorageAdapter:
+    if dry_run:
+        from mesa_runner.adapters.dry_run import DryRunStorageAdapter
+
+        return DryRunStorageAdapter()
+
+    if source_config.storage.type == "snowflake":
+        from mesa_runner.adapters.snowflake import SnowflakeWrapper
+
+        # Assertion to help type checking
+        assert isinstance(source_config.storage, SnowflakeStorageConfig)
+
+        return SnowflakeWrapper(
+            params=source_config.storage.connection_params, storage_config=source_config.storage
+        )
+    elif source_config.storage.type == "s3":
+        from mesa_runner.adapters.s3 import S3Adapter
+
+        assert isinstance(source_config.storage, S3StorageConfig)
+
+        # TODO CHANGE - this needs to be passed through more intelligently
+        return S3Adapter(
+            storage_config=source_config.storage,
+            model_name=source_config.model_name,
+        )
+    elif source_config.storage.type == "filesystem":
+        from mesa_runner.adapters.filesystem import FileSystemAdapter
+
+        assert isinstance(source_config.storage, FilesystemStorageConfig)
+
+        # TODO CHANGE - this needs to be passed through more intelligently
+        return FileSystemAdapter(
+            storage_config=source_config.storage,
+            model_name=source_config.model_name,
+        )
+
+    else:
+        raise ValueError(f"Unknown storage type: {source_config.storage.type}")

londonaicentre_mesa_runner-1.0.0/src/mesa_runner/adapters/dry_run.py

@@ -0,0 +1,69 @@
+import logging
+import random
+from datetime import datetime
+from typing import Generator
+
+from mesa_runner.adapters.io_schemas import DocumentInput, InferenceResult
+from mesa_runner.adapters.storage_base import AbstractStorageAdapter
+
+
+class DryRunStorageAdapter(AbstractStorageAdapter):
+    _WORD_POOL: list[str] = [
+        "patient",
+        "diagnosis",
+        "treatment",
+        "history",
+        "medical",
+        "clinical",
+        "symptoms",
+        "examination",
+        "results",
+        "assessment",
+        "procedure",
+        "medication",
+        "therapy",
+        "condition",
+        "report",
+        "findings",
+        "analysis",
+        "consultation",
+        "follow",
+        "up",
+        "review",
+        "outcome",
+        "prognosis",
+        "investigation",
+        "referral",
+    ]
+
+    def __init__(self, document_count: int = 5):
+        self.__document_count: int = document_count
+        self.__logger: logging.Logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        super().__init__()
+
+    def _generate_random_content(self, min_words: int = 10, max_words: int = 100) -> str:
+        return " ".join(random.choices(self._WORD_POOL, k=random.randint(min_words, max_words)))
+
+    def get_unprocessed_document_count(self) -> int:
+        return self.__document_count
+
+    def get_documents(self) -> Generator[DocumentInput, None, None]:
+        for index in range(self.__document_count):
+            yield DocumentInput(
+                document_id=f"dry_run_doc_{index}",
+                document_content=self._generate_random_content(),
+                document_update_dt=datetime.now(),
+                document_source={"source": "dry_run", "index": index},
+            )
+
+    def write_document(self, documents: list[InferenceResult]):
+        self.__logger.info(f"Would write {len(documents)} documents")
+
+    def _check_document_output_writable(self):
+        pass
+
+    def _check_document_source_exists(self):
+        pass
+
+    def flush(self):
+        pass