merlina 2.2.0__tar.gz

merlina-2.2.0/.env.example

@@ -0,0 +1,165 @@
+# Merlina Configuration Template
+# Copy this file to .env and customize for your environment
+
+# ==========================================
+# Server Configuration
+# ==========================================
+
+# Server host (0.0.0.0 allows external connections, 127.0.0.1 is localhost only)
+HOST=0.0.0.0
+
+# Server port
+PORT=8000
+
+# Production domain (leave empty for development)
+# Examples:
+#   http://localhost:8000
+#   https://merlina.example.com
+#   http://192.168.1.100:8000
+# DOMAIN=
+
+# ==========================================
+# Directory Paths (Optional)
+# ==========================================
+
+# Uncomment to customize data directories
+# DATA_DIR=./data
+# MODELS_DIR=./models
+# RESULTS_DIR=./results
+# UPLOADS_DIR=./data/uploads
+# FRONTEND_DIR=./frontend
+
+# Database location
+# DATABASE_PATH=./data/jobs.db
+
+# ==========================================
+# External Services — API Secrets
+# ==========================================
+# Setting these here means clients don't need to send the tokens in HTTP
+# requests. The server will automatically fall back to these values for
+# any training/upload/preload/inference request that omits them. A token
+# sent in a request still overrides the env value.
+#
+# The GET /env/secrets endpoint reports which of these are configured
+# (booleans only — never the values themselves) so the frontend can hide
+# token fields when the server already has them.
+
+# Weights & Biases API Key
+# Get your key from: https://wandb.ai/authorize
+# WANDB_API_KEY=
+
+# HuggingFace Token (required for gated models like Llama, and for Hub uploads)
+# Get your token from: https://huggingface.co/settings/tokens
+# HF_TOKEN=
+
+# ==========================================
+# Training Defaults (Optional)
+# ==========================================
+
+# Default LoRA settings
+# DEFAULT_LORA_R=64
+# DEFAULT_LORA_ALPHA=32
+
+# Default training hyperparameters
+# DEFAULT_LEARNING_RATE=0.000005
+# DEFAULT_BATCH_SIZE=1
+# DEFAULT_GRADIENT_ACCUMULATION_STEPS=16
+# DEFAULT_NUM_EPOCHS=2
+
+# ==========================================
+# Job Queue Configuration
+# ==========================================
+
+# Maximum number of training jobs to run simultaneously
+# Set to 1 for safety (prevents OOM on single GPU)
+# Increase if you have multiple GPUs with sufficient VRAM
+# MAX_CONCURRENT_JOBS=1
+
+# ==========================================
+# System Configuration
+# ==========================================
+
+# GPU selection (comma-separated GPU IDs)
+# Examples:
+#   CUDA_VISIBLE_DEVICES=0        # Use only GPU 0
+#   CUDA_VISIBLE_DEVICES=0,1      # Use GPUs 0 and 1
+#   CUDA_VISIBLE_DEVICES=1,2,3    # Use GPUs 1, 2, and 3
+# CUDA_VISIBLE_DEVICES=
+
+# Logging level (DEBUG, INFO, WARNING, ERROR)
+LOG_LEVEL=INFO
+
+# Offline mode: never contact the HuggingFace Hub. Models and datasets are
+# loaded from the local HF cache or local paths only (sets HF_HUB_OFFLINE
+# and TRANSFORMERS_OFFLINE). Use GET /models/local or the "Local Models"
+# picker in the UI to see what's available offline.
+OFFLINE_MODE=false
+
+# ==========================================
+# Unified-Memory Protection (DGX Spark / Grace-Blackwell)
+# ==========================================
+# On unified-memory machines (NVIDIA GB10 "DGX Spark", GH200/GB200, Jetson)
+# the GPU and system RAM share one physical pool. A training memory spike
+# there doesn't raise a clean CUDA OOM — it can starve the OS and take the
+# whole machine down. Merlina's memory guard detects these systems, caps
+# the CUDA allocator so spikes fail as ordinary OOMs, and runs a watchdog
+# that gracefully stops (or, as a last resort, aborts) training before the
+# OS runs out of RAM. It also writes an fsync'd forensic log to
+# data/memory_guard.log so post-crash investigation has a memory timeline.
+#
+# The guard is a no-op on discrete-GPU systems. Defaults are tuned for a
+# 128GB GB10; you rarely need to change anything.
+
+# Master switch for the guard
+# MEMORY_GUARD_ENABLED=true
+
+# Detection override: auto (default) | true (force on) | false (force off)
+# UNIFIED_MEMORY=auto
+
+# RAM kept off-limits to the CUDA allocator (GB). Capped at 25% of the pool
+# on small boards.
+# MEMORY_GUARD_RESERVE_GB=12
+
+# Free-RAM floors for the watchdog (GB): below the soft floor training is
+# gracefully stopped (checkpoint saved); below the hard floor the job is
+# aborted to keep the machine alive. Scaled down automatically on systems
+# with less total RAM.
+# MEMORY_GUARD_SOFT_FREE_GB=8
+# MEMORY_GUARD_HARD_FREE_GB=3
+
+# Watchdog sampling interval (seconds) and forensic log switch
+# MEMORY_GUARD_POLL_SECONDS=5
+# MEMORY_GUARD_LOG_ENABLED=true
+
+# At the soft floor the watchdog first releases the CUDA allocator's idle
+# cache (e.g. eval leftovers) and only stops training if free RAM is still
+# below the floor — or if pressure returns within this cooldown (seconds).
+# MEMORY_GUARD_RECLAIM_COOLDOWN_SECONDS=60
+
+# ==========================================
+# Security & Limits
+# ==========================================
+
+# CORS origins (comma-separated list or "*" for all)
+# CORS_ORIGINS=*
+
+# Maximum upload file size in MB
+# MAX_UPLOAD_SIZE_MB=500
+
+# ==========================================
+# llama.cpp Integration (Optional)
+# ==========================================
+# Merlina uses llama.cpp for GGUF export/quantization and (optionally)
+# spinning up llama-server for synthetic rejected-response generation.
+#
+# Resolution precedence (first hit wins):
+#   1. LLAMA_CPP_DIR       — path to a llama.cpp checkout (contains
+#                            convert_hf_to_gguf.py and build/bin/)
+#   2. LLAMA_CPP_BIN_DIR   — path to a directory of built binaries only
+#   3. System PATH         — llama-quantize, llama-server, etc. on PATH
+#   4. ./vendor/llama.cpp  — conventional local clone
+#
+# If none of the above are present, llama.cpp-backed features stay disabled
+# in the UI and pre-flight surfaces a warning (never an error).
+# LLAMA_CPP_DIR=/opt/llama.cpp
+# LLAMA_CPP_BIN_DIR=/opt/llama.cpp/build/bin

merlina-2.2.0/.github/workflows/README.md

@@ -0,0 +1,144 @@
+# GitHub Actions Workflows
+
+This directory contains CI/CD workflows for automated testing and quality checks.
+
+## Workflows
+
+### 1. Tests (`tests.yml`)
+
+**Triggers**: Push to main/master/develop, Pull Requests
+
+**What it does**:
+- Runs on Python 3.10 and 3.11 (matrix testing)
+- Installs all dependencies
+- Runs comprehensive API test suite
+- Generates coverage reports
+- Uploads coverage to Codecov
+- Runs code quality checks (ruff, black, mypy)
+
+**Jobs**:
+- `test`: Runs all tests with coverage
+- `lint`: Code quality checks
+- `test-summary`: Posts summary to GitHub
+
+**Duration**: ~3-5 minutes
+
+### 2. Quick Tests (`quick-test.yml`)
+
+**Triggers**: Push to any branch
+
+**What it does**:
+- Fast feedback on every commit
+- Runs only comprehensive API tests
+- Times out after 5 minutes
+- Comments on PR if tests fail
+
+**Duration**: ~2-3 minutes
+
+## Status Badges
+
+Add these badges to your README:
+
+```markdown
+[![Tests](https://github.com/Schneewolf-Labs/Merlina/actions/workflows/tests.yml/badge.svg)](https://github.com/Schneewolf-Labs/Merlina/actions/workflows/tests.yml)
+[![Quick Tests](https://github.com/Schneewolf-Labs/Merlina/actions/workflows/quick-test.yml/badge.svg)](https://github.com/Schneewolf-Labs/Merlina/actions/workflows/quick-test.yml)
+[![codecov](https://codecov.io/gh/Schneewolf-Labs/Merlina/branch/main/graph/badge.svg)](https://codecov.io/gh/Schneewolf-Labs/Merlina)
+```
+
+## Configuration
+
+### Required Secrets
+
+None required for basic testing. Optional:
+
+- `CODECOV_TOKEN`: For private repo coverage reports
+- `HF_TOKEN`: For testing with gated models
+
+### Permissions
+
+Default permissions are sufficient. The workflows need:
+- Read access to repository
+- Write access to checks (automatic)
+
+## Running Locally
+
+To run the same tests locally:
+
+```bash
+# Run all tests
+pytest tests/test_api_comprehensive.py -v
+
+# Run with coverage
+pytest --cov=src --cov=merlina --cov-report=html
+
+# Quick test (matches quick-test.yml)
+pytest tests/test_api_comprehensive.py -v --tb=short -x
+```
+
+## Customization
+
+### Change Python Versions
+
+Edit `tests.yml`:
+
+```yaml
+strategy:
+  matrix:
+    python-version: ['3.10', '3.11', '3.12']
+```
+
+### Add More Tests
+
+Edit the test step:
+
+```yaml
+- name: Run all tests
+  run: |
+    pytest tests/ -v
+```
+
+### Disable Lint Checks
+
+Remove or comment out the `lint` job in `tests.yml`.
+
+## Troubleshooting
+
+### Tests Fail Locally but Pass in CI
+
+- Check Python version: `python --version`
+- Clear pytest cache: `pytest --cache-clear`
+- Install exact dependencies: `pip install -r requirements.txt -r tests/requirements-test.txt`
+
+### Slow Tests
+
+- Use `quick-test.yml` for fast feedback
+- Run specific tests: `pytest tests/test_api_comprehensive.py::TestTrainingEndpoints`
+- Enable parallel testing: `pytest -n auto`
+
+### Coverage Not Uploading
+
+- Check Codecov token (for private repos)
+- Verify coverage.xml is generated
+- Check Codecov action logs
+
+## Best Practices
+
+1. **Keep tests fast**: Use mocking to avoid slow operations
+2. **Fail fast**: Use `-x` flag to stop on first failure
+3. **Matrix testing**: Test multiple Python versions
+4. **Branch protection**: Require tests to pass before merge
+5. **Coverage goals**: Maintain >80% coverage
+
+## Maintenance
+
+These workflows are automatically maintained and updated with the test suite. No manual updates needed unless:
+
+- Adding new Python versions
+- Changing test structure
+- Adding new dependencies
+- Modifying coverage requirements
+
+---
+
+**Last Updated**: 2025-11-16
+**Maintained by**: Merlina Development Team

merlina-2.2.0/.gitignore

@@ -0,0 +1,234 @@
+.env
+
+# Node.js (installed by CI for Playwright tests). Anchored to repo root
+# so legitimate sub-package.json files (e.g. frontend/js/package.json,
+# which marks that directory as ESM for older Node versions) stay tracked.
+node_modules/
+/package.json
+/package-lock.json
+
+# Playwright test artifacts
+test-results/
+playwright-report/
+
+# ML stuff
+wandb/
+
+# Merlina app outputs
+models/
+results/
+data/
+uploads/
+*.safetensors
+*.bin
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env               # Merlina configuration (API keys, secrets)
+.env.*             # Environment-specific configs (.env.production, etc.)
+!.env.example      # Keep example config template
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+.test-venv/