programasweights 0.1.0.dev5__tar.gz → 0.1.0.dev6__tar.gz

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: programasweights
-Version: 0.1.0.dev5
+Version: 0.1.0.dev6
 Summary: Compile natural language specifications into neural programs that run locally via llama.cpp.
 Author-email: ProgramAsWeights <support@programasweights.com>
 License: MIT

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/programasweights/__init__.py

@@ -19,7 +19,7 @@ API reference:
     paw.api_key                API key (set via login() or PAW_API_KEY env var)
 """
 
-__version__ = "0.1.0.dev5"
+__version__ = "0.1.0.dev6"
 
 from .config import get_api_url, get_api_key, set_api_key
 

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "programasweights"
-version = "0.1.0.dev5"
+version = "0.1.0.dev6"
 description = "Compile natural language specifications into neural programs that run locally via llama.cpp."
 readme = "PYPI_README.md"
 requires-python = ">=3.9"

programasweights-0.1.0.dev6/server/alembic/versions/002_namespaced_aliases_hf_url.py

@@ -0,0 +1,37 @@
+"""Add hf_url to programs, widen alias slug for username/name format
+
+Revision ID: 002
+Revises: 001
+Create Date: 2026-03-26
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "002"
+down_revision: Union[str, None] = "001"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("programs", sa.Column("hf_url", sa.Text, nullable=True))
+    op.alter_column("aliases", "slug", type_=sa.String(120))
+    op.drop_constraint("ck_alias_slug_format", "aliases", type_="check")
+    op.create_check_constraint(
+        "ck_alias_slug_format",
+        "aliases",
+        "slug ~ '^[a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*[a-z0-9]$' OR slug ~ '^[a-z0-9][a-z0-9-]*[a-z0-9]$'",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint("ck_alias_slug_format", "aliases", type_="check")
+    op.create_check_constraint(
+        "ck_alias_slug_format",
+        "aliases",
+        "slug ~ '^[a-z0-9][a-z0-9-]*[a-z0-9]$'",
+    )
+    op.alter_column("aliases", "slug", type_=sa.String(50))
+    op.drop_column("programs", "hf_url")

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/server/api/models/orm.py

@@ -51,6 +51,7 @@ class Program(Base):
     tags: Mapped[list[str] | None] = mapped_column(ARRAY(Text), nullable=True)
     adapter_path: Mapped[str | None] = mapped_column(Text, nullable=True)
     paw_path: Mapped[str | None] = mapped_column(Text, nullable=True)
+    hf_url: Mapped[str | None] = mapped_column(Text, nullable=True)
     downloads: Mapped[int] = mapped_column(Integer, default=0)
     upvotes: Mapped[int] = mapped_column(Integer, default=0)
     downvotes: Mapped[int] = mapped_column(Integer, default=0)
@@ -72,7 +73,7 @@ class Program(Base):
 class Alias(Base):
     __tablename__ = "aliases"
 
-    slug: Mapped[str] = mapped_column(String(50), primary_key=True)
+    slug: Mapped[str] = mapped_column(String(120), primary_key=True)
     program_id: Mapped[str] = mapped_column(
         Text, ForeignKey("programs.id"), nullable=False
     )

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/server/api/routes/compile.py

@@ -10,7 +10,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from api.dependencies import get_current_user
-from api.models.database import get_db
+from api.models.database import get_db, get_session_factory
 from api.models.orm import CompileLog, Program, User
 from api.models.schemas import CompileRequest, CompileResponse, CompileStatusResponse, CompileTimings
 
@@ -81,9 +81,9 @@ async def submit_compile(
         "error": None,
     }
 
-    # For now, run compilation synchronously (will switch to Celery for async)
     import asyncio
     from api.services.compile_service import SpecTooLongError, get_compile_service
+    from api.services.storage_service import upload_paw_to_hf, delete_lora_params
 
     try:
         service = get_compile_service()
@@ -119,6 +119,29 @@ async def submit_compile(
         ))
         await db.commit()
 
+        # Background: upload .paw to HF + delete lora_params.pt
+        paw_path = result.get("paw_path")
+
+        async def _background_hf_upload():
+            try:
+                from pathlib import Path
+                if paw_path and Path(paw_path).exists():
+                    hf_url = await asyncio.to_thread(upload_paw_to_hf, pid, paw_path)
+                    async with get_session_factory()() as bg_db:
+                        prog = await bg_db.get(Program, pid)
+                        if prog:
+                            prog.hf_url = hf_url
+                            await bg_db.commit()
+
+                adapter_dir = Path(paw_path).parent if paw_path else None
+                if adapter_dir and adapter_dir.exists():
+                    delete_lora_params(adapter_dir)
+            except Exception:
+                logger.exception("background_hf_upload_failed", program_id=pid)
+
+        if not result.get("cached"):
+            asyncio.create_task(_background_hf_upload())
+
         t = result.get("timings", {})
         def _ms(key: str) -> float | None:
             v = t.get(key)

{programasweights-0.1.0.dev5 → programasweights-0.1.0.dev6}/server/api/routes/programs.py

@@ -6,7 +6,7 @@ import re
 from pathlib import Path
 
 from fastapi import APIRouter, Depends, HTTPException, Request
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, RedirectResponse
 from sqlalchemy import func, select, or_
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -14,23 +14,38 @@ from api.config import get_settings
 from api.dependencies import get_current_user, require_user
 from api.models.database import get_db
 from api.models.orm import Alias, Case, Program, User, Vote
+from api.services.storage_service import get_hf_url
 
 router = APIRouter()
 
-SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$")
+SLUG_PART_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,48}[a-z0-9]$")
+SYSTEM_NAMESPACE = "programasweights"
 
 
 async def resolve_program(id_or_slug: str, db: AsyncSession) -> Program | None:
-    """Resolve a program by hash ID or alias slug."""
+    """Resolve a program by hash ID or alias slug.
+    Bare slugs (no /) are tried as programasweights/slug first."""
     result = await db.execute(select(Program).where(Program.id == id_or_slug))
     program = result.scalar_one_or_none()
     if program:
         return program
 
+    slug = id_or_slug
     result = await db.execute(
-        select(Program).join(Alias).where(Alias.slug == id_or_slug)
+        select(Program).join(Alias).where(Alias.slug == slug)
     )
-    return result.scalar_one_or_none()
+    program = result.scalar_one_or_none()
+    if program:
+        return program
+
+    if "/" not in slug:
+        full_slug = f"{SYSTEM_NAMESPACE}/{slug}"
+        result = await db.execute(
+            select(Program).join(Alias).where(Alias.slug == full_slug)
+        )
+        return result.scalar_one_or_none()
+
+    return None
 
 
 # --- Programs CRUD ---
@@ -95,10 +110,16 @@ async def list_programs(
     }
 
 
-@router.get("/programs/resolve/{slug}")
+@router.get("/programs/resolve/{slug:path}")
 async def resolve_slug(slug: str, db: AsyncSession = Depends(get_db)):
     result = await db.execute(select(Alias).where(Alias.slug == slug))
     alias = result.scalar_one_or_none()
+
+    if not alias and "/" not in slug:
+        full_slug = f"{SYSTEM_NAMESPACE}/{slug}"
+        result = await db.execute(select(Alias).where(Alias.slug == full_slug))
+        alias = result.scalar_one_or_none()
+
     if not alias:
         raise HTTPException(status_code=404, detail={
             "error": "alias_not_found",
@@ -134,6 +155,7 @@ async def get_program(id_or_slug: str, db: AsyncSession = Depends(get_db)):
         "public": program.public,
         "aliases": aliases,
         "download_url": f"/api/v1/programs/{program.id}/download",
+        "hf_url": program.hf_url,
         "created_at": program.created_at.isoformat() if program.created_at else None,
     }
 
@@ -142,29 +164,25 @@ async def get_program(id_or_slug: str, db: AsyncSession = Depends(get_db)):
 async def download_program(id_or_slug: str, db: AsyncSession = Depends(get_db)):
     program = await resolve_program(id_or_slug, db)
     if not program:
-        settings = get_settings()
-        paw_file = Path(settings.STORAGE_PATH) / id_or_slug / f"{id_or_slug}.paw"
-        if paw_file.exists():
-            return FileResponse(
-                path=str(paw_file),
-                media_type="application/octet-stream",
-                filename=f"{id_or_slug}.paw",
-            )
         raise HTTPException(status_code=404, detail=f"Program '{id_or_slug}' not found")
 
     program.downloads = (program.downloads or 0) + 1
     await db.commit()
 
+    if program.hf_url:
+        return RedirectResponse(url=program.hf_url, status_code=302)
+
     settings = get_settings()
     paw_file = Path(settings.STORAGE_PATH) / program.id / f"{program.id}.paw"
-    if not paw_file.exists():
-        raise HTTPException(status_code=404, detail=f".paw file not found for '{program.id}'")
+    if paw_file.exists():
+        return FileResponse(
+            path=str(paw_file),
+            media_type="application/octet-stream",
+            filename=f"{program.id}.paw",
+        )
 
-    return FileResponse(
-        path=str(paw_file),
-        media_type="application/octet-stream",
-        filename=f"{program.id}.paw",
-    )
+    hf_url = get_hf_url(program.id)
+    return RedirectResponse(url=hf_url, status_code=302)
 
 
 # --- Voting ---
@@ -314,26 +332,29 @@ async def create_alias(
         raise HTTPException(status_code=404, detail="Program not found")
 
     body = await request.json()
-    slug = body.get("slug", "").lower().strip()
+    raw_slug = body.get("slug", "").lower().strip()
 
-    if not SLUG_RE.match(slug):
+    if not SLUG_PART_RE.match(raw_slug):
         raise HTTPException(status_code=422, detail={
             "error": "invalid_slug",
-            "message": "Slug must be 3-50 chars, lowercase alphanumeric and hyphens, "
+            "message": "Name must be 2-50 chars, lowercase alphanumeric and hyphens, "
                        "start and end with alphanumeric.",
         })
 
-    existing = await db.execute(select(Alias).where(Alias.slug == slug))
+    username = user.username or f"user-{user.id}"
+    full_slug = f"{username}/{raw_slug}"
+
+    existing = await db.execute(select(Alias).where(Alias.slug == full_slug))
     if existing.scalar_one_or_none():
         raise HTTPException(status_code=409, detail={
             "error": "slug_taken",
-            "message": f"The name '{slug}' is already taken.",
+            "message": f"You already have a program named '{raw_slug}'.",
         })
 
-    alias = Alias(slug=slug, program_id=program_id, owner_id=user.id)
+    alias = Alias(slug=full_slug, program_id=program_id, owner_id=user.id)
     db.add(alias)
     await db.commit()
-    return {"slug": slug, "program_id": program_id}
+    return {"slug": full_slug, "program_id": program_id}
 
 
 @router.delete("/programs/{program_id}/alias/{slug}")

programasweights-0.1.0.dev6/server/api/services/storage_service.py

@@ -0,0 +1,114 @@
+"""
+Storage service: upload .paw files to HuggingFace, manage local cleanup.
+"""
+
+import os
+import time
+from pathlib import Path
+
+import structlog
+
+logger = structlog.get_logger()
+
+HF_REPO_ID = "yuntian-deng/paw-programs"
+HF_BASE_URL = f"https://huggingface.co/{HF_REPO_ID}/resolve/main"
+
+
+def get_hf_url(program_id: str) -> str:
+    return f"{HF_BASE_URL}/{program_id}.paw"
+
+
+def upload_paw_to_hf(program_id: str, paw_path: str) -> str:
+    """Upload a .paw file to HuggingFace. Returns the download URL."""
+    from huggingface_hub import HfApi
+
+    api = HfApi()
+    url = get_hf_url(program_id)
+
+    try:
+        api.upload_file(
+            path_or_fileobj=paw_path,
+            path_in_repo=f"{program_id}.paw",
+            repo_id=HF_REPO_ID,
+            repo_type="model",
+        )
+        logger.info("hf_upload_success", program_id=program_id, url=url)
+    except Exception:
+        logger.exception("hf_upload_failed", program_id=program_id)
+        raise
+
+    return url
+
+
+def delete_lora_params(program_dir: Path) -> int:
+    """Delete lora_params.pt if it exists. Returns bytes freed."""
+    pt_file = program_dir / "lora_params.pt"
+    if pt_file.exists():
+        size = pt_file.stat().st_size
+        pt_file.unlink()
+        logger.info("deleted_lora_params", dir=str(program_dir), freed_mb=round(size / 1024 / 1024, 1))
+        return size
+    return 0
+
+
+def cleanup_cold_programs(
+    storage_path: Path,
+    max_age_days: int = 30,
+    keep_peft: bool = True,
+) -> dict:
+    """Remove local files for programs that have HF backups and haven't been accessed recently.
+
+    Args:
+        storage_path: Path to /data/programs
+        max_age_days: Programs not accessed in this many days are eligible
+        keep_peft: If True, keep adapter_model.safetensors for vLLM inference
+    """
+    from api.models.database import get_session_factory
+    from api.models.orm import Program
+    from sqlalchemy import select
+    import asyncio
+
+    cutoff = time.time() - max_age_days * 86400
+    stats = {"checked": 0, "cleaned": 0, "bytes_freed": 0, "skipped_no_hf": 0}
+
+    async def _cleanup():
+        factory = get_session_factory()
+        async with factory() as db:
+            result = await db.execute(
+                select(Program).where(Program.hf_url.isnot(None))
+            )
+            programs = result.scalars().all()
+
+            for prog in programs:
+                stats["checked"] += 1
+                prog_dir = storage_path / prog.id
+
+                if not prog_dir.exists():
+                    continue
+
+                paw_file = prog_dir / f"{prog.id}.paw"
+                if not paw_file.exists():
+                    continue
+
+                last_access = paw_file.stat().st_atime
+                if last_access > cutoff:
+                    continue
+
+                for fname in ["lora_params.pt", f"{prog.id}.paw", "adapter.gguf"]:
+                    f = prog_dir / fname
+                    if f.exists():
+                        stats["bytes_freed"] += f.stat().st_size
+                        f.unlink()
+
+                if not keep_peft:
+                    for fname in ["adapter_model.safetensors"]:
+                        f = prog_dir / fname
+                        if f.exists():
+                            stats["bytes_freed"] += f.stat().st_size
+                            f.unlink()
+
+                stats["cleaned"] += 1
+                logger.info("cleaned_program", program_id=prog.id)
+
+    asyncio.run(_cleanup())
+    return stats

programasweights-0.1.0.dev6/server/scripts/cleanup_storage.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""
+Cleanup script: delete lora_params.pt from all programs, optionally
+clean cold programs that have HF backups.
+
+Usage:
+    # Delete all lora_params.pt (always safe)
+    python scripts/cleanup_storage.py --delete-pt
+
+    # Clean cold programs (30+ days, have HF backup)
+    python scripts/cleanup_storage.py --clean-cold --max-age-days 30
+
+    # Both
+    python scripts/cleanup_storage.py --delete-pt --clean-cold
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+
+def delete_all_lora_params(storage_path: Path):
+    """Delete lora_params.pt from all program directories."""
+    total_freed = 0
+    count = 0
+    for d in storage_path.iterdir():
+        if not d.is_dir():
+            continue
+        pt_file = d / "lora_params.pt"
+        if pt_file.exists():
+            size = pt_file.stat().st_size
+            pt_file.unlink()
+            total_freed += size
+            count += 1
+    print(f"Deleted {count} lora_params.pt files, freed {total_freed / 1024 / 1024:.0f} MB")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="PAW storage cleanup")
+    parser.add_argument("--storage-path", default="/data/programs")
+    parser.add_argument("--delete-pt", action="store_true",
+                        help="Delete all lora_params.pt files (always safe)")
+    parser.add_argument("--clean-cold", action="store_true",
+                        help="Clean cold programs with HF backups")
+    parser.add_argument("--max-age-days", type=int, default=30)
+    args = parser.parse_args()
+
+    storage = Path(args.storage_path)
+    if not storage.exists():
+        print(f"Storage path {storage} does not exist")
+        return
+
+    if args.delete_pt:
+        delete_all_lora_params(storage)
+
+    if args.clean_cold:
+        from api.services.storage_service import cleanup_cold_programs
+        stats = cleanup_cold_programs(storage, max_age_days=args.max_age_days)
+        print(f"Cleanup: checked={stats['checked']}, cleaned={stats['cleaned']}, "
+              f"freed={stats['bytes_freed'] / 1024 / 1024:.0f} MB")
+
+
+if __name__ == "__main__":
+    main()

programasweights-0.1.0.dev6/server/tests/test_storage.py

@@ -0,0 +1,134 @@
+"""
+Tests for the storage + alias + HF upload system.
+TDD: these tests define the expected behavior BEFORE implementation.
+"""
+
+import pytest
+
+# ── Alias resolution ──
+
+class TestAliasResolution:
+    """Namespaced aliases: username/slug format with programasweights/ shorthand."""
+
+    async def test_bare_name_resolves_to_programasweights_namespace(self, client):
+        """'email-triage' should resolve to programasweights/email-triage."""
+        resp = await client.get("/api/v1/programs/resolve/email-triage")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert "program_id" in data
+        assert data["slug"] == "programasweights/email-triage"
+
+    async def test_namespaced_slug_resolves(self, client):
+        """'da03/my-program' should resolve to the correct program."""
+        resp = await client.get("/api/v1/programs/resolve/da03/my-program")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert "program_id" in data
+
+    async def test_unknown_slug_returns_404(self, client):
+        resp = await client.get("/api/v1/programs/resolve/nonexistent/nope")
+        assert resp.status_code == 404
+
+    async def test_create_alias_requires_auth(self, client):
+        resp = await client.post(
+            "/api/v1/programs/some_program_id/alias",
+            json={"slug": "my-cool-program"},
+        )
+        assert resp.status_code == 401
+
+    async def test_create_alias_adds_username_prefix(self, client, auth_headers):
+        """Creating alias 'my-prog' as user 'da03' should store 'da03/my-prog'."""
+        resp = await client.post(
+            "/api/v1/programs/some_program_id/alias",
+            json={"slug": "my-prog"},
+            headers=auth_headers,
+        )
+        assert resp.status_code in (200, 201)
+        data = resp.json()
+        assert data["slug"] == "da03/my-prog"
+
+    async def test_duplicate_slug_returns_409(self, client, auth_headers):
+        """Same user can't create the same alias twice."""
+        await client.post(
+            "/api/v1/programs/some_program_id/alias",
+            json={"slug": "unique-name"},
+            headers=auth_headers,
+        )
+        resp = await client.post(
+            "/api/v1/programs/some_program_id/alias",
+            json={"slug": "unique-name"},
+            headers=auth_headers,
+        )
+        assert resp.status_code == 409
+
+
+# ── Download redirect ──
+
+class TestDownloadRedirect:
+    """Download endpoint should redirect to HuggingFace URL."""
+
+    async def test_download_redirects_to_hf(self, client):
+        """GET /programs/{id}/download should return 302 to HF URL."""
+        resp = await client.get(
+            "/api/v1/programs/9b57fd6fccf77885400e/download",
+            follow_redirects=False,
+        )
+        assert resp.status_code in (302, 307)
+        location = resp.headers.get("location", "")
+        assert "huggingface.co" in location
+        assert "9b57fd6fccf77885400e" in location
+
+    async def test_download_by_slug_redirects(self, client):
+        """GET /programs/email-triage/download should also redirect."""
+        resp = await client.get(
+            "/api/v1/programs/email-triage/download",
+            follow_redirects=False,
+        )
+        assert resp.status_code in (302, 307)
+
+    async def test_download_unknown_returns_404(self, client):
+        resp = await client.get("/api/v1/programs/nonexistent_id/download")
+        assert resp.status_code == 404
+
+
+# ── HF upload ──
+
+class TestHFUpload:
+    """After compile, .paw should be uploaded to HuggingFace."""
+
+    async def test_compile_sets_hf_url(self, client, auth_headers):
+        """After successful compile, program should have hf_url set in DB."""
+        resp = await client.post(
+            "/api/v1/compile",
+            json={"spec": "Classify text as spam or not spam"},
+            headers=auth_headers,
+        )
+        if resp.status_code == 202:
+            data = resp.json()
+            if data["status"] == "ready":
+                program_id = data["program_id"]
+                prog_resp = await client.get(f"/api/v1/programs/{program_id}")
+                prog_data = prog_resp.json()
+                assert prog_data.get("hf_url") is not None
+                assert "huggingface.co" in prog_data["hf_url"]
+
+
+# ── Cleanup ──
+
+class TestCleanup:
+    """Cleanup should remove local files for cold programs that have HF backups."""
+
+    def test_lora_params_never_needed(self):
+        """lora_params.pt should never exist after compile+cleanup."""
+        import os
+        from pathlib import Path
+        programs_dir = Path("/data/programs")
+        if programs_dir.exists():
+            for d in programs_dir.iterdir():
+                pt_file = d / "lora_params.pt"
+                if pt_file.exists():
+                    size_mb = pt_file.stat().st_size / 1024 / 1024
+                    pytest.fail(
+                        f"lora_params.pt still exists in {d.name} ({size_mb:.0f} MB). "
+                        "Should be deleted after GGUF conversion."
+                    )