matyan-backend 0.1.0__tar.gz

matyan_backend-0.1.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: matyan-backend
+Version: 0.1.0
+Summary: Production-ready experiment tracker
+Author-email: Tigran Grigoryan <grigoryan.tigran119@gmail.com>
+Requires-Python: <4,>=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi~=0.115.12
+Requires-Dist: uvicorn~=0.34.2
+Requires-Dist: pydantic~=2.0
+Requires-Dist: pydantic-settings~=2.0
+Requires-Dist: loguru~=0.7.3
+Requires-Dist: websockets~=15.0
+Requires-Dist: RestrictedPython~=8.0
+Requires-Dist: numpy~=2.0
+Requires-Dist: tqdm~=4.0
+Requires-Dist: msgpack~=1.0
+Requires-Dist: frozendict~=2.4
+Requires-Dist: foundationdb==7.3.69
+Requires-Dist: boto3~=1.35
+Requires-Dist: cryptography~=44.0
+Requires-Dist: aiokafka~=0.13
+Requires-Dist: click~=8.0
+Requires-Dist: stream-zip~=0.0.83
+Requires-Dist: matyan-api-models~=0.1.0

matyan_backend-0.1.0/pyproject.toml

@@ -0,0 +1,82 @@
+[build-system]
+requires = ["setuptools>=61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "matyan-backend"
+version = "0.1.0"
+description = "Production-ready experiment tracker"
+readme = "README.md"
+authors = [
+    { name = "Tigran Grigoryan", email = "grigoryan.tigran119@gmail.com" },
+]
+requires-python = ">=3.12, <4"
+dependencies = [
+    "fastapi~=0.115.12",
+    "uvicorn~=0.34.2",
+    "pydantic~=2.0",
+    "pydantic-settings~=2.0",
+    "loguru~=0.7.3",
+    "websockets~=15.0",
+    "RestrictedPython~=8.0",
+    "numpy~=2.0",
+    "tqdm~=4.0",
+    "msgpack~=1.0",
+    "frozendict~=2.4",
+    "foundationdb==7.3.69",
+    "boto3~=1.35",
+    "cryptography~=44.0",
+    "aiokafka~=0.13",
+    "click~=8.0",
+    "stream-zip~=0.0.83",
+    "matyan-api-models~=0.1.0",
+]
+
+
+[project.scripts]
+matyan-backend = "matyan_backend.cli:main"
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+exclude = ["*.ipynb"]
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "FBT001",
+    "FBT002",
+    "D100",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
+    "D105",
+    "D106",
+    "D107",
+    "D203",
+    "D213",
+    "D417",
+    "FBT003",
+    "EXE002",
+    "S311",
+    "TD003",
+    "D205",
+    "PLR2004",
+    "PLR0913",
+    "S101",
+    "S104"
+]
+
+[tool.uv.sources]
+matyan-api-models = { path = "../matyan-api-models", editable = true }
+
+
+[dependency-groups]
+dev = [
+    "pytest~=8.0",
+    "httpx~=0.28",
+    "matyan-api-models",
+    "types-boto3[s3]",
+    "pytest-cov>=7.0.0",
+]

matyan_backend-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

matyan_backend-0.1.0/src/matyan_backend/api/__init__.py

@@ -0,0 +1,3 @@
+from ._main import main_router
+
+__all__ = ["main_router"]

matyan_backend-0.1.0/src/matyan_backend/api/_main.py

@@ -0,0 +1,18 @@
+from fastapi import APIRouter
+
+from .dashboard_apps.views import dashboard_apps_router
+from .dashboards.views import dashboards_router
+from .experiments import rest_router_experiments
+from .projects import rest_router_projects
+from .reports.views import reports_router
+from .runs import rest_router_runs
+from .tags import tags_router
+
+main_router = APIRouter(prefix="/rest")
+main_router.include_router(rest_router_runs)
+main_router.include_router(tags_router)
+main_router.include_router(rest_router_experiments)
+main_router.include_router(rest_router_projects)
+main_router.include_router(dashboards_router, prefix="/dashboards", tags=["dashboards"])
+main_router.include_router(dashboard_apps_router, prefix="/apps", tags=["apps"])
+main_router.include_router(reports_router, prefix="/reports", tags=["reports"])

matyan_backend-0.1.0/src/matyan_backend/api/dashboard_apps/pydantic_models.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class ExploreStateCreateIn(BaseModel):
+    type: str
+    state: dict
+
+
+class ExploreStateUpdateIn(BaseModel):
+    type: str | None = None
+    state: dict | None = None
+
+
+class ExploreStateGetOut(BaseModel):
+    id: str
+    type: str
+    updated_at: float | None = None
+    created_at: float | None = None
+    state: dict
+
+
+ExploreStateListOut = list[ExploreStateGetOut]

matyan_backend-0.1.0/src/matyan_backend/api/dashboard_apps/views.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from matyan_backend.deps import FdbDb  # noqa: TC001
+from matyan_backend.storage import entities
+
+from .pydantic_models import (
+    ExploreStateCreateIn,
+    ExploreStateGetOut,
+    ExploreStateListOut,
+    ExploreStateUpdateIn,
+)
+
+dashboard_apps_router = APIRouter()
+
+
+def _app_to_out(a: dict) -> dict:
+    return {
+        "id": a["id"],
+        "type": a.get("type", ""),
+        "updated_at": a.get("updated_at"),
+        "created_at": a.get("created_at"),
+        "state": a.get("state", {}),
+    }
+
+
+@dashboard_apps_router.get("/", response_model=ExploreStateListOut)
+async def get_apps_api(db: FdbDb) -> list[dict]:
+    apps = entities.list_dashboard_apps(db)
+    return [_app_to_out(a) for a in apps]
+
+
+@dashboard_apps_router.post("/", response_model=ExploreStateGetOut, status_code=201)
+async def create_app_api(body: ExploreStateCreateIn, db: FdbDb) -> dict:
+    a = entities.create_dashboard_app(db, body.type, state=body.state)
+    return _app_to_out(a)
+
+
+@dashboard_apps_router.get("/{app_id}/", response_model=ExploreStateGetOut)
+async def get_app_api(app_id: str, db: FdbDb) -> dict:
+    a = entities.get_dashboard_app(db, app_id)
+    if not a:
+        raise HTTPException(status_code=404)
+    return _app_to_out(a)
+
+
+@dashboard_apps_router.put("/{app_id}/", response_model=ExploreStateGetOut)
+async def update_app_api(app_id: str, body: ExploreStateUpdateIn, db: FdbDb) -> dict:
+    a = entities.get_dashboard_app(db, app_id)
+    if not a:
+        raise HTTPException(status_code=404)
+    updates = {}
+    if body.type is not None:
+        updates["type"] = body.type
+    if body.state is not None:
+        updates["state"] = body.state
+    if updates:
+        entities.update_dashboard_app(db, app_id, **updates)
+    updated = entities.get_dashboard_app(db, app_id)
+    if not updated:
+        raise HTTPException(status_code=404)
+    return _app_to_out(updated)
+
+
+@dashboard_apps_router.delete("/{app_id}/", status_code=204, response_model=None)
+async def delete_app_api(app_id: str, db: FdbDb) -> None:
+    a = entities.get_dashboard_app(db, app_id)
+    if not a:
+        raise HTTPException(status_code=404)
+    entities.delete_dashboard_app(db, app_id)

matyan_backend-0.1.0/src/matyan_backend/api/dashboards/pydantic_models.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class DashboardOut(BaseModel):
+    id: str
+    name: str
+    description: str | None = None
+    app_id: str | None = None
+    app_type: str | None = None
+    updated_at: float | None = None
+    created_at: float | None = None
+
+
+class DashboardUpdateIn(BaseModel):
+    name: str | None = None
+    description: str | None = None
+
+
+class DashboardCreateIn(BaseModel):
+    name: str
+    description: str | None = None
+    app_id: str | None = None

matyan_backend-0.1.0/src/matyan_backend/api/dashboards/views.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from matyan_backend.deps import FdbDb  # noqa: TC001
+from matyan_backend.storage import entities
+
+from .pydantic_models import DashboardCreateIn, DashboardOut, DashboardUpdateIn
+
+dashboards_router = APIRouter()
+
+
+def _dash_to_out(d: dict, db: object | None = None) -> dict:
+    app_type: str | None = None
+    app_id = d.get("app_id")
+    if app_id and db is not None:
+        app = entities.get_dashboard_app(db, app_id)
+        if app:
+            app_type = app.get("type")
+    return {
+        "id": d["id"],
+        "name": d.get("name", ""),
+        "description": d.get("description"),
+        "app_id": app_id,
+        "app_type": app_type,
+        "updated_at": d.get("updated_at"),
+        "created_at": d.get("created_at"),
+    }
+
+
+@dashboards_router.get("/", response_model=list[DashboardOut])
+async def get_dashboards_api(db: FdbDb) -> list[dict]:
+    dashboards = entities.list_dashboards(db)
+    return [_dash_to_out(d, db) for d in dashboards if not d.get("is_archived")]
+
+
+@dashboards_router.post("/", response_model=DashboardOut, status_code=201)
+async def create_dashboard_api(body: DashboardCreateIn, db: FdbDb) -> dict:
+    d = entities.create_dashboard(
+        db,
+        body.name,
+        description=body.description,
+        app_id=str(body.app_id) if body.app_id else None,
+    )
+    return _dash_to_out(d, db)
+
+
+@dashboards_router.get("/{dashboard_id}/", response_model=DashboardOut)
+async def get_dashboard_api(dashboard_id: str, db: FdbDb) -> dict:
+    d = entities.get_dashboard(db, dashboard_id)
+    if not d:
+        raise HTTPException(status_code=404)
+    return _dash_to_out(d, db)
+
+
+@dashboards_router.put("/{dashboard_id}/", response_model=DashboardOut)
+async def update_dashboard_api(dashboard_id: str, body: DashboardUpdateIn, db: FdbDb) -> dict:
+    d = entities.get_dashboard(db, dashboard_id)
+    if not d:
+        raise HTTPException(status_code=404)
+    updates = {}
+    if body.name is not None:
+        updates["name"] = body.name
+    if body.description is not None:
+        updates["description"] = body.description
+    if updates:
+        entities.update_dashboard(db, dashboard_id, **updates)
+    updated = entities.get_dashboard(db, dashboard_id)
+    if not updated:
+        raise HTTPException(status_code=404)
+    return _dash_to_out(updated, db)
+
+
+@dashboards_router.delete("/{dashboard_id}/", status_code=204, response_model=None)
+async def delete_dashboard_api(dashboard_id: str, db: FdbDb) -> None:
+    d = entities.get_dashboard(db, dashboard_id)
+    if not d:
+        raise HTTPException(status_code=404)
+    entities.update_dashboard(db, dashboard_id, is_archived=True)

matyan_backend-0.1.0/src/matyan_backend/api/experiments/__init__.py

@@ -0,0 +1,3 @@
+from ._main import rest_router_experiments
+
+__all__ = ["rest_router_experiments"]

matyan_backend-0.1.0/src/matyan_backend/api/experiments/_main.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+import datetime
+from collections import Counter
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Header, HTTPException, Query
+from pydantic import BaseModel
+
+from matyan_backend.deps import FdbDb, KafkaProducerDep  # noqa: TC001
+from matyan_backend.kafka.producer import emit_control_event
+from matyan_backend.storage import entities
+from matyan_backend.storage.runs import get_run_meta
+
+from ._pydantic_models import (
+    ExperimentActivityApiOut,
+    ExperimentCreateRequest,
+    ExperimentGetOut,
+    ExperimentGetRunsResponse,
+    ExperimentListOut,
+    ExperimentUpdateOut,
+    ExperimentUpdateRequest,
+)
+
+rest_router_experiments = APIRouter(prefix="/experiments")
+
+
+class NoteIn(BaseModel):
+    content: str
+
+
+def _exp_to_out(exp: dict, run_count: int) -> dict:
+    return {
+        "id": exp["id"],
+        "name": exp.get("name", ""),
+        "description": exp.get("description", ""),
+        "run_count": run_count,
+        "archived": exp.get("is_archived", False),
+        "creation_time": exp.get("created_at"),
+    }
+
+
+@rest_router_experiments.get("/", response_model=ExperimentListOut)
+async def get_experiments_list_api(db: FdbDb) -> list[dict]:
+    exps = entities.list_experiments(db)
+    result: list[dict] = []
+    for exp in exps:
+        run_hashes = entities.get_runs_for_experiment(db, exp["id"])
+        result.append(_exp_to_out(exp, run_count=len(run_hashes)))
+    return result
+
+
+@rest_router_experiments.get("/search/", response_model=ExperimentListOut)
+async def search_experiments_by_name_api(
+    db: FdbDb,
+    q: Annotated[str | None, Query()] = None,
+) -> list[dict]:
+    exps = entities.list_experiments(db)
+    search_term = q.strip().lower() if q else ""
+    if search_term:
+        exps = [e for e in exps if search_term in e.get("name", "").lower()]
+    result: list[dict] = []
+    for exp in exps:
+        run_hashes = entities.get_runs_for_experiment(db, exp["id"])
+        result.append(_exp_to_out(exp, run_count=len(run_hashes)))
+    return result
+
+
+@rest_router_experiments.post("/", response_model=ExperimentUpdateOut)
+async def create_experiment_api(request: ExperimentCreateRequest, db: FdbDb) -> dict[str, Any]:
+    try:
+        exp = entities.create_experiment(db, name=request.name.strip())
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    return {"id": exp["id"], "status": "OK"}
+
+
+@rest_router_experiments.get("/{uuid}/", response_model=ExperimentGetOut)
+async def get_experiment_api(uuid: str, db: FdbDb) -> dict:
+    exp = entities.get_experiment(db, uuid)
+    if not exp:
+        raise HTTPException(status_code=404)
+    run_hashes = entities.get_runs_for_experiment(db, uuid)
+    return _exp_to_out(exp, run_count=len(run_hashes))
+
+
+@rest_router_experiments.delete("/{uuid}/")
+async def delete_experiment_api(uuid: str, db: FdbDb, producer: KafkaProducerDep) -> dict[str, str]:
+    exp = entities.get_experiment(db, uuid)
+    if not exp:
+        raise HTTPException(status_code=404)
+    entities.delete_experiment(db, uuid)
+    await emit_control_event(producer, "experiment_deleted", experiment_id=uuid)
+    return {"status": "OK"}
+
+
+@rest_router_experiments.put("/{uuid}/", response_model=ExperimentUpdateOut)
+async def update_experiment_properties_api(
+    uuid: str,
+    exp_in: ExperimentUpdateRequest,
+    db: FdbDb,
+) -> dict[str, str]:
+    exp = entities.get_experiment(db, uuid)
+    if not exp:
+        raise HTTPException(status_code=404)
+
+    updates: dict = {}
+    if exp_in.name:
+        updates["name"] = exp_in.name.strip()
+    if exp_in.description is not None:
+        updates["description"] = exp_in.description
+    if exp_in.archived is not None:
+        updates["is_archived"] = exp_in.archived
+
+    if updates:
+        entities.update_experiment(db, uuid, **updates)
+
+    return {"id": uuid, "status": "OK"}
+
+
+@rest_router_experiments.get("/{uuid}/runs/", response_model=ExperimentGetRunsResponse)
+async def get_experiment_runs_api(
+    uuid: str,
+    db: FdbDb,
+    limit: Annotated[int | None, Query()] = None,
+    offset: Annotated[str | None, Query()] = None,
+) -> dict[str, Any]:
+    exp = entities.get_experiment(db, uuid)
+    if not exp:
+        raise HTTPException(status_code=404)
+
+    run_hashes = entities.get_runs_for_experiment(db, uuid)
+
+    offset_idx = 0
+    if offset and offset in run_hashes:
+        offset_idx = run_hashes.index(offset) + 1
+    run_hashes = run_hashes[offset_idx : offset_idx + limit] if limit else run_hashes[offset_idx:]
+
+    runs = []
+    for rh in run_hashes:
+        meta = get_run_meta(db, rh)
+        runs.append(
+            {
+                "run_id": rh,
+                "name": meta.get("name", ""),
+                "creation_time": meta.get("created_at", 0),
+                "end_time": meta.get("finalized_at"),
+                "archived": meta.get("is_archived", False),
+            },
+        )
+
+    return {"id": uuid, "runs": runs}
+
+
+# ---------------------------------------------------------------------------
+# Notes
+# ---------------------------------------------------------------------------
+
+
+@rest_router_experiments.get("/{exp_id}/note/")
+async def list_note_api(exp_id: str, db: FdbDb) -> list[dict]:
+    exp = entities.get_experiment(db, exp_id)
+    if not exp:
+        raise HTTPException(status_code=404)
+    return entities.list_notes_for_experiment(db, exp_id)
+
+
+@rest_router_experiments.post("/{exp_id}/note/", status_code=201)
+async def create_note_api(exp_id: str, note_in: NoteIn, db: FdbDb) -> dict[str, Any]:
+    exp = entities.get_experiment(db, exp_id)
+    if not exp:
+        raise HTTPException(status_code=404)
+    note = entities.create_note(db, note_in.content.strip(), experiment_id=exp_id)
+    return {"id": note["id"], "created_at": note["created_at"]}
+
+
+@rest_router_experiments.get("/{exp_id}/note/{_id}/")
+async def get_note_api(exp_id: str, _id: str, db: FdbDb) -> dict[str, Any]:  # noqa: ARG001
+    note = entities.get_note(db, _id)
+    if not note:
+        raise HTTPException(status_code=404)
+    return {"id": note["id"], "content": note.get("content", ""), "updated_at": note.get("updated_at")}
+
+
+@rest_router_experiments.put("/{exp_id}/note/{_id}/")
+async def update_note_api(exp_id: str, _id: str, note_in: NoteIn, db: FdbDb) -> dict[str, Any]:  # noqa: ARG001
+    note = entities.get_note(db, _id)
+    if not note:
+        raise HTTPException(status_code=404)
+    entities.update_note(db, _id, content=note_in.content.strip())
+    updated = entities.get_note(db, _id)
+    if not updated:
+        raise HTTPException(status_code=404)
+    return {"id": _id, "content": updated.get("content", ""), "updated_at": updated.get("updated_at")}
+
+
+@rest_router_experiments.delete("/{exp_id}/note/{_id}/")
+async def delete_note_api(exp_id: str, _id: str, db: FdbDb) -> dict[str, str]:  # noqa: ARG001
+    note = entities.get_note(db, _id)
+    if not note:
+        raise HTTPException(status_code=404)
+    entities.delete_note(db, _id)
+    return {"status": "OK"}
+
+
+@rest_router_experiments.get("/{exp_id}/activity/", response_model=ExperimentActivityApiOut)
+async def experiment_runs_activity_api(
+    exp_id: str,
+    db: FdbDb,
+    x_timezone_offset: Annotated[int, Header()] = 0,
+) -> dict[str, Any]:
+    exp = entities.get_experiment(db, exp_id)
+    if not exp:
+        raise HTTPException(status_code=404)
+
+    run_hashes = entities.get_runs_for_experiment(db, exp_id)
+
+    num_runs = 0
+    num_archived = 0
+    num_active = 0
+    activity_counter: Counter[str] = Counter()
+
+    for rh in run_hashes:
+        meta = get_run_meta(db, rh)
+        num_runs += 1
+        if meta.get("is_archived"):
+            num_archived += 1
+        if meta.get("active"):
+            num_active += 1
+
+        created = meta.get("created_at")
+        if created:
+            ts = created - x_timezone_offset * 60
+            day = datetime.datetime.fromtimestamp(ts, tz=datetime.UTC).strftime("%Y-%m-%dT%H:00:00")
+            activity_counter[day] += 1
+
+    return {
+        "num_runs": num_runs,
+        "num_archived_runs": num_archived,
+        "num_active_runs": num_active,
+        "activity_map": dict(activity_counter),
+    }

matyan_backend-0.1.0/src/matyan_backend/api/experiments/_pydantic_models.py

@@ -0,0 +1,52 @@
+from __future__ import annotations  # noqa: I001
+
+
+from pydantic import BaseModel
+
+from uuid import UUID  # noqa: TC003
+
+
+class ExperimentCreateRequest(BaseModel):
+    name: str
+
+
+class ExperimentUpdateRequest(BaseModel):
+    name: str | None = ""
+    description: str | None = ""
+    archived: bool | None = None
+
+
+class ExperimentGetOut(BaseModel):
+    id: UUID
+    name: str
+    description: str | None = ""
+    run_count: int
+    archived: bool
+    creation_time: float | None = None
+
+
+ExperimentListOut = list[ExperimentGetOut]
+
+
+class ExperimentUpdateOut(BaseModel):
+    id: UUID
+    status: str = "OK"
+
+
+class ExperimentGetRunsResponse(BaseModel):
+    class Run(BaseModel):
+        run_id: str
+        name: str
+        creation_time: float
+        end_time: float | None
+        archived: bool
+
+    id: UUID
+    runs: list[Run]
+
+
+class ExperimentActivityApiOut(BaseModel):
+    num_runs: int
+    num_archived_runs: int
+    num_active_runs: int
+    activity_map: dict[str, int] = {"2021-01-01": 54}

matyan_backend-0.1.0/src/matyan_backend/api/projects/__init__.py

@@ -0,0 +1,3 @@
+from ._main import rest_router_projects
+
+__all__ = ["rest_router_projects"]