paperus 0.0.8__py3-none-any.whl

paperus/__init__.py

@@ -0,0 +1,5 @@
+"""paperus: a self-hosted research library for groups."""
+
+from importlib.metadata import version
+
+__version__ = version("paperus")

paperus/access.py

@@ -0,0 +1,202 @@
+"""Who may read and change which library.
+
+A person reads their own library and those of the groups they belong to,
+including every group inside those (membership is inherited downwards, and a
+subgroup can only raise a role). Site administrators manage users and groups
+but read no library because of it. All of this is worked out once per
+request (``Access.of``) from the group tree, which is small, and the person's
+memberships; lists and searches then filter by library ID in SQL.
+"""
+
+import uuid
+from dataclasses import dataclass, field
+
+from sqlalchemy import ColumnElement, select
+from sqlalchemy.orm import Session
+
+from paperus.models import (
+    ROLE_RANK,
+    Group,
+    Item,
+    Library,
+    Membership,
+    Note,
+    Role,
+    User,
+)
+
+
+class Forbidden(Exception):
+    """The person may see this but not change it (HTTP 403)."""
+
+
+class NotVisible(Exception):
+    """The person may not see this; answered like a missing object (404)."""
+
+
+@dataclass
+class Access:
+    user: User
+    own_library: uuid.UUID
+    roles: dict[uuid.UUID, Role] = field(default_factory=dict)
+    """Effective role per group (direct or inherited)."""
+    group_libraries: dict[uuid.UUID, uuid.UUID] = field(default_factory=dict)
+    """Library of each group the person belongs to."""
+    own_allowed: bool = True
+    """False for an access token limited to some groups' libraries."""
+    site_admin: bool = False
+    """A site administrator acting in person (never through a token, which
+    gets no administration)."""
+
+    @classmethod
+    def of(
+        cls,
+        db: Session,
+        user: User,
+        only: list[uuid.UUID] | None = None,
+        *,
+        token: bool = False,
+    ) -> "Access":
+        """What ``user`` may do; with ``only`` (an access token's owner limit),
+        nothing outside these libraries; ``token``: through an access token."""
+        own = db.scalar(select(Library.id).where(Library.user_id == user.id))
+        if own is None:  # users from before libraries existed get one now
+            library = Library(user_id=user.id)
+            db.add(library)
+            db.flush()
+            own = library.id
+        parents: dict[uuid.UUID, uuid.UUID | None] = {
+            group_id: parent
+            for group_id, parent in db.execute(select(Group.id, Group.parent_id))
+        }
+        direct = {
+            group_id: Role(role)
+            for group_id, role in db.execute(
+                select(Membership.group_id, Membership.role).where(
+                    Membership.user_id == user.id
+                )
+            )
+        }
+        roles: dict[uuid.UUID, Role] = {}
+        for group_id in parents:
+            # The highest role on the way up to the top.
+            best: Role | None = None
+            current: uuid.UUID | None = group_id
+            while current is not None:
+                role = direct.get(current)
+                if role and (best is None or ROLE_RANK[role] > ROLE_RANK[best]):
+                    best = role
+                current = parents.get(current)
+            if best is not None:
+                roles[group_id] = best
+        libraries = {
+            group_id: library_id
+            for group_id, library_id in db.execute(
+                select(Library.group_id, Library.id).where(Library.group_id.in_(roles))
+            )
+            if group_id is not None
+        }
+        site_admin = user.is_admin and not token
+        if only is None:
+            return cls(user, own, roles, libraries, site_admin=site_admin)
+        allowed = set(only)
+        libraries = {g: lib for g, lib in libraries.items() if lib in allowed}
+        roles = {g: r for g, r in roles.items() if g in libraries}
+        return cls(
+            user,
+            own,
+            roles,
+            libraries,
+            own_allowed=own in allowed,
+            site_admin=site_admin,
+        )
+
+    @property
+    def readable(self) -> list[uuid.UUID]:
+        """Libraries this person may read: their own and their groups'."""
+        own = [self.own_library] if self.own_allowed else []
+        return [*own, *self.group_libraries.values()]
+
+    def role_in_library(self, library_id: uuid.UUID) -> Role | None:
+        for group_id, group_library in self.group_libraries.items():
+            if group_library == library_id:
+                return self.roles[group_id]
+        return None
+
+    def can_read(self, library_id: uuid.UUID) -> bool:
+        """The person's own library (unless a token is limited to others) or
+        one of their groups' (only those a limited token allows)."""
+        if library_id == self.own_library:
+            return self.own_allowed
+        return library_id in self.group_libraries.values()
+
+    def can_write(self, library_id: uuid.UUID) -> bool:
+        """Add, change and delete items, tags and collections."""
+        if library_id == self.own_library:
+            return self.own_allowed
+        role = self.role_in_library(library_id)
+        return role is not None and ROLE_RANK[role] >= ROLE_RANK[Role.MEMBER]
+
+    @property
+    def writable(self) -> list[uuid.UUID]:
+        return [library for library in self.readable if self.can_write(library)]
+
+    def is_group_admin(self, group_id: uuid.UUID) -> bool:
+        return self.roles.get(group_id) == Role.ADMIN
+
+    def require_write(self, library_id: uuid.UUID) -> None:
+        if not self.can_write(library_id):
+            raise Forbidden("readers cannot change this library")
+
+    # Items have one owner (``Item.library_id``): a person or a group.
+
+    def can_read_item(self, item: Item) -> bool:
+        return self.can_read(item.library_id)
+
+    def can_write_item(self, item: Item) -> bool:
+        return self.can_write(item.library_id)
+
+    def require_write_item(self, item: Item) -> None:
+        if not self.can_write_item(item):
+            raise Forbidden("readers cannot change this item")
+
+    def item_filter(
+        self, libraries: list[uuid.UUID] | None = None
+    ) -> ColumnElement[bool]:
+        """Items owned by these (by default: everything the person reads)."""
+        return Item.library_id.in_(self.readable if libraries is None else libraries)
+
+    def may_tag(self, library_id: uuid.UUID, item: Item) -> bool:
+        """Whether the person may put a tag of ``library_id`` on (or take it
+        off) the item: their private tags on any item they see; a group's
+        tags (they read) on any item they may change, e.g. their own papers
+        with the group's vocabulary."""
+        if library_id == self.own_library:
+            return self.own_allowed and self.can_read_item(item)
+        return self.can_read(library_id) and self.can_write_item(item)
+
+    def may_hold(self, library_id: uuid.UUID, item: Item | Note) -> bool:
+        """Whether a collection of ``library_id`` may hold the item (or note):
+        a group's only the group's own, a person's private ones anything the
+        person can read."""
+        if library_id == self.own_library:
+            return self.own_allowed and self.can_read(item.library_id)
+        return library_id == item.library_id
+
+    def target_library(self, requested: uuid.UUID | None) -> uuid.UUID:
+        """Where something new goes: the library asked for (if the person may
+        add to it), else their default library, else their own."""
+        if requested is not None:
+            if not self.can_read(requested):
+                raise NotVisible("library not found")
+            self.require_write(requested)
+            return requested
+        default = self.user.default_library_id
+        if default is not None and self.can_write(default):
+            return default
+        if self.own_allowed:
+            return self.own_library
+        writable = self.writable
+        if not writable:
+            raise Forbidden("this token may not add anything")
+        return writable[0]

paperus/api/__init__.py

@@ -0,0 +1 @@
+"""HTTP API routes."""

paperus/api/admin.py

@@ -0,0 +1,364 @@
+"""Admin page: settings, the ML service, progress and jobs."""
+
+from datetime import UTC, datetime
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+
+from paperus import auth, integrity, metrics, oidc, readable, system, usage
+from paperus.api.auth import OIDC_CALLBACK, public_base
+from paperus.api.deps import DB, CurrentUser, request_token
+from paperus.jobs import (
+    queue_embedding_sweep,
+    queue_empty_trash,
+    queue_integrity_check,
+    queue_ml_configuration,
+    queue_readable_copy,
+    queue_text_sweep,
+)
+from paperus.ml import MLClient, MLLoading, MLRejected, MLUnavailable
+from paperus.models import LoginSession, User
+from paperus.processing import reread_all
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+
+def current_admin(user: CurrentUser) -> User:
+    if not user.is_admin:
+        raise HTTPException(403, "admins only")
+    return user
+
+
+Admin = Annotated[User, Depends(current_admin)]
+
+
+class MLService(BaseModel):
+    url: str
+    ok: bool
+    error: str | None
+
+
+class MLModel(BaseModel):
+    name: str
+    task: str
+    dimensions: int
+    max_tokens: int
+    loaded: bool
+    cached_bytes: int | None
+    # For choosing (older ML services do not send these).
+    description: str = ""
+    licence: str = ""
+    download_mb: int = 0
+    scores: dict[str, float] = {}
+    """Published retrieval scores (nDCG@10, out of 100), by benchmark."""
+    benchmarks: dict[str, str] = {}
+    """What the keys of ``scores`` mean."""
+
+
+class MLSpeed(BaseModel):
+    model: str
+    loading: bool = False
+    """The model is still downloading or loading: ask again in a while."""
+    passages: int = 0
+    seconds: float = 0
+    passages_per_second: float = 0
+    url: str | None = None
+    """The ML service that measured it."""
+
+
+class MLServiceState(BaseModel):
+    url: str
+    threads: int
+    keep: list[str]
+    cores: int
+    """CPU cores this ML service may use."""
+    loaded: list[str]
+    loading: list[str] = []
+
+
+class MLStatus(BaseModel):
+    services: list[MLService]
+    models: list[MLModel]
+    """Empty if no service answers."""
+    service: MLServiceState | None = None
+    """The first service's settings and cores (None for older services)."""
+
+
+class Stats(BaseModel):
+    items: int
+    files: int
+    files_pending: int
+    """Files whose text (and, for uploads, metadata) is still being read."""
+    chunks: int
+    """Passages searchable by meaning (reference lists are left out)."""
+    embedded: int
+    """Chunks with a vector from the current embedding model."""
+    embedding_model: str
+    jobs: dict[str, dict[str, int]]
+    """Background jobs per queue and status (todo, doing, failed, …)."""
+    storage: usage.Usage
+    """Space the library and the backups take, and what is free."""
+
+
+def _ml(request: Request) -> MLClient:
+    ml: MLClient = request.app.state.ml
+    return ml
+
+
+def _models(ml: MLClient) -> list[MLModel]:
+    return [MLModel.model_validate(m) for m in ml.models()]
+
+
+@router.get("/config")
+def get_config(request: Request, db: DB, admin: Admin) -> system.SystemConfig:
+    return system.load_config(db, request.app.state.settings)
+
+
+@router.put("/config")
+def set_config(
+    body: system.SystemConfig, request: Request, db: DB, admin: Admin
+) -> system.SystemConfig:
+    """Save settings. A new embedding model must be one the ML service
+    offers; the library is then re-embedded in the background, while keyword
+    search keeps working and search by meaning covers more and more."""
+    current = system.load_config(db, request.app.state.settings)
+    model_changed = body.embedding_model != current.embedding_model
+    if model_changed:
+        try:
+            known = {m.name for m in _models(_ml(request)) if m.task == "embedding"}
+        except (MLUnavailable, MLRejected) as error:
+            raise HTTPException(
+                503, "ML service unreachable; cannot check the model"
+            ) from error
+        if body.embedding_model not in known:
+            raise HTTPException(
+                422, f"the ML service has no model {body.embedding_model!r}"
+            )
+    system.save_config(db, body)
+    db.commit()
+    system.apply(body, request.app.state.http)
+    if model_changed:
+        queue_embedding_sweep(request.app.state.jobs)
+    copy = ("copy_enabled", "copy_template", "copy_links")
+    if body.copy_enabled and any(
+        getattr(body, name) != getattr(current, name) for name in copy
+    ):
+        queue_readable_copy(request.app.state.jobs)
+    if model_changed or body.ml_threads != current.ml_threads:
+        # The worker sends it at once (first in its queue), and every minute.
+        queue_ml_configuration(request.app.state.jobs)
+    return body
+
+
+@router.get("/ml")
+def ml_status(request: Request, admin: Admin) -> MLStatus:
+    ml = _ml(request)
+    services = [MLService(url=u, ok=e is None, error=e) for u, e in ml.status()]
+    try:
+        models = _models(ml)
+    except (MLUnavailable, MLRejected):
+        models = []
+    try:
+        service = MLServiceState.model_validate(ml.service_config())
+    except (MLUnavailable, MLRejected, ValueError):
+        service = None
+    return MLStatus(services=services, models=models, service=service)
+
+
+@router.post("/ml/models/{name}/speed")
+def measure_ml_speed(name: str, request: Request, admin: Admin) -> MLSpeed:
+    """Time a model on this installation's ML service with sample passages.
+    A model not downloaded yet starts downloading: the answer then says
+    ``loading``; ask again in a while."""
+    try:
+        return MLSpeed.model_validate(_ml(request).measure_speed(name))
+    except MLLoading:
+        return MLSpeed(model=name, loading=True)
+    except MLRejected as error:
+        raise HTTPException(
+            404 if error.status == 404 else 502, error.detail
+        ) from error
+    except MLUnavailable as error:
+        raise HTTPException(503, f"ML service unreachable: {error}") from error
+
+
+@router.delete("/ml/models/{name}", status_code=204)
+def delete_ml_model(name: str, request: Request, db: DB, admin: Admin) -> None:
+    """Remove a downloaded model from the ML service (not the one in use)."""
+    if name == system.load_config(db, request.app.state.settings).embedding_model:
+        raise HTTPException(409, "this model is in use")
+    try:
+        _ml(request).delete_model(name)
+    except MLRejected as error:
+        raise HTTPException(404, error.detail) from error
+    except MLUnavailable as error:
+        raise HTTPException(503, "ML service unreachable") from error
+
+
+@router.get("/stats")
+def stats(request: Request, db: DB, admin: Admin) -> Stats:
+    figures = metrics.library_figures(db, request.app.state.settings)
+    return Stats(
+        items=figures.items,
+        files=figures.files,
+        files_pending=figures.files_pending,
+        chunks=figures.chunks,
+        embedded=figures.embedded,
+        embedding_model=figures.embedding_model,
+        jobs=figures.jobs,
+        storage=usage.usage(db, request.app.state.settings),
+    )
+
+
+@router.post("/jobs/{job}", status_code=202)
+def run_job(job: str, request: Request, db: DB, admin: Admin) -> dict[str, Any]:
+    """Start a sweep now: ``embeddings`` (embed what is missing), ``text``
+    (read files still pending), ``text-all`` (read every file again),
+    ``trash`` (delete what is older than the trash days), ``integrity``
+    (check stored contents), ``integrity-deep`` (also their SHA-256) or
+    ``integrity-clean`` (check and remove unused stored contents), ``copy``
+    (update the readable copy) or ``copy-remove`` (delete it)."""
+    jobs = request.app.state.jobs
+    sweeps = {
+        "embeddings": queue_embedding_sweep,
+        "text": queue_text_sweep,
+        "trash": queue_empty_trash,
+        "integrity": queue_integrity_check,
+        "integrity-deep": lambda app: queue_integrity_check(app, deep=True),
+        "integrity-clean": lambda app: queue_integrity_check(app, remove_unused=True),
+        "copy": queue_readable_copy,
+        "copy-remove": lambda app: queue_readable_copy(app, remove=True),
+    }
+    if job == "text-all":
+        count = reread_all(db)
+        queue_text_sweep(request.app.state.jobs)
+        return {"queued": job, "files": count}
+    if job not in sweeps:
+        raise HTTPException(404, f"unknown job {job!r}")
+    sweeps[job](jobs)
+    return {"queued": job}
+
+
+@router.get("/integrity")
+def integrity_report(db: DB, admin: Admin) -> integrity.Report | None:
+    """The last integrity check's report (None before the first)."""
+    return integrity.last_report(db)
+
+
+class Authentication(BaseModel):
+    redirect_uri: str
+    """To register at the identity provider."""
+    secret_set: bool
+    secret_from_environment: bool
+    """Set with ``PAPERUS_OIDC_CLIENT_SECRET`` (then not changeable here)."""
+    linked: int
+    """Accounts that have logged in with the OIDC login."""
+    sessions: int
+    """Devices logged in now."""
+
+
+@router.get("/authentication")
+def authentication(request: Request, db: DB, admin: Admin) -> Authentication:
+    """What the Authentication settings show besides the settings."""
+    settings = request.app.state.settings
+    config = system.load_config(db, settings)
+    now = datetime.now(UTC)
+    return Authentication(
+        redirect_uri=public_base(request, config) + OIDC_CALLBACK,
+        secret_set=system.oidc_secret(db, settings) is not None,
+        secret_from_environment=bool(settings.oidc_client_secret),
+        linked=db.scalar(select(func.count()).where(User.oidc_subject.is_not(None)))
+        or 0,
+        sessions=db.scalar(select(func.count()).where(LoginSession.expires_at > now))
+        or 0,
+    )
+
+
+class Secret(BaseModel):
+    secret: str | None = Field(None, max_length=1000)
+    """None or empty: no secret (a public client)."""
+
+
+@router.put("/authentication/secret", status_code=204)
+def set_oidc_secret(body: Secret, db: DB, admin: Admin) -> None:
+    """The OIDC login's client secret; never sent back."""
+    system.save_oidc_secret(db, body.secret or None)
+    db.commit()
+
+
+class ProviderCheck(BaseModel):
+    issuer: str = Field(max_length=500)
+
+
+class ProviderFound(BaseModel):
+    ok: bool
+    error: str | None = None
+    authorization_endpoint: str | None = None
+
+
+@router.post("/authentication/check")
+def check_provider(
+    body: ProviderCheck, request: Request, admin: Admin
+) -> ProviderFound:
+    """Whether the identity provider answers at this address."""
+    try:
+        provider = oidc.discover(request.app.state.http, body.issuer, fresh=True)
+    except oidc.LoginFailed as error:
+        return ProviderFound(ok=False, error=str(error))
+    return ProviderFound(
+        ok=True, authorization_endpoint=provider.authorization_endpoint
+    )
+
+
+@router.delete("/sessions")
+def end_all_sessions(request: Request, db: DB, admin: Admin) -> dict[str, int]:
+    """Log everyone out on every device, except this one."""
+    ended = auth.end_all_sessions(db, request_token(request) or "")
+    db.commit()
+    return {"ended": ended}
+
+
+class ReadableCopy(BaseModel):
+    path: str
+    """The folder on the server (``PAPERUS_COPY_PATH``)."""
+    free_bytes: int | None
+    report: readable.CopyReport | None
+    """The last update."""
+    variables: dict[str, str]
+    links_possible: bool
+    """Hard links work only with files stored on a disk."""
+
+
+@router.get("/readable-copy")
+def readable_copy(request: Request, db: DB, admin: Admin) -> ReadableCopy:
+    settings = request.app.state.settings
+    root = settings.copy_path.resolve()
+    return ReadableCopy(
+        path=str(root),
+        free_bytes=readable.free_space(root),
+        report=readable.last_report(db),
+        variables=readable.VARIABLES,
+        links_possible=settings.storage_backend == "filesystem",
+    )
+
+
+class TemplateIn(BaseModel):
+    template: str = Field(max_length=300)
+
+
+class TemplatePreview(BaseModel):
+    paths: list[str]
+    """Where some of the library's files would go."""
+    error: str | None = None
+
+
+@router.post("/readable-copy/preview")
+def preview_template(body: TemplateIn, db: DB, admin: Admin) -> TemplatePreview:
+    try:
+        template = readable.check_template(body.template)
+    except readable.TemplateError as error:
+        return TemplatePreview(paths=[], error=str(error))
+    found = readable.wanted(db, template, limit=40)
+    return TemplatePreview(paths=[w.path for w in found[:8]])