flickpick 0.0.2__py3-none-any.whl

flickpick/data.py

@@ -0,0 +1,129 @@
+"""Download (with a cache) and parse the MovieLens datasets the build uses.
+
+Zips are cached under ``<data home>/raw/`` -- ``~/.local/share/flickpick/raw/`` unless
+the ``FLICKPICK_DATA_HOME`` environment variable says otherwise -- through the ``graze``
+URL cache when it is installed, else plain ``urllib``. CSVs are streamed out of the
+zip with ``zipfile``, so the 900 MB ``ratings.csv`` of ml-32m is never held as bytes.
+
+MovieLens licence: research and non-commercial use, and redistributed transformations
+carry the same conditions (https://grouplens.org/datasets/movielens/).
+"""
+
+import os
+import sys
+import urllib.request
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+DATA_HOME_ENVVAR = "FLICKPICK_DATA_HOME"
+DFLT_DATA_HOME = "~/.local/share/flickpick"
+MOVIELENS_BASE_URL = "https://files.grouplens.org/datasets/movielens/"
+MOVIELENS_HOME = "https://grouplens.org/datasets/movielens/"
+MOVIELENS_LICENCE = "research, non-commercial, share-alike on transformations"
+
+SAMPLE_DATASET = "ml-latest-small"
+FULL_DATASET = "ml-32m"
+DATASETS = {
+    name: f"{MOVIELENS_BASE_URL}{name}.zip" for name in (SAMPLE_DATASET, FULL_DATASET)
+}
+
+_DTYPES = {
+    "ratings": {
+        "userId": "int32",
+        "movieId": "int32",
+        "rating": "float32",
+        "timestamp": "int64",
+    },
+    "movies": {"movieId": "int32", "title": "string", "genres": "string"},
+    "links": {"movieId": "int32", "imdbId": "Int64", "tmdbId": "Int64"},
+    "tags": {
+        "userId": "int32",
+        "movieId": "int32",
+        "tag": "string",
+        "timestamp": "int64",
+    },
+}
+
+
+def data_home() -> Path:
+    """The package's data folder (``$FLICKPICK_DATA_HOME`` or ``~/.local/share/flickpick``)."""
+    return Path(os.environ.get(DATA_HOME_ENVVAR, DFLT_DATA_HOME)).expanduser()
+
+
+def raw_dir() -> Path:
+    """Where downloaded source archives are cached."""
+    return data_home() / "raw"
+
+
+def _download(url: str, path: Path) -> Path:
+    try:
+        import graze
+    except ImportError:
+        graze = None
+    print(f"Downloading {url} -> {path}", file=sys.stderr)
+    if graze is not None:
+        return Path(graze.graze(url, cache_key=str(path), return_key=True))
+    tmp = path.with_suffix(path.suffix + ".part")
+    urllib.request.urlretrieve(url, tmp)
+    tmp.replace(path)
+    return path
+
+
+def fetch_dataset(name: str = SAMPLE_DATASET, *, cache_dir=None) -> Path:
+    """Path to the cached zip of MovieLens dataset ``name``, downloading it if needed."""
+    if name not in DATASETS:
+        raise ValueError(f"Unknown dataset {name!r}; choose one of {sorted(DATASETS)}")
+    folder = Path(cache_dir).expanduser() if cache_dir else raw_dir()
+    folder.mkdir(parents=True, exist_ok=True)
+    path = folder / f"{name}.zip"
+    if path.is_file() and zipfile.is_zipfile(path):
+        return path
+    return _download(DATASETS[name], path)
+
+
+@dataclass
+class MovieLens:
+    """The four MovieLens tables used by the build, as DataFrames."""
+
+    name: str
+    ratings: pd.DataFrame
+    movies: pd.DataFrame
+    links: pd.DataFrame
+    tags: pd.DataFrame
+
+
+def _read_member(zf: zipfile.ZipFile, table: str) -> pd.DataFrame:
+    members = [
+        n for n in zf.namelist() if n.endswith(f"/{table}.csv") or n == f"{table}.csv"
+    ]
+    if not members:
+        raise FileNotFoundError(f"{table}.csv not found in {zf.filename}")
+    dtypes = _DTYPES[table]
+    with zf.open(members[0]) as f:
+        return pd.read_csv(f, dtype=dtypes, usecols=list(dtypes))
+
+
+def read_movielens(zip_path, *, name: str | None = None) -> MovieLens:
+    """Parse ratings, movies, links and tags out of a MovieLens zip."""
+    zip_path = Path(zip_path)
+    with zipfile.ZipFile(zip_path) as zf:
+        tables = {t: _read_member(zf, t) for t in _DTYPES}
+    return MovieLens(name=name or zip_path.stem, **tables)
+
+
+def load_movielens(name: str = SAMPLE_DATASET, *, cache_dir=None) -> MovieLens:
+    """Fetch (cached) and parse MovieLens dataset ``name``."""
+    return read_movielens(fetch_dataset(name, cache_dir=cache_dir), name=name)
+
+
+def movielens_source(name: str) -> dict:
+    """The manifest ``sources`` entry for a MovieLens dataset."""
+    return {
+        "name": f"MovieLens {name}",
+        "version": name,
+        "licence": MOVIELENS_LICENCE,
+        "url": MOVIELENS_HOME,
+    }

flickpick/defaults.py

@@ -0,0 +1,67 @@
+"""Named defaults shared by the build pipeline, the scorer and the CLI.
+
+One frozen dataclass, ``DFLT``, is the single source of truth for every number the
+core contract (``docs/core-contract.md``) and the artifact format
+(``docs/artifact-format.md``) fix. Functions take these as keyword-only defaults, so a
+caller can override any of them without touching this module.
+"""
+
+from dataclasses import dataclass
+
+#: The canonical ratings scale is 0-100; these convert from each source's scale.
+STARS_TO_100 = 20  # Letterboxd and MovieLens: 0.5-5 stars
+TEN_TO_100 = 10  # IMDb: 1-10
+SCALE_MIN, SCALE_MAX = 0, 100  # bounds every imported score must lie in
+
+
+@dataclass(frozen=True)
+class Defaults:
+    """Every tunable number in one place (see the docs cited on each field)."""
+
+    # Collaborative filtering (EASE, Steck 2019)
+    #: L2 regularisation; the recsys report's probe found 200, 500 and 1000 tie on
+    #: NDCG@100 for MovieLens 32M (docs/research/recsys-methods-libraries-evaluation.md §3)
+    ease_lambda: float = 500.0
+    #: kept neighbours per item row (98% of dense NDCG@100 in the same probe)
+    topk: int = 100
+    #: number of most-rated items in the CF artifact (they carry ~97% of ratings)
+    n_items: int = 10_000
+    #: a population rating counts as a "like" at or above this, on the 0-100 scale
+    #: (3.5 stars)
+    like_threshold: float = 70.0
+
+    # The user's liked set (core contract: "Defaults that are not seams")
+    #: with fewer ratings than this, the fixed like_threshold replaces the median
+    min_ratings_for_median: int = 5
+
+    # Fusion
+    #: how each component is normalised over the candidate set before the weighted
+    #: sum (core contract: "Defaults that are not seams"); the only value is
+    #: "percentile_rank" (average ranks scaled to [0, 1])
+    normalisation: str = "percentile_rank"
+    w_cf: float = 1.0
+    w_semantic: float = 1.0
+    w_popularity: float = 0.1
+    k: int = 10
+    #: number of liked items named in "because you liked ..."
+    n_because: int = 3
+
+    # Embeddings
+    embedding_model: str = "BAAI/bge-small-en-v1.5"
+    embedding_dim: int = 384
+    query_prefix: str = ""
+    embed_batch_size: int = 64
+    #: cap on the distinct user tags appended to semantic_text (most frequent first)
+    max_tags: int = 50
+
+    # Evaluation
+    folds: int = 5
+    seed: int = 0
+    n_bootstrap: int = 1000
+    ci_level: float = 0.95
+    #: hit_rate is a per-fold quantity; below this many folds its bootstrap interval
+    #: (over folds) is too coarse to report (core contract, Science)
+    min_folds_for_fold_ci: int = 10
+
+
+DFLT = Defaults()

flickpick/ease.py

@@ -0,0 +1,99 @@
+"""EASE (Steck 2019): a closed-form linear item-item model, sparsified to top-k.
+
+Given a binary user x item matrix X:
+
+    G = X^T X + lambda I;   P = G^-1;   B = -P / diag(P) (column-wise);   diag(B) = 0
+
+A user's score vector is then ``x B``. ``B`` is dense, so each source row keeps only its
+``topk`` largest positive weights (the recsys report, section 3, measured 98% of dense
+NDCG@100 at top-100 on MovieLens 32M). Rows of the result are source items, columns
+target items, and each row's entries are sorted by descending weight, which is the
+order ``docs/artifact-format.md`` requires.
+"""
+
+import numpy as np
+from scipy import sparse
+
+from flickpick.artifacts import CF
+from flickpick.defaults import DFLT
+
+#: rows processed at a time when sparsifying a dense B (bounds peak memory)
+SPARSIFY_CHUNK_ROWS = 1024
+
+
+def most_rated_items(X, *, n_items: int = DFLT.n_items) -> np.ndarray:
+    """Column indices of the ``n_items`` columns of ``X`` with most nonzeros.
+
+    Ties are broken by column index, so the result is deterministic.
+    """
+    counts = np.asarray((sparse.csc_matrix(X) != 0).sum(axis=0)).ravel()
+    order = np.lexsort((np.arange(len(counts)), -counts))
+    return order[:n_items]
+
+
+def ease_dense(X, *, lam: float = DFLT.ease_lambda, dtype=np.float64) -> np.ndarray:
+    """The dense EASE weight matrix B for a (sparse or dense) binary matrix X."""
+    if lam <= 0:
+        raise ValueError(f"lam must be positive (it makes G invertible), got {lam}")
+    X = sparse.csr_matrix(X, dtype=dtype)
+    G = (X.T @ X).toarray()
+    G[np.diag_indices_from(G)] += lam
+    del X
+    B = np.linalg.inv(G)  # P; turned into B in place to keep one n x n array alive
+    del G
+    B /= -np.diag(B).copy()
+    B[np.diag_indices_from(B)] = 0.0
+    return B
+
+
+def sparsify_topk(B: np.ndarray, *, topk: int = DFLT.topk) -> CF:
+    """Keep each row's ``topk`` largest positive weights, sorted descending, as CSR."""
+    n_rows, n_cols = B.shape
+    kk = min(topk, n_cols)
+    indptr = np.zeros(n_rows + 1, dtype=np.int64)
+    rows_idx, rows_val = [], []
+    for start in range(0, n_rows, SPARSIFY_CHUNK_ROWS):
+        chunk = B[start : start + SPARSIFY_CHUNK_ROWS]
+        top = (
+            np.argpartition(-chunk, kk - 1, axis=1)[:, :kk]
+            if kk < n_cols
+            else (np.tile(np.arange(n_cols), (len(chunk), 1)))
+        )
+        vals = np.take_along_axis(chunk, top, axis=1)
+        order = np.lexsort((top, -vals), axis=1)
+        top = np.take_along_axis(top, order, axis=1)
+        vals = np.take_along_axis(vals, order, axis=1)
+        for r in range(len(chunk)):
+            keep = vals[r] > 0
+            rows_idx.append(top[r][keep])
+            rows_val.append(vals[r][keep])
+            indptr[start + r + 1] = keep.sum()
+    np.cumsum(indptr, out=indptr)
+    concat = np.concatenate if rows_idx else (lambda _: np.zeros(0))
+    return CF(
+        indptr=indptr.astype(np.int32),
+        indices=concat(rows_idx).astype(np.int32),
+        values=concat(rows_val).astype(np.float32),
+    )
+
+
+def fit_ease(
+    X,
+    *,
+    lam: float = DFLT.ease_lambda,
+    topk: int = DFLT.topk,
+    n_items: int | None = DFLT.n_items,
+) -> tuple[CF, np.ndarray]:
+    """Fit sparse EASE on X restricted to its ``n_items`` most-rated columns (None: all).
+
+    Returns ``(cf, item_cols)``: the CSR matrix over the kept items, and the original
+    column index of each kept item (row ``r`` of ``cf`` is column ``item_cols[r]``).
+    """
+    X = sparse.csr_matrix(X)
+    cols = (
+        np.arange(X.shape[1])
+        if n_items is None
+        else most_rated_items(X, n_items=n_items)
+    )
+    B = ease_dense(X[:, cols], lam=lam)
+    return sparsify_topk(B, topk=topk), cols

flickpick/embedding.py

@@ -0,0 +1,82 @@
+"""Sentence embeddings of catalogue text and of mood queries.
+
+The model is fixed per artifact set (``manifest["embedding"]["model"]``, by default
+``BAAI/bge-small-en-v1.5``, 384-d). Encoding goes through the in-house ``ef`` package's
+sentence-transformers adapter when ``ef`` is importable, else through
+``sentence-transformers`` directly; either way it needs the optional ``[embed]`` extra
+(``pip install 'flickpick[embed]'``). Vectors are always returned L2-normalised float32.
+"""
+
+from collections.abc import Callable, Sequence
+from functools import lru_cache
+
+import numpy as np
+
+from flickpick.defaults import DFLT
+
+INSTALL_HINT = (
+    "Embeddings need sentence-transformers: pip install 'flickpick[embed]' "
+    "(or build with --no-embeddings)."
+)
+
+
+def l2_normalise(x: np.ndarray) -> np.ndarray:
+    """Rows of ``x`` scaled to unit L2 norm (zero rows stay zero), as float32."""
+    x = np.asarray(x, dtype=np.float32)
+    norms = np.linalg.norm(x, axis=-1, keepdims=True)
+    return (x / np.where(norms == 0, 1, norms)).astype(np.float32)
+
+
+@lru_cache(maxsize=4)
+def get_encoder(
+    model: str = DFLT.embedding_model, *, batch_size: int = DFLT.embed_batch_size
+) -> Callable[[Sequence[str]], np.ndarray]:
+    """A batch text encoder ``texts -> array(n, dim)`` for ``model`` (cached)."""
+    try:
+        import ef
+
+        return ef.sentence_transformers_embedder(
+            model, normalize=True, batch_size=batch_size
+        )
+    except ImportError:
+        pass
+    try:
+        from sentence_transformers import SentenceTransformer
+    except ImportError as exc:
+        raise ImportError(INSTALL_HINT) from exc
+    st = SentenceTransformer(model)
+
+    def encode(texts):
+        return st.encode(
+            list(texts),
+            batch_size=batch_size,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+
+    return encode
+
+
+def embed_texts(
+    texts: Sequence[str], *, model: str = DFLT.embedding_model, prefix: str = ""
+) -> np.ndarray:
+    """L2-normalised float32 embeddings of ``prefix + text`` for each text."""
+    encoder = get_encoder(model)
+    return l2_normalise(encoder([prefix + t for t in texts]))
+
+
+def embedding_spec(
+    *,
+    model: str = DFLT.embedding_model,
+    dim: int = DFLT.embedding_dim,
+    query_prefix: str = DFLT.query_prefix,
+) -> dict:
+    """The manifest ``embedding`` entry for vectors of ``semantic_text``."""
+    return {
+        "model": model,
+        "dim": int(dim),
+        "dtype": "float32",
+        "query_prefix": query_prefix,
+        "text_field": "semantic_text",
+    }

flickpick/importers.py

@@ -0,0 +1,188 @@
+"""Parse ratings exports into ``Rating`` lists on the canonical 0-100 scale.
+
+Formats (detected from the header by :func:`detect_format`; header cells are trimmed
+and matched case-insensitively, and the first format whose required columns are all
+present wins, in this order -- the rules of ``docs/core-contract.md``, Importers):
+
+- **imdb** -- IMDb's ratings export (requires ``Const``, ``Your Rating``; 1-10, x10).
+- **flickpick** -- ``movie_id,imdb_id,tmdb_id,rating,average_rating,title`` (requires
+  ``movie_id``, ``imdb_id``, ``rating``) with ``rating`` already 0-100 and ``imdb_id``
+  numeric or ``tt``-prefixed.
+- **letterboxd** -- ``ratings.csv`` or ``diary.csv`` from Letterboxd's data export
+  (requires ``Name``, ``Year``, ``Rating``; ``Date`` and ``Letterboxd URI`` optional;
+  0.5-5 stars, x20). No IMDb id: the rating carries ``"Name (Year)"`` as its title and
+  is matched by title.
+- **movielens** -- a MovieLens ``ratings.csv`` (requires ``movieId``, ``rating``;
+  ``userId`` and ``timestamp`` optional; x20). Items are ``ml:<movieId>``; a file
+  holding several users needs ``user_id=``.
+
+>>> parse_ratings("Const,Your Rating,Title,Year\\ntt0114369,9,Se7en,1995\\n")
+[Rating(item_id='tt0114369', score=90.0, rated_at=None, title='Se7en (1995)')]
+"""
+
+import csv
+import io
+from collections.abc import Callable
+from datetime import datetime, timezone
+
+from flickpick.defaults import SCALE_MAX, SCALE_MIN, STARS_TO_100, TEN_TO_100
+from flickpick.ratings import ML_PREFIX, TMDB_PREFIX, Rating, imdb_id
+
+#: format -> lower-cased columns whose presence identifies it (checked in this order;
+#: the same table as the TypeScript ``SIGNATURES``)
+SIGNATURES = {
+    "imdb": {"const", "your rating"},
+    "flickpick": {"movie_id", "imdb_id", "rating"},
+    "letterboxd": {"name", "year", "rating"},
+    "movielens": {"movieid", "rating"},
+}
+
+
+def _rows(csv_text: str) -> list[dict]:
+    reader = csv.DictReader(io.StringIO(csv_text.lstrip("")))
+    # header cells are matched stripped and lower-cased (as in ``detect_format``), so
+    # "Const, Your Rating" or "RATING" does not silently yield no rows
+    reader.fieldnames = [c.strip().lower() for c in reader.fieldnames or []]
+    return list(reader)
+
+
+def _header(csv_text: str) -> set[str]:
+    first = next(csv.reader(io.StringIO(csv_text.lstrip(""))), [])
+    return {c.strip().lower() for c in first}
+
+
+def detect_format(csv_text: str) -> str:
+    """The name of the export format of ``csv_text`` (a key of ``SIGNATURES``)."""
+    header = _header(csv_text)
+    for name, required in SIGNATURES.items():
+        if required <= header:
+            return name
+    raise ValueError(
+        f"Unrecognised ratings CSV header {sorted(header)}. Expected one of: "
+        + "; ".join(f"{n} ({', '.join(sorted(c))})" for n, c in SIGNATURES.items())
+    )
+
+
+def _blank(value) -> bool:
+    return value is None or not str(value).strip()
+
+
+def _score(value, factor: float) -> float:
+    """``value`` converted to the 0-100 scale; raises when the result falls outside it."""
+    score = float(value) * factor
+    if not SCALE_MIN <= score <= SCALE_MAX:
+        raise ValueError(
+            f"Rating {value!r} (x{factor} = {score}) is outside the "
+            f"{SCALE_MIN}-{SCALE_MAX} scale"
+        )
+    return score
+
+
+def _with_year(title: str, year) -> str:
+    return f"{title} ({year})" if not _blank(year) else title
+
+
+def parse_letterboxd(csv_text: str) -> list[Rating]:
+    """Letterboxd ``ratings.csv`` -> ratings (unrated rows are skipped)."""
+    return [
+        Rating(
+            item_id="",
+            score=_score(r["rating"], STARS_TO_100),
+            rated_at=r.get("date") or None,
+            title=_with_year(r["name"], r["year"]),
+        )
+        for r in _rows(csv_text)
+        if not _blank(r.get("rating"))
+    ]
+
+
+def parse_imdb(csv_text: str) -> list[Rating]:
+    """IMDb ratings export -> ratings."""
+    return [
+        Rating(
+            item_id=imdb_id(r["const"]),
+            score=_score(r["your rating"], TEN_TO_100),
+            rated_at=r.get("date rated") or None,
+            title=_with_year(r.get("title") or "", r.get("year"))
+            if r.get("title")
+            else None,
+        )
+        for r in _rows(csv_text)
+        if not _blank(r.get("your rating"))
+    ]
+
+
+def _iso(timestamp) -> str | None:
+    if _blank(timestamp):
+        return None
+    dt = datetime.fromtimestamp(int(timestamp), tz=timezone.utc)
+    return dt.isoformat(timespec="seconds")
+
+
+def parse_movielens(csv_text: str, *, user_id=None) -> list[Rating]:
+    """MovieLens ``ratings.csv`` -> ratings of one user (``ml:<movieId>`` ids).
+
+    ``userId`` is optional; a file holding several users needs ``user_id``.
+    """
+    rows = _rows(csv_text)
+    users = {r["userid"].strip() for r in rows if not _blank(r.get("userid"))}
+    if user_id is None and len(users) > 1:
+        raise ValueError(
+            f"The file holds {len(users)} users; pass user_id= to pick one."
+        )
+    if user_id is not None:
+        rows = [r for r in rows if (r.get("userid") or "").strip() == str(user_id)]
+    return [
+        Rating(
+            item_id=f"{ML_PREFIX}{int(r['movieid'])}",
+            score=_score(r["rating"], STARS_TO_100),
+            rated_at=_iso(r.get("timestamp")),
+        )
+        for r in rows
+    ]
+
+
+def _flickpick_id(row: dict) -> str:
+    if not _blank(row.get("imdb_id")):
+        return imdb_id(row["imdb_id"])
+    if not _blank(row.get("movie_id")):
+        return f"{ML_PREFIX}{int(row['movie_id'])}"
+    if not _blank(row.get("tmdb_id")):
+        return f"{TMDB_PREFIX}{int(row['tmdb_id'])}"
+    return ""
+
+
+def parse_flickpick(csv_text: str) -> list[Rating]:
+    """The flickpick CSV (the example file's columns) -> ratings (already 0-100)."""
+    return [
+        Rating(
+            item_id=_flickpick_id(r),
+            score=_score(r["rating"], 1),
+            rated_at=r.get("rated_at") or None,
+            title=r.get("title") or None,
+        )
+        for r in _rows(csv_text)
+        if not _blank(r.get("rating"))
+    ]
+
+
+PARSERS: dict[str, Callable[[str], list[Rating]]] = {
+    "letterboxd": parse_letterboxd,
+    "imdb": parse_imdb,
+    "movielens": parse_movielens,
+    "flickpick": parse_flickpick,
+}
+
+
+def parse_ratings(csv_text: str, *, format: str | None = None) -> list[Rating]:
+    """Parse ``csv_text`` with the parser for ``format`` (detected when None)."""
+    fmt = format or detect_format(csv_text)
+    if fmt not in PARSERS:
+        raise ValueError(f"Unknown format {fmt!r}; choose one of {sorted(PARSERS)}")
+    return PARSERS[fmt](csv_text)
+
+
+def read_ratings(path, *, format: str | None = None, encoding="utf-8") -> list[Rating]:
+    """Read and parse a ratings CSV file."""
+    with open(path, encoding=encoding) as f:
+        return parse_ratings(f.read(), format=format)