flickpick 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flickpick/__init__.py +108 -0
- flickpick/__main__.py +15 -0
- flickpick/artifacts.py +264 -0
- flickpick/build.py +180 -0
- flickpick/data/examples/movie_ratings_various.csv +117 -0
- flickpick/data.py +129 -0
- flickpick/defaults.py +67 -0
- flickpick/ease.py +99 -0
- flickpick/embedding.py +82 -0
- flickpick/importers.py +188 -0
- flickpick/ratings.py +166 -0
- flickpick/science.py +384 -0
- flickpick/score.py +367 -0
- flickpick/tools.py +207 -0
- flickpick-0.0.3.dist-info/METADATA +75 -0
- flickpick-0.0.3.dist-info/RECORD +19 -0
- flickpick-0.0.3.dist-info/WHEEL +4 -0
- flickpick-0.0.3.dist-info/entry_points.txt +2 -0
- flickpick-0.0.3.dist-info/licenses/LICENSE +21 -0
flickpick/__init__.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""flickpick: a personal movie recommender (Python half: build pipeline and scorer).
|
|
2
|
+
|
|
3
|
+
Build an artifact set, then recommend from a ratings export:
|
|
4
|
+
|
|
5
|
+
>>> from flickpick import read_artifacts, read_ratings, recommend, Query # doctest: +SKIP
|
|
6
|
+
>>> a = read_artifacts('path/to/artifacts') # doctest: +SKIP
|
|
7
|
+
>>> recommend(a, read_ratings('my_ratings.csv'), Query(mood='slow-burn sci-fi')) # doctest: +SKIP
|
|
8
|
+
|
|
9
|
+
From the shell: ``python -m flickpick build --sample`` then
|
|
10
|
+
``python -m flickpick recommend --ratings my_ratings.csv --mood "..."``.
|
|
11
|
+
The formats and the scoring contract are in ``docs/artifact-format.md`` and
|
|
12
|
+
``docs/core-contract.md``.
|
|
13
|
+
|
|
14
|
+
``import flickpick`` stays light (numpy only): the names from ``build``, ``ease`` and
|
|
15
|
+
``science``, which pull in pandas and scipy, are imported on first access.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from importlib import import_module
|
|
19
|
+
|
|
20
|
+
from flickpick.artifacts import (
|
|
21
|
+
CF,
|
|
22
|
+
ArtifactFormatError,
|
|
23
|
+
Artifacts,
|
|
24
|
+
read_artifacts,
|
|
25
|
+
validate_artifacts,
|
|
26
|
+
write_artifacts,
|
|
27
|
+
)
|
|
28
|
+
from flickpick.defaults import DFLT, Defaults
|
|
29
|
+
from flickpick.importers import (
|
|
30
|
+
detect_format,
|
|
31
|
+
parse_flickpick,
|
|
32
|
+
parse_imdb,
|
|
33
|
+
parse_letterboxd,
|
|
34
|
+
parse_movielens,
|
|
35
|
+
parse_ratings,
|
|
36
|
+
read_ratings,
|
|
37
|
+
)
|
|
38
|
+
from flickpick.ratings import Rating, resolve_ratings
|
|
39
|
+
from flickpick.score import (
|
|
40
|
+
Query,
|
|
41
|
+
Recommendation,
|
|
42
|
+
Weights,
|
|
43
|
+
embed_query,
|
|
44
|
+
format_recommendations,
|
|
45
|
+
recommend,
|
|
46
|
+
score_cf,
|
|
47
|
+
score_semantic,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"CF",
|
|
52
|
+
"ArtifactFormatError",
|
|
53
|
+
"Artifacts",
|
|
54
|
+
"read_artifacts",
|
|
55
|
+
"validate_artifacts",
|
|
56
|
+
"write_artifacts",
|
|
57
|
+
"build_artifacts",
|
|
58
|
+
"DFLT",
|
|
59
|
+
"Defaults",
|
|
60
|
+
"ease_dense",
|
|
61
|
+
"fit_ease",
|
|
62
|
+
"sparsify_topk",
|
|
63
|
+
"detect_format",
|
|
64
|
+
"parse_flickpick",
|
|
65
|
+
"parse_imdb",
|
|
66
|
+
"parse_letterboxd",
|
|
67
|
+
"parse_movielens",
|
|
68
|
+
"parse_ratings",
|
|
69
|
+
"read_ratings",
|
|
70
|
+
"Rating",
|
|
71
|
+
"resolve_ratings",
|
|
72
|
+
"agreement",
|
|
73
|
+
"cross_validated_fit",
|
|
74
|
+
"holdout_evaluate",
|
|
75
|
+
"ranking_metrics",
|
|
76
|
+
"Query",
|
|
77
|
+
"Recommendation",
|
|
78
|
+
"Weights",
|
|
79
|
+
"embed_query",
|
|
80
|
+
"format_recommendations",
|
|
81
|
+
"recommend",
|
|
82
|
+
"score_cf",
|
|
83
|
+
"score_semantic",
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
#: public names whose modules import pandas or scipy: name -> module, loaded on access
|
|
87
|
+
_LAZY = {
|
|
88
|
+
"build_artifacts": "flickpick.build",
|
|
89
|
+
"ease_dense": "flickpick.ease",
|
|
90
|
+
"fit_ease": "flickpick.ease",
|
|
91
|
+
"sparsify_topk": "flickpick.ease",
|
|
92
|
+
"agreement": "flickpick.science",
|
|
93
|
+
"cross_validated_fit": "flickpick.science",
|
|
94
|
+
"holdout_evaluate": "flickpick.science",
|
|
95
|
+
"ranking_metrics": "flickpick.science",
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def __getattr__(name: str):
|
|
100
|
+
if name in _LAZY:
|
|
101
|
+
value = getattr(import_module(_LAZY[name]), name)
|
|
102
|
+
globals()[name] = value # later lookups skip this hook
|
|
103
|
+
return value
|
|
104
|
+
raise AttributeError(f"module 'flickpick' has no attribute {name!r}")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def __dir__() -> list[str]:
|
|
108
|
+
return sorted({*globals(), *_LAZY})
|
flickpick/__main__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# PYTHON_ARGCOMPLETE_OK
|
|
2
|
+
"""CLI entry point: ``python -m flickpick {build,recommend,evaluate} --help``."""
|
|
3
|
+
|
|
4
|
+
import cw
|
|
5
|
+
|
|
6
|
+
from flickpick.tools import CLI_CONFIG, CLI_FUNCS, cli_egress
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main():
|
|
10
|
+
"""Dispatch the tool functions to the command line."""
|
|
11
|
+
raise SystemExit(cw.dispatch(CLI_FUNCS, config=CLI_CONFIG, egress=cli_egress))
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
main()
|
flickpick/artifacts.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Write and read an artifact set as specified in ``docs/artifact-format.md``.
|
|
2
|
+
|
|
3
|
+
An artifact set is one directory: ``manifest.json``, ``catalog.json``, the sparse EASE
|
|
4
|
+
matrix as three CSR arrays (``cf_indptr.i32``, ``cf_indices.i32``, ``cf_values.f32``)
|
|
5
|
+
and, optionally, ``embeddings.f32``. Binaries are little-endian, row-major, headerless;
|
|
6
|
+
their shapes are recorded in ``manifest["files"]`` and checked on read.
|
|
7
|
+
|
|
8
|
+
>>> import numpy as np, tempfile
|
|
9
|
+
>>> rows = [{**dict.fromkeys(CATALOG_FIELDS), 'idx': i, 'imdb_id': f'tt000000{i}',
|
|
10
|
+
... 'title': f'Film {i}', 'genres': [], 'n_ratings': 1} for i in range(2)]
|
|
11
|
+
>>> a = Artifacts(
|
|
12
|
+
... manifest=mk_manifest(name='toy', n_items=2, sources=[], cf={'method': 'ease'}),
|
|
13
|
+
... catalog=rows,
|
|
14
|
+
... cf=CF(np.array([0, 1, 1]), np.array([1]), np.array([0.5])),
|
|
15
|
+
... )
|
|
16
|
+
>>> d = tempfile.mkdtemp()
|
|
17
|
+
>>> _ = write_artifacts(a, d)
|
|
18
|
+
>>> read_artifacts(d).cf.values.tolist()
|
|
19
|
+
[0.5]
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
from collections.abc import Mapping
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from functools import cached_property
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
import numpy as np
|
|
30
|
+
|
|
31
|
+
FORMAT_VERSION = 1
|
|
32
|
+
MANIFEST_FILE = "manifest.json"
|
|
33
|
+
CATALOG_FILE = "catalog.json"
|
|
34
|
+
|
|
35
|
+
#: name in ``manifest["files"]`` -> (file name, numpy little-endian dtype, dtype label)
|
|
36
|
+
BINARY_FILES = {
|
|
37
|
+
"cf_indptr": ("cf_indptr.i32", "<i4", "int32"),
|
|
38
|
+
"cf_indices": ("cf_indices.i32", "<i4", "int32"),
|
|
39
|
+
"cf_values": ("cf_values.f32", "<f4", "float32"),
|
|
40
|
+
"embeddings": ("embeddings.f32", "<f4", "float32"),
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
CATALOG_FIELDS = (
|
|
44
|
+
"idx",
|
|
45
|
+
"imdb_id",
|
|
46
|
+
"tmdb_id",
|
|
47
|
+
"ml_id",
|
|
48
|
+
"qid",
|
|
49
|
+
"title",
|
|
50
|
+
"year",
|
|
51
|
+
"genres",
|
|
52
|
+
"n_ratings",
|
|
53
|
+
"mean_rating",
|
|
54
|
+
"semantic_text",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ArtifactFormatError(ValueError):
|
|
59
|
+
"""An artifact set does not match ``docs/artifact-format.md``."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class CF:
|
|
64
|
+
"""The sparsified EASE item-item matrix B in CSR form (rows = source items)."""
|
|
65
|
+
|
|
66
|
+
indptr: np.ndarray
|
|
67
|
+
indices: np.ndarray
|
|
68
|
+
values: np.ndarray
|
|
69
|
+
|
|
70
|
+
def to_scipy(self, n_items: int):
|
|
71
|
+
"""The same matrix as a ``scipy.sparse.csr_matrix`` of shape (n, n)."""
|
|
72
|
+
from scipy.sparse import csr_matrix
|
|
73
|
+
|
|
74
|
+
return csr_matrix(
|
|
75
|
+
(self.values, self.indices, self.indptr), shape=(n_items, n_items)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class Artifacts:
|
|
81
|
+
"""One loaded artifact set: manifest, catalogue, CF matrix, optional embeddings."""
|
|
82
|
+
|
|
83
|
+
manifest: dict
|
|
84
|
+
catalog: list[dict]
|
|
85
|
+
cf: CF
|
|
86
|
+
embeddings: np.ndarray | None = field(default=None)
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def n_items(self) -> int:
|
|
90
|
+
"""Number of items (rows of every file)."""
|
|
91
|
+
return len(self.catalog)
|
|
92
|
+
|
|
93
|
+
@cached_property
|
|
94
|
+
def n_ratings(self) -> np.ndarray:
|
|
95
|
+
"""Population rating counts, as an array in ``idx`` order."""
|
|
96
|
+
return np.array([row["n_ratings"] for row in self.catalog], dtype=np.int64)
|
|
97
|
+
|
|
98
|
+
@cached_property
|
|
99
|
+
def imdb_to_idx(self) -> dict[str, int]:
|
|
100
|
+
"""IMDb id -> row index."""
|
|
101
|
+
return {row["imdb_id"]: row["idx"] for row in self.catalog}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def mk_manifest(
|
|
105
|
+
*,
|
|
106
|
+
name: str,
|
|
107
|
+
n_items: int,
|
|
108
|
+
sources: list[dict],
|
|
109
|
+
cf: dict,
|
|
110
|
+
embedding: dict | None = None,
|
|
111
|
+
built_at: str | None = None,
|
|
112
|
+
) -> dict:
|
|
113
|
+
"""A manifest without ``files`` (``write_artifacts`` fills that in)."""
|
|
114
|
+
return {
|
|
115
|
+
"format_version": FORMAT_VERSION,
|
|
116
|
+
"built_at": built_at
|
|
117
|
+
or datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
118
|
+
"name": name,
|
|
119
|
+
"n_items": int(n_items),
|
|
120
|
+
"sources": list(sources),
|
|
121
|
+
"cf": dict(cf),
|
|
122
|
+
"embedding": None if embedding is None else dict(embedding),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _binary_arrays(a: Artifacts) -> dict[str, np.ndarray]:
|
|
127
|
+
arrays = {
|
|
128
|
+
"cf_indptr": a.cf.indptr,
|
|
129
|
+
"cf_indices": a.cf.indices,
|
|
130
|
+
"cf_values": a.cf.values,
|
|
131
|
+
}
|
|
132
|
+
if a.embeddings is not None:
|
|
133
|
+
arrays["embeddings"] = a.embeddings
|
|
134
|
+
return arrays
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def validate_artifacts(a: Artifacts) -> Artifacts:
|
|
138
|
+
"""Raise ``ArtifactFormatError`` unless every shape and the catalogue agree."""
|
|
139
|
+
m, n = a.manifest, a.n_items
|
|
140
|
+
|
|
141
|
+
def check(ok, msg):
|
|
142
|
+
if not ok:
|
|
143
|
+
raise ArtifactFormatError(msg)
|
|
144
|
+
|
|
145
|
+
check(
|
|
146
|
+
m.get("format_version") == FORMAT_VERSION,
|
|
147
|
+
f"format_version is {m.get('format_version')!r}; this reader supports "
|
|
148
|
+
f"{FORMAT_VERSION}",
|
|
149
|
+
)
|
|
150
|
+
check(
|
|
151
|
+
m.get("n_items") == n,
|
|
152
|
+
f"manifest n_items={m.get('n_items')} but the catalogue has {n} rows",
|
|
153
|
+
)
|
|
154
|
+
check(
|
|
155
|
+
[row.get("idx") for row in a.catalog] == list(range(n)),
|
|
156
|
+
"catalogue rows must be in idx order with idx = 0..n_items-1",
|
|
157
|
+
)
|
|
158
|
+
for i, row in enumerate(a.catalog):
|
|
159
|
+
missing = set(CATALOG_FIELDS) - set(row)
|
|
160
|
+
check(not missing, f"catalogue row {i} lacks {sorted(missing)}")
|
|
161
|
+
indptr, indices, values = a.cf.indptr, a.cf.indices, a.cf.values
|
|
162
|
+
check(
|
|
163
|
+
indptr.shape == (n + 1,),
|
|
164
|
+
f"cf_indptr has shape {indptr.shape}, expected ({n + 1},)",
|
|
165
|
+
)
|
|
166
|
+
nnz = int(indptr[-1]) if len(indptr) else 0
|
|
167
|
+
check(
|
|
168
|
+
indices.shape == (nnz,) and values.shape == (nnz,),
|
|
169
|
+
f"cf_indices {indices.shape} and cf_values {values.shape} must both have "
|
|
170
|
+
f"length indptr[-1] = {nnz}",
|
|
171
|
+
)
|
|
172
|
+
check(len(indptr) == 0 or int(indptr[0]) == 0, "cf_indptr[0] must be 0")
|
|
173
|
+
check(bool(np.all(np.diff(indptr) >= 0)), "cf_indptr must be non-decreasing")
|
|
174
|
+
check(
|
|
175
|
+
nnz == 0 or (indices.min() >= 0 and indices.max() < n),
|
|
176
|
+
f"cf_indices must lie in [0, {n})",
|
|
177
|
+
)
|
|
178
|
+
spec = m.get("embedding")
|
|
179
|
+
if spec is None:
|
|
180
|
+
check(a.embeddings is None, "embeddings present but manifest.embedding is null")
|
|
181
|
+
else:
|
|
182
|
+
check(
|
|
183
|
+
a.embeddings is not None,
|
|
184
|
+
"manifest.embedding is set but embeddings are missing",
|
|
185
|
+
)
|
|
186
|
+
expected = (n, int(spec["dim"]))
|
|
187
|
+
check(
|
|
188
|
+
a.embeddings.shape == expected,
|
|
189
|
+
f"embeddings have shape {a.embeddings.shape}, expected {expected}",
|
|
190
|
+
)
|
|
191
|
+
return a
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _write_json(path: Path, obj) -> None:
|
|
195
|
+
path.write_text(
|
|
196
|
+
json.dumps(obj, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def write_artifacts(a: Artifacts, out_dir) -> Path:
|
|
201
|
+
"""Write ``a`` to ``out_dir`` (created if needed) and return the directory path.
|
|
202
|
+
|
|
203
|
+
The manifest's ``files`` entry is (re)computed from the arrays.
|
|
204
|
+
"""
|
|
205
|
+
out = Path(out_dir).expanduser()
|
|
206
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
207
|
+
files = {}
|
|
208
|
+
for name, arr in _binary_arrays(a).items():
|
|
209
|
+
filename, dtype, label = BINARY_FILES[name]
|
|
210
|
+
np.ascontiguousarray(arr, dtype=dtype).tofile(out / filename)
|
|
211
|
+
files[name] = {"path": filename, "dtype": label, "shape": list(arr.shape)}
|
|
212
|
+
manifest = {**a.manifest, "files": files}
|
|
213
|
+
validate_artifacts(Artifacts(manifest, a.catalog, a.cf, a.embeddings))
|
|
214
|
+
_write_json(out / CATALOG_FILE, a.catalog)
|
|
215
|
+
(out / MANIFEST_FILE).write_text(
|
|
216
|
+
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
217
|
+
)
|
|
218
|
+
return out
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _read_binary(root: Path, name: str, files: Mapping) -> np.ndarray:
|
|
222
|
+
if name not in files:
|
|
223
|
+
raise ArtifactFormatError(f"manifest.files has no entry {name!r}")
|
|
224
|
+
entry = files[name]
|
|
225
|
+
_, dtype, label = BINARY_FILES[name]
|
|
226
|
+
if entry.get("dtype") != label:
|
|
227
|
+
raise ArtifactFormatError(
|
|
228
|
+
f"{name}: dtype {entry.get('dtype')!r}, expected {label!r}"
|
|
229
|
+
)
|
|
230
|
+
path = root / entry["path"]
|
|
231
|
+
if not path.is_file():
|
|
232
|
+
raise ArtifactFormatError(f"{name}: file not found: {path}")
|
|
233
|
+
arr = np.fromfile(path, dtype=dtype)
|
|
234
|
+
shape = tuple(entry["shape"])
|
|
235
|
+
if arr.size != int(np.prod(shape)):
|
|
236
|
+
raise ArtifactFormatError(
|
|
237
|
+
f"{name}: {arr.size} values on disk but the manifest "
|
|
238
|
+
f"says shape {list(shape)}"
|
|
239
|
+
)
|
|
240
|
+
return arr.reshape(shape).astype(dtype[1:], copy=False)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def read_artifacts(source) -> Artifacts:
|
|
244
|
+
"""Load and validate the artifact set in directory ``source``."""
|
|
245
|
+
root = Path(source).expanduser()
|
|
246
|
+
manifest_path = root / MANIFEST_FILE
|
|
247
|
+
if not manifest_path.is_file():
|
|
248
|
+
raise FileNotFoundError(
|
|
249
|
+
f"No {MANIFEST_FILE} in {root}. Build an artifact set first: "
|
|
250
|
+
"python -m flickpick build --sample"
|
|
251
|
+
)
|
|
252
|
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
253
|
+
catalog = json.loads((root / CATALOG_FILE).read_text(encoding="utf-8"))
|
|
254
|
+
files = manifest.get("files", {})
|
|
255
|
+
cf = CF(
|
|
256
|
+
*(
|
|
257
|
+
_read_binary(root, n, files)
|
|
258
|
+
for n in ("cf_indptr", "cf_indices", "cf_values")
|
|
259
|
+
)
|
|
260
|
+
)
|
|
261
|
+
embeddings = None
|
|
262
|
+
if manifest.get("embedding") is not None:
|
|
263
|
+
embeddings = _read_binary(root, "embeddings", files)
|
|
264
|
+
return validate_artifacts(Artifacts(manifest, catalog, cf, embeddings))
|
flickpick/build.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""The build pipeline: MovieLens tables -> catalogue, sparse EASE matrix, embeddings.
|
|
2
|
+
|
|
3
|
+
``build_artifacts`` is pure (tables in, ``Artifacts`` out; the only side effect is
|
|
4
|
+
running the embedding model). ``tools.build`` adds the download and the write.
|
|
5
|
+
|
|
6
|
+
The CF set is the ``n_items`` most-rated films that have an IMDb id; ``idx`` orders them
|
|
7
|
+
by rating count (descending, ties by MovieLens id). EASE is trained on ratings at or
|
|
8
|
+
above ``like_threshold`` (on the 0-100 scale; 70 = 3.5 stars), binarised.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from scipy import sparse
|
|
17
|
+
|
|
18
|
+
from flickpick.artifacts import Artifacts, mk_manifest
|
|
19
|
+
from flickpick.data import MovieLens, movielens_source
|
|
20
|
+
from flickpick.defaults import DFLT, STARS_TO_100
|
|
21
|
+
from flickpick.ease import ease_dense, sparsify_topk
|
|
22
|
+
from flickpick.embedding import embed_texts, embedding_spec
|
|
23
|
+
from flickpick.ratings import imdb_id, split_title_year
|
|
24
|
+
|
|
25
|
+
NO_GENRES = "(no genres listed)"
|
|
26
|
+
GENRE_SEP = "|"
|
|
27
|
+
TEXT_SEP = ", "
|
|
28
|
+
MEAN_RATING_DECIMALS = 2
|
|
29
|
+
_TRAILING_ARTICLE = re.compile(r"^(.*), (The|A|An)$")
|
|
30
|
+
_PARENTHETICAL_SPLIT = re.compile(r"(\s*\([^()]*\))")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _front_article(text: str) -> str:
|
|
34
|
+
m = _TRAILING_ARTICLE.match(text.strip())
|
|
35
|
+
return f"{m.group(2)} {m.group(1)}" if m else text.strip()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def clean_title(ml_title: str) -> tuple[str, int | None]:
|
|
39
|
+
"""MovieLens ``"Matrix, The (1999)"`` -> ``("The Matrix", 1999)``.
|
|
40
|
+
|
|
41
|
+
Parenthesised alternate titles are kept (they help match other services' titles),
|
|
42
|
+
with their articles moved to the front too.
|
|
43
|
+
|
|
44
|
+
>>> clean_title('Seven (a.k.a. Se7en) (1995)')
|
|
45
|
+
('Seven (a.k.a. Se7en)', 1995)
|
|
46
|
+
>>> clean_title('Shawshank Redemption, The (1994)')
|
|
47
|
+
('The Shawshank Redemption', 1994)
|
|
48
|
+
"""
|
|
49
|
+
title, year = split_title_year(ml_title)
|
|
50
|
+
parts = _PARENTHETICAL_SPLIT.split(title)
|
|
51
|
+
head, rest = parts[0], parts[1:]
|
|
52
|
+
rest = [
|
|
53
|
+
re.sub(r"\(([^()]*)\)", lambda m: f"({_front_article(m.group(1))})", p)
|
|
54
|
+
for p in rest
|
|
55
|
+
]
|
|
56
|
+
return _front_article(head) + "".join(rest), year
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parse_genres(text) -> list[str]:
|
|
60
|
+
"""``"Action|Sci-Fi"`` -> ``["Action", "Sci-Fi"]``; ``"(no genres listed)"`` -> []."""
|
|
61
|
+
if text is None or pd.isna(text) or text == NO_GENRES:
|
|
62
|
+
return []
|
|
63
|
+
return [g for g in str(text).split(GENRE_SEP) if g]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _optional_int(x):
|
|
67
|
+
return None if x is None or pd.isna(x) else int(x)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def select_items(ml: MovieLens, *, n_items: int) -> pd.DataFrame:
|
|
71
|
+
"""One row per CF item (movieId, imdbId, tmdbId, title, genres, n_ratings, mean)."""
|
|
72
|
+
stats = ml.ratings.groupby("movieId")["rating"].agg(n_ratings="size", mean="mean")
|
|
73
|
+
items = (
|
|
74
|
+
stats.join(ml.links.set_index("movieId"), how="inner")
|
|
75
|
+
.join(ml.movies.set_index("movieId"), how="inner")
|
|
76
|
+
.dropna(subset=["imdbId"])
|
|
77
|
+
.reset_index()
|
|
78
|
+
.sort_values(["n_ratings", "movieId"], ascending=[False, True])
|
|
79
|
+
)
|
|
80
|
+
items = items.drop_duplicates(subset="imdbId", keep="first")
|
|
81
|
+
return items.head(n_items).reset_index(drop=True)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def movie_tags(tags: pd.DataFrame, *, max_tags: int = DFLT.max_tags) -> dict[int, list]:
|
|
85
|
+
"""movieId -> its distinct user tags (lower-cased), most applied first."""
|
|
86
|
+
t = tags.assign(tag=tags["tag"].astype(str).str.strip().str.lower())
|
|
87
|
+
t = t[t["tag"] != ""]
|
|
88
|
+
counts = t.groupby(["movieId", "tag"]).size().reset_index(name="n")
|
|
89
|
+
counts = counts.sort_values(["movieId", "n", "tag"], ascending=[True, False, True])
|
|
90
|
+
return {
|
|
91
|
+
int(mid): list(g["tag"].head(max_tags))
|
|
92
|
+
for mid, g in counts.groupby("movieId", sort=False)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def catalog_rows(items: pd.DataFrame, tags_by_movie: dict) -> list[dict]:
|
|
97
|
+
"""Catalogue rows (``docs/artifact-format.md``) in ``idx`` order."""
|
|
98
|
+
rows = []
|
|
99
|
+
for idx, it in enumerate(items.itertuples(index=False)):
|
|
100
|
+
title, year = clean_title(str(it.title))
|
|
101
|
+
genres = parse_genres(it.genres)
|
|
102
|
+
tags = tags_by_movie.get(int(it.movieId), [])
|
|
103
|
+
rows.append(
|
|
104
|
+
{
|
|
105
|
+
"idx": idx,
|
|
106
|
+
"imdb_id": imdb_id(int(it.imdbId)),
|
|
107
|
+
"tmdb_id": _optional_int(it.tmdbId),
|
|
108
|
+
"ml_id": int(it.movieId),
|
|
109
|
+
"qid": None,
|
|
110
|
+
"title": title,
|
|
111
|
+
"year": year,
|
|
112
|
+
"genres": genres,
|
|
113
|
+
"n_ratings": int(it.n_ratings),
|
|
114
|
+
"mean_rating": round(
|
|
115
|
+
float(it.mean) * STARS_TO_100, MEAN_RATING_DECIMALS
|
|
116
|
+
),
|
|
117
|
+
"semantic_text": TEXT_SEP.join([title, *genres, *tags]),
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
return rows
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def like_matrix(
|
|
124
|
+
ratings: pd.DataFrame, movie_ids, *, like_threshold: float = DFLT.like_threshold
|
|
125
|
+
) -> sparse.csr_matrix:
|
|
126
|
+
"""Binary users x items matrix of likes, columns in the order of ``movie_ids``."""
|
|
127
|
+
col = pd.Series(np.arange(len(movie_ids)), index=np.asarray(movie_ids))
|
|
128
|
+
liked = ratings[ratings["rating"] * STARS_TO_100 >= like_threshold]
|
|
129
|
+
liked = liked[liked["movieId"].isin(col.index)]
|
|
130
|
+
users, user_codes = np.unique(liked["userId"].to_numpy(), return_inverse=True)
|
|
131
|
+
cols = col.loc[liked["movieId"].to_numpy()].to_numpy()
|
|
132
|
+
data = np.ones(len(cols), dtype=np.float64)
|
|
133
|
+
return sparse.csr_matrix(
|
|
134
|
+
(data, (user_codes, cols)), shape=(len(users), len(movie_ids))
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def build_artifacts(
|
|
139
|
+
ml: MovieLens,
|
|
140
|
+
*,
|
|
141
|
+
n_items: int = DFLT.n_items,
|
|
142
|
+
topk: int = DFLT.topk,
|
|
143
|
+
lam: float = DFLT.ease_lambda,
|
|
144
|
+
like_threshold: float = DFLT.like_threshold,
|
|
145
|
+
embeddings: bool = True,
|
|
146
|
+
embedding_model: str = DFLT.embedding_model,
|
|
147
|
+
embed: Callable | None = None,
|
|
148
|
+
max_tags: int = DFLT.max_tags,
|
|
149
|
+
) -> Artifacts:
|
|
150
|
+
"""Build an in-memory artifact set from MovieLens tables.
|
|
151
|
+
|
|
152
|
+
``embed`` (texts -> L2-normalised float32 array) replaces the default encoder,
|
|
153
|
+
which is ``embed_texts`` with ``embedding_model``.
|
|
154
|
+
"""
|
|
155
|
+
items = select_items(ml, n_items=n_items)
|
|
156
|
+
catalog = catalog_rows(items, movie_tags(ml.tags, max_tags=max_tags))
|
|
157
|
+
X = like_matrix(ml.ratings, items["movieId"], like_threshold=like_threshold)
|
|
158
|
+
cf = sparsify_topk(ease_dense(X, lam=lam), topk=topk)
|
|
159
|
+
spec, vectors = None, None
|
|
160
|
+
if embeddings:
|
|
161
|
+
embed = embed or (lambda texts: embed_texts(texts, model=embedding_model))
|
|
162
|
+
vectors = np.asarray(
|
|
163
|
+
embed([r["semantic_text"] for r in catalog]), dtype=np.float32
|
|
164
|
+
)
|
|
165
|
+
spec = embedding_spec(model=embedding_model, dim=vectors.shape[1])
|
|
166
|
+
manifest = mk_manifest(
|
|
167
|
+
name=f"flickpick-{ml.name}-{len(catalog)}",
|
|
168
|
+
n_items=len(catalog),
|
|
169
|
+
sources=[movielens_source(ml.name)],
|
|
170
|
+
cf={
|
|
171
|
+
"method": "ease",
|
|
172
|
+
"topk": int(topk),
|
|
173
|
+
"lambda": float(lam),
|
|
174
|
+
"n_train_users": int(X.shape[0]),
|
|
175
|
+
"n_train_ratings": int(X.nnz),
|
|
176
|
+
"like_threshold": float(like_threshold),
|
|
177
|
+
},
|
|
178
|
+
embedding=spec,
|
|
179
|
+
)
|
|
180
|
+
return Artifacts(manifest=manifest, catalog=catalog, cf=cf, embeddings=vectors)
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
movie_id,imdb_id,tmdb_id,rating,average_rating,title
|
|
2
|
+
47,114369,807,95,81.88,Se7en (1995)
|
|
3
|
+
50,114814,629,98,85.1,The Usual Suspects (1995)
|
|
4
|
+
293,110413,101,99,81.83,Léon: The Professional (1994)
|
|
5
|
+
296,110912,680,95,83.9,Pulp Fiction (1994)
|
|
6
|
+
318,111161,278,99,88.29,The Shawshank Redemption (1994)
|
|
7
|
+
527,108052,424,94,84.7,Schindler's List (1993)
|
|
8
|
+
593,102926,274,95,83,The Silence of the Lambs (1991)
|
|
9
|
+
858,68646,238,99,86.43,The Godfather (1972)
|
|
10
|
+
1193,73486,510,99,84.14,One Flew Over the Cuckoo's Nest (1975)
|
|
11
|
+
1196,80684,1891,94,82.29,The Empire Strikes Back (1980)
|
|
12
|
+
1198,82971,85,97,81.91,Raiders of the Lost Ark (1981)
|
|
13
|
+
1221,71562,240,99,85.38,The Godfather Part II (1974)
|
|
14
|
+
1246,97165,207,94,78.97,Dead Poets Society (1989)
|
|
15
|
+
1258,81505,694,99,80.76,The Shining (1980)
|
|
16
|
+
2324,118799,637,96,83.14,Life Is Beautiful (1997)
|
|
17
|
+
2571,133093,603,95,83.31,The Matrix (1999)
|
|
18
|
+
2762,167404,745,96,80.09,The Sixth Sense (1999)
|
|
19
|
+
2858,169547,14,95,81.85,American Beauty (1999)
|
|
20
|
+
2959,137523,550,96,84.75,Fight Club (1999)
|
|
21
|
+
2997,120601,492,97,78.66,Being John Malkovich (1999)
|
|
22
|
+
4973,211915,194,99,81.58,Amélie (2001)
|
|
23
|
+
5418,258463,2501,98,78.15,The Bourne Identity (2002)
|
|
24
|
+
7361,338013,38,98,81.33,Eternal Sunshine of the Spotless Mind (2004)
|
|
25
|
+
48516,407887,1422,99,82.68,The Departed (2006)
|
|
26
|
+
58559,468569,155,98,83.77,The Dark Knight (2008)
|
|
27
|
+
99114,1853728,68718,97,80.8,Django Unchained (2012)
|
|
28
|
+
260,76759,11,90,81.72,Star Wars (1977)
|
|
29
|
+
356,109830,13,92,81.44,Forrest Gump (1994)
|
|
30
|
+
608,116282,275,91,82.34,Fargo (1996)
|
|
31
|
+
1704,119217,489,89,82.33,Good Will Hunting (1997)
|
|
32
|
+
2329,120586,73,94,82.58,American History X (1998)
|
|
33
|
+
2692,130827,104,95,79.07,Run Lola Run (1998)
|
|
34
|
+
4226,209144,77,92,82.71,Memento (2000)
|
|
35
|
+
4963,240772,161,95,76.4,Ocean's Eleven (2001)
|
|
36
|
+
34162,396269,9522,94,68.58,Wedding Crashers (2005)
|
|
37
|
+
50872,382932,2062,93,76.85,Ratatouille (2007)
|
|
38
|
+
111759,1631867,137113,94,79.39,Edge of Tomorrow (2014)
|
|
39
|
+
134853,2096673,150540,93,78.16,Inside Out (2015)
|
|
40
|
+
202439,6751668,496243,93,86.04,Parasite (2019)
|
|
41
|
+
32,114746,63,84,77.93,Twelve Monkeys (1995)
|
|
42
|
+
480,107290,329,80,74.06,Jurassic Park (1993)
|
|
43
|
+
589,103064,280,83,79.44,Terminator 2: Judgment Day (1991)
|
|
44
|
+
778,117951,627,84,80.27,Trainspotting (1996)
|
|
45
|
+
1036,95016,562,82,78.96,Die Hard (1988)
|
|
46
|
+
1089,105236,500,89,81.84,Reservoir Dogs (1992)
|
|
47
|
+
1136,71853,762,84,82.59,Monty Python and the Holy Grail (1975)
|
|
48
|
+
1213,99685,769,81,83.9,GoodFellas (1990)
|
|
49
|
+
1214,78748,348,82,81.5,Alien (1979)
|
|
50
|
+
1247,61722,37247,80,80.26,The Graduate (1967)
|
|
51
|
+
1265,107048,137,82,78.16,Groundhog Day (1993)
|
|
52
|
+
1625,119174,2649,83,77.42,The Game (1997)
|
|
53
|
+
1732,118715,115,86,79.9,The Big Lebowski (1998)
|
|
54
|
+
1961,95953,380,83,78.21,Rain Man (1988)
|
|
55
|
+
2502,151804,1542,84,79.79,Office Space (1999)
|
|
56
|
+
3578,172495,98,82,79.58,Gladiator (2000)
|
|
57
|
+
4022,162222,8358,80,73.99,Cast Away (2000)
|
|
58
|
+
4262,86250,111,81,78.83,Scarface (1983)
|
|
59
|
+
5218,268380,425,81,70.62,Ice Age (2002)
|
|
60
|
+
5989,264464,640,85,79.44,Catch Me If You Can (2002)
|
|
61
|
+
6874,266697,24,84,77.21,Kill Bill: Vol. 1 (2003)
|
|
62
|
+
7438,378194,393,84,76.55,Kill Bill: Vol. 2 (2004)
|
|
63
|
+
30749,395169,205,83,79.17,Hotel Rwanda (2004)
|
|
64
|
+
44555,405094,582,84,83.73,The Lives of Others (2006)
|
|
65
|
+
63082,1010048,12405,83,76.45,Slumdog Millionaire (2008)
|
|
66
|
+
68954,1049413,14160,82,79.14,Up (2009)
|
|
67
|
+
69122,1119646,18785,83,72.18,The Hangover (2009)
|
|
68
|
+
79132,1375666,27205,82,83.46,Inception (2010)
|
|
69
|
+
106920,1798709,152601,85,77.93,Her (2013)
|
|
70
|
+
152081,2948356,269149,83,76.92,Zootopia (2016)
|
|
71
|
+
168250,5052448,419430,82,79.57,Get Out (2017)
|
|
72
|
+
207313,8946378,546554,87,80.66,Knives Out (2019)
|
|
73
|
+
225173,2948372,508442,83,77.88,Soul (2020)
|
|
74
|
+
322321,12042730,687163,86,82.26,Project Hail Mary (2026)
|
|
75
|
+
327081,33764258,1368337,82,80.04,The Odyssey (2026)
|
|
76
|
+
508,107818,9800,72,75.96,Philadelphia (1993)
|
|
77
|
+
541,83658,78,73,82.21,Blade Runner (1982)
|
|
78
|
+
750,57012,935,70,83.85,Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964)
|
|
79
|
+
912,34583,289,70,83.75,Casablanca (1943)
|
|
80
|
+
1097,83866,601,73,74.74,E.T. the Extra-Terrestrial (1982)
|
|
81
|
+
1203,50083,389,71,85.55,12 Angry Men (1957)
|
|
82
|
+
1263,77416,11778,73,79.23,The Deer Hunter (1978)
|
|
83
|
+
1270,88763,105,72,79.46,Back to the Future (1985)
|
|
84
|
+
1527,119116,18,74,75.87,The Fifth Element (1997)
|
|
85
|
+
3147,120689,497,70,80.94,The Green Mile (1999)
|
|
86
|
+
3948,212338,1597,68,67.83,Meet the Parents (2000)
|
|
87
|
+
3994,217869,9741,77,70.26,Unbreakable (2000)
|
|
88
|
+
4027,190590,134,72,77.48,"O Brother, Where Art Thou? (2000)"
|
|
89
|
+
4995,268978,453,73,79.27,A Beautiful Mind (2001)
|
|
90
|
+
5618,245429,129,74,84.44,Spirited Away (2001)
|
|
91
|
+
5669,310793,1430,72,73.38,Bowling for Columbine (2002)
|
|
92
|
+
6502,289043,170,76,75.56,28 Days Later (2002)
|
|
93
|
+
8874,365748,747,69,77.53,Shaun of the Dead (2004)
|
|
94
|
+
8961,317705,9806,70,76.92,The Incredibles (2004)
|
|
95
|
+
44191,434409,752,71,78.08,V for Vendetta (2006)
|
|
96
|
+
45720,458352,350,74,70.85,The Devil Wears Prada (2006)
|
|
97
|
+
48385,443453,496,73,68.62,Borat: Cultural Learnings of America for Make Benefit Glorious Nation of Kazakhstan (2006)
|
|
98
|
+
49272,381061,36557,71,76.83,Casino Royale (2006)
|
|
99
|
+
55247,758758,5915,69,76.84,Into the Wild (2007)
|
|
100
|
+
59784,441773,9502,71,72.91,Kung Fu Panda (2008)
|
|
101
|
+
68157,361748,16869,70,80.97,Inglourious Basterds (2009)
|
|
102
|
+
72998,499549,19995,72,71.86,Avatar (2009)
|
|
103
|
+
84152,1219289,51876,71,76.83,Limitless (2011)
|
|
104
|
+
106782,993846,106646,74,79.06,The Wolf of Wall Street (2013)
|
|
105
|
+
112552,2582802,244786,70,83.17,Whiplash (2014)
|
|
106
|
+
112556,2267998,210577,75,79.17,Gone Girl (2014)
|
|
107
|
+
1968,88847,2108,66,76.48,The Breakfast Club (1985)
|
|
108
|
+
2115,87469,87,62,73.99,Indiana Jones and the Temple of Doom (1984)
|
|
109
|
+
4816,196229,9398,61,64.92,Zoolander (2001)
|
|
110
|
+
8464,390521,9372,60,68.73,Super Size Me (2004)
|
|
111
|
+
32587,401792,187,60,74.97,Sin City (2005)
|
|
112
|
+
33493,121766,1895,61,68.07,Star Wars: Episode III - Revenge of the Sith (2005)
|
|
113
|
+
96079,1074638,37724,60,74.95,Skyfall (2012)
|
|
114
|
+
116797,2084970,205596,63,79.9,The Imitation Game (2014)
|
|
115
|
+
3949,180093,641,37,79.62,Requiem for a Dream (2000)
|
|
116
|
+
5507,295701,7451,42,55.74,xXx (2002)
|
|
117
|
+
8984,349903,163,45,67.11,Ocean's Twelve (2004)
|