haystack-ml-stack 0.4.12__tar.gz → 0.4.13__tar.gz

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-ml-stack
-Version: 0.4.12
+Version: 0.4.13
 Summary: Functions related to Haystack ML
 Author-email: Oscar Vega <oscar@haystack.tv>
 License: MIT
@@ -8,6 +8,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: protobuf==6.33.2
 Requires-Dist: orjson==3.11.7
+Requires-Dist: numpy>=1.26
 Provides-Extra: server
 Requires-Dist: pydantic==2.5.0; extra == "server"
 Requires-Dist: cachetools==5.5.2; extra == "server"

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/pyproject.toml

@@ -5,13 +5,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "haystack-ml-stack"
-version = "0.4.12"
+version = "0.4.13"
 description = "Functions related to Haystack ML"
 readme = "README.md"
 authors = [{ name = "Oscar Vega", email = "oscar@haystack.tv" }]
 requires-python = ">=3.11"
 dependencies = [
-    "protobuf==6.33.2", "orjson==3.11.7"
+    "protobuf==6.33.2", "orjson==3.11.7", "numpy>=1.26"
 ]
 license = { text = "MIT" }
 
@@ -25,4 +25,4 @@ server = [
     "pydantic-settings==2.2",
     "newrelic==11.1.0",
     "confluent-kafka==2.13.0"
-]
+]

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack/_serializers.py

@@ -1,6 +1,8 @@
 from .generated.v1 import features_pb2 as features_pb2_v1
+from .history import CONTEXT_CODES, STREAM_URL_PREFIX
 from google.protobuf.message import Message
 from google.protobuf.json_format import ParseDict as ProtoParseDict
+import numpy as np
 import typing as _t
 from abc import ABC, abstractmethod
 
@@ -305,6 +307,64 @@ class GlobalChannelsSerializerV1(SimpleSerializer):
         return root_msg
 
 
+class StreamEmbeddingSerializerV1(SimpleSerializer):
+    """The raw 512-coordinate prefix, normalized in float32 then stored as float16."""
+
+    def __init__(self):
+        super().__init__(msg_class=features_pb2_v1.StreamEmbedding)
+
+    def serialize(self, value) -> bytes:
+        return self.build_msg(value).SerializeToString()
+
+    def build_msg(self, value) -> features_pb2_v1.StreamEmbedding:
+        assert value["version"] == 1, "Wrong version given!"
+        data = np.asarray(value["data"], dtype=np.float32)
+        assert data.shape == (512,), "Expected the first 512 coordinates"
+        norm = np.linalg.norm(data)
+        if not np.isfinite(norm) or norm <= 0:
+            raise ValueError("Expected a finite, nonzero 512-coordinate vector norm")
+        message = self.msg_class()
+        message.version = 1
+        message.data = (data / norm).astype("<f2").tobytes()
+        return message
+
+
+class UserHistorySerializerV1(SimpleSerializer):
+    """Pack the SQL's selected states."""
+
+    def __init__(self):
+        super().__init__(msg_class=features_pb2_v1.UserHistory)
+
+    def serialize(self, value) -> bytes:
+        return self.build_msg(value).SerializeToString()
+
+    def build_msg(self, value) -> features_pb2_v1.UserHistory:
+        assert value["version"] == 1, "Wrong version given!"
+        data = value["data"]
+        columns = ["stream_urls", "event_at_ms", "seconds_watched", "contexts", "browsed"]
+        lengths = {len(data[column]) for column in columns}
+        assert len(lengths) == 1, "History columns must have equal lengths"
+        message = self.msg_class()
+        message.version = 1
+        message.data.snapshot_at_ms = data["snapshot_at_ms"]
+        count = len(data["stream_urls"])
+        assert count <= 256, "Expected at most 256 history states"
+        flags = bytearray()
+        for url, at_ms, context, browsed in zip(
+            data["stream_urls"], data["event_at_ms"], data["contexts"], data["browsed"]
+        ):
+            suffix = url.removeprefix(STREAM_URL_PREFIX)
+            message.data.stream_ids.append(
+                suffix if url.startswith(STREAM_URL_PREFIX) and not suffix.startswith(":")
+                else ":" + url
+            )
+            message.data.event_age_ms.append(data["snapshot_at_ms"] - at_ms)
+            flags.append(CONTEXT_CODES.get(context, 1) | (128 if browsed else 0))
+        message.data.seconds_watched.extend(data["seconds_watched"])
+        message.data.context_browsed = bytes(flags)
+        return message
+
+
 class PassThroughSerializer(Serializer):
     def serialize(self, value):
         return value
@@ -326,6 +386,8 @@ stream_similarity_scores_serializer_v1 = StreamSimilaritySerializerV1()
 global_playlist_stats_serializer_v1 = GlobalPlaylistStatsSerializerV1()
 user_playlist_stats_serializer_v1 = UserPlaylistStatsSerializerV1()
 global_channels_serializer_v1 = GlobalChannelsSerializerV1()
+stream_embedding_serializer_v1 = StreamEmbeddingSerializerV1()
+user_history_serializer_v1 = UserHistorySerializerV1()
 
 
 class FeatureRegistryId(_t.NamedTuple):
@@ -468,6 +530,18 @@ global_channels_v1_features: list[FeatureRegistryId] = [
     ),
 ]
 
+gemini_title_transcript_512_v1_features: list[FeatureRegistryId] = [
+    FeatureRegistryId(
+        entity_type="STREAM",
+        feature_id="EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512",
+        version="v1",
+    ),
+]
+
+user_history_v1_features: list[FeatureRegistryId] = [
+    FeatureRegistryId(entity_type="USER", feature_id="HISTORY#5M#256", version="v1"),
+]
+
 features_serializer_tuples: list[tuple[list[FeatureRegistryId], Serializer]] = [
     (stream_pwatched_v0_features, stream_pwatched_serializer_v0),
     (stream_pwatched_v1_features, stream_pwatched_serializer_v1),
@@ -494,6 +568,8 @@ features_serializer_tuples: list[tuple[list[FeatureRegistryId], Serializer]] = [
         global_channels_v1_features,
         global_channels_serializer_v1,
     ),
+    (gemini_title_transcript_512_v1_features, stream_embedding_serializer_v1),
+    (user_history_v1_features, user_history_serializer_v1),
 ]
 
 SerializerRegistry: dict[FeatureRegistryId, Serializer] = {

haystack_ml_stack-0.4.13/src/haystack_ml_stack/_version.py

@@ -0,0 +1 @@
+__version__ = "0.4.13"

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack/app.py

@@ -70,9 +70,14 @@ async def load_model(state, cfg: Settings) -> None:
                 (entity_type, feature_id)
                 for entity_type, feature_id, _ in SerializerRegistry.keys()
             )
+            # Optional {"user_feature": ..., "stream_features": [...]}: the streams
+            # in that history record get these stream features.
+            history = state["model"].get("history")
+            history_features = history["stream_features"] if history else []
             all_features = set(
                 [("STREAM", feature_name) for feature_name in state["stream_features"]]
                 + [("USER", feature_name) for feature_name in state["user_features"]]
+                + [("STREAM", feature_name) for feature_name in history_features]
             )
             invalid_features = all_features.difference(valid_features)
             if invalid_features:
@@ -230,6 +235,8 @@ def create_app(
         model = state["model"]
         stream_features = model.get("stream_features", []) or []
         user_features = model.get("user_features", []) or []
+        history = model.get("history")
+        history_streams: List[Dict[str, Any]] = []
         retrieval_meta = FeatureRetrievalMeta(
             cache_misses=0,
             stream_cache_misses=0,
@@ -240,7 +247,7 @@ def create_app(
             dynamo_ms=0,
             parsing_ms=0,
         )
-        if stream_features:
+        if stream_features or user_features:
             try:
                 retrieval_meta = await set_all_features(
                     dynamo_client=state["dynamo_client"],
@@ -252,6 +259,8 @@ def create_app(
                     user_features_cache=user_features_cache,
                     features_table=cfg.features_table,
                     cache_sep=cfg.cache_separator,
+                    history=history,
+                    history_streams=history_streams,
                 )
             except exceptions.InvalidFeaturesException as e:
                 logger.error(
@@ -280,6 +289,8 @@ def create_app(
         try:
             preprocess_start = time.perf_counter_ns()
             model["params"]["query_params"] = query_params
+            # Outside `user`, so that the history vectors are not logged.
+            model["params"]["history_streams"] = history_streams
             model_input = model["preprocess"](
                 user,
                 streams,

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack/dynamo.py

@@ -9,6 +9,7 @@ from boto3.dynamodb.types import TypeDeserializer
 
 from . import exceptions
 from ._serializers import FeatureRegistryId, SerializerRegistry
+from .history import history_stream_urls
 from .utils import _complete_features_for_channels, DEFAULT_CHANNELS
 
 logger = logging.getLogger(__name__)
@@ -106,7 +107,13 @@ async def set_all_features(
     features_table: str,
     cache_sep: str,
     dynamo_client,
+    history: Dict[str, Any] | None = None,
+    history_streams: List[Dict[str, Any]] | None = None,
 ) -> FeatureRetrievalMeta:
+    """Fetch features in parallel, filling history_streams when history is configured.
+
+    Cached history vectors join the primary fetch; cold history uses a second stage.
+    """
     time_start = time.perf_counter_ns()
     if not streams or (not stream_features and not user_features):
         return FeatureRetrievalMeta(
@@ -119,24 +126,13 @@ async def set_all_features(
             dynamo_ms=0,
             parsing_ms=0,
         )
-    cache_miss: Dict[str, Dict[str, Any]] = {}
-    all_feature_keys = [*stream_features, *user_features]
+    cache_miss: Dict[str, List[Dict[str, Any]]] = {}
+    history_features = history["stream_features"] if history else []
+    if history_streams is None:
+        history_streams = []
+    all_feature_keys = [*stream_features, *user_features, *history_features]
     cache_delay_obj: dict[str, float] = {f: 0 for f in all_feature_keys}
     now = datetime.datetime.utcnow()
-    for f in stream_features:
-        for s in streams:
-            cache_miss, cache_delay_obj = _check_cache(
-                obj=s,
-                id_type="STREAM",
-                id_key=s["streamUrl"],
-                feature_key=f,
-                cache_sep=cache_sep,
-                features_cache=stream_features_cache,
-                cache_miss=cache_miss,
-                cache_delay=cache_delay_obj,
-                now=now,
-            )
-    stream_cache_misses = len(cache_miss)
     for f in user_features:
         cache_miss, cache_delay_obj = _check_cache(
             obj=user,
@@ -149,7 +145,26 @@ async def set_all_features(
             cache_delay=cache_delay_obj,
             now=now,
         )
-    user_cache_misses = len(cache_miss) - stream_cache_misses
+    user_cache_misses = len(cache_miss)
+    cached_history = user.get(history["user_feature"]) if history else None
+    if history:
+        history_streams[:] = [{"streamUrl": url} for url in history_stream_urls(cached_history)]
+    # A cached history reveals its vector keys before the first parallel fetch.
+    for targets, features in ((streams, stream_features), (history_streams, history_features)):
+        for f in features:
+            for s in targets:
+                cache_miss, cache_delay_obj = _check_cache(
+                    obj=s,
+                    id_type="STREAM",
+                    id_key=s["streamUrl"],
+                    feature_key=f,
+                    cache_sep=cache_sep,
+                    features_cache=stream_features_cache,
+                    cache_miss=cache_miss,
+                    cache_delay=cache_delay_obj,
+                    now=now,
+                )
+    stream_cache_misses = len(cache_miss) - user_cache_misses
     valid_cache_delays = list(v for v in cache_delay_obj.values() if v > 0)
     cache_delay = min(valid_cache_delays) if valid_cache_delays else 0
     if not cache_miss:
@@ -198,7 +213,7 @@ async def set_all_features(
     updated_keys = set()
     for item in items:
         full_id = item["pk"]["S"]
-        id_type, id_key = full_id.split("#")
+        id_type, id_key = full_id.split("#", 1)
         feature_name = item["sk"]["S"]
         if id_type == "STREAM":
             cache_to_use = stream_features_cache
@@ -242,7 +257,8 @@ async def set_all_features(
         }
 
         if cache_key in cache_miss:
-            cache_miss[cache_key][feature_name] = value
+            for obj in cache_miss[cache_key]:
+                obj[feature_name] = value
             updated_keys.add(cache_key)
     parsing_end = time.perf_counter_ns()
     # Save keys that were not found in DynamoDB with None value
@@ -258,7 +274,7 @@ async def set_all_features(
                     "cache_ttl_in_seconds": 6 * 3600,
                 }
     end_time = time.perf_counter_ns()
-    return FeatureRetrievalMeta(
+    meta = FeatureRetrievalMeta(
         cache_misses=user_cache_misses + stream_cache_misses,
         user_cache_misses=user_cache_misses,
         stream_cache_misses=stream_cache_misses,
@@ -268,6 +284,26 @@ async def set_all_features(
         dynamo_ms=_perf_counter_ns_delta_in_ms(dynamo_start, dynamo_end),
         parsing_ms=_perf_counter_ns_delta_in_ms(dynamo_end, parsing_end),
     )
+    if history and cached_history is None:
+        record = user.get(history["user_feature"])
+        history_streams[:] = [{"streamUrl": url} for url in history_stream_urls(record)]
+        # An uncached history needs a second stage; its batches are still parallel.
+        history_meta = await set_all_features(
+            user=user, streams=history_streams, stream_features=history_features,
+            user_features=[], stream_features_cache=stream_features_cache,
+            user_features_cache=user_features_cache, features_table=features_table,
+            cache_sep=cache_sep, dynamo_client=dynamo_client,
+        )
+        meta = meta._replace(
+            cache_misses=meta.cache_misses + history_meta.cache_misses,
+            stream_cache_misses=meta.stream_cache_misses + history_meta.stream_cache_misses,
+            retrieval_ms=_perf_counter_ns_delta_in_ms(time_start, time.perf_counter_ns()),
+            success=meta.success and history_meta.success,
+            cache_delay_minutes=max(meta.cache_delay_minutes, history_meta.cache_delay_minutes),
+            dynamo_ms=meta.dynamo_ms + history_meta.dynamo_ms,
+            parsing_ms=meta.parsing_ms + history_meta.parsing_ms,
+        )
+    return meta
 
 
 _MOBILE_OS = {"ios", "android", "iphone", "galaxy"}
@@ -296,7 +332,7 @@ async def create_channel_candidates(
     os_cat = _get_os_cat(user.get("clientOs", ""))
 
     global_holder: Dict[str, Any] = {}
-    cache_miss: Dict[str, Dict[str, Any]] = {}
+    cache_miss: Dict[str, List[Dict[str, Any]]] = {}
     all_feature_keys = [*global_features, *user_features]
     cache_delay_obj: dict[str, float] = {f: 0 for f in all_feature_keys}
     now = datetime.datetime.utcnow()
@@ -437,7 +473,8 @@ async def create_channel_candidates(
         }
 
         if cache_key in cache_miss:
-            cache_miss[cache_key][feature_name] = value
+            for obj in cache_miss[cache_key]:
+                obj[feature_name] = value
             updated_keys.add(cache_key)
 
     parsing_end = time.perf_counter_ns()
@@ -594,7 +631,7 @@ def _check_cache(
                 (now - cached["inserted_at"]).total_seconds(),
             )
     else:
-        cache_miss[key] = obj
+        cache_miss.setdefault(key, []).append(obj)
     return cache_miss, cache_delay
 
 

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack/generated/v1/features_pb2.py

@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x66\x65\x61tures.proto\x12\x1ahaystack_ml_stack.features\"7\n\x12\x45ntryContextCounts\x12\x10\n\x08\x61ttempts\x18\x01 \x01(\x05\x12\x0f\n\x07watched\x18\x02 \x01(\x05\"_\n\x0cSelectCounts\x12\x15\n\rtotal_selects\x18\x01 \x01(\x05\x12!\n\x19total_selects_and_watched\x18\x02 \x01(\x05\x12\x15\n\rtotal_browsed\x18\x03 \x01(\x05\"\xf3\x02\n\x14\x45ntryContextPWatched\x12@\n\x08\x61utoplay\x18\x01 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12\x41\n\tsel_thumb\x18\x02 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12\x43\n\x0b\x63hoose_next\x18\x03 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12@\n\x08\x63h_swtch\x18\x04 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12O\n\x17launch_first_in_session\x18\x05 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\"\x85\x02\n\x0fPositionPSelect\x12;\n\tfirst_pos\x18\x01 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12<\n\nsecond_pos\x18\x02 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12;\n\tthird_pos\x18\x03 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12:\n\x08rest_pos\x18\x04 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\"\xa9\x01\n\x1f\x42rowsedDebiasedPositionPSelects\x12\x44\n\x0fup_to_4_browsed\x18\x01 \x01(\x0b\x32+.haystack_ml_stack.features.PositionPSelect\x12@\n\x0b\x61ll_browsed\x18\x02 \x01(\x0b\x32+.haystack_ml_stack.features.PositionPSelect\"\xb8\x01\n\x16PlaylistStatsForGlobal\x12\x15\n\rwatched_count\x18\x01 \x01(\x05\x12\x19\n\x11not_watched_count\x18\x02 \x01(\x05\x12\x1b\n\x13\x63\x61pped_watched_secs\x18\x03 \x01(\x02\x12\x1f\n\x17\x63\x61pped_not_watched_secs\x18\x04 \x01(\x02\x12\x14\n\x0cwatched_secs\x18\x05 \x01(\x02\x12\x18\n\x10not_watched_secs\x18\x06 \x01(\x02\"\xc5\x01\n\x14PlaylistStatsForUser\x12\x12\n\ntotal_days\x18\x01 \x01(\x05\x12\x12\n\nstart_days\x18\x02 \x01(\x05\x12\x13\n\x0b\x61\x63tive_days\x18\x03 \x01(\x05\x12\x15\n\rtotal_watched\x18\x04 \x01(\x02\x12\x1c\n\x14\x63\x61pped_total_watched\x18\x05 \x01(\x02\x12 \n\x18\x63\x61pped_total_log_watched\x18\x06 \x01(\x02\x12\x19\n\x11\x66irst_active_date\x18\x07 \x01(\t\"^\n\x07\x43hannel\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x16\n\x0e\x63\x61tegory_group\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\x05\x12\x19\n\x11required_features\x18\x04 \x03(\t\"k\n\rStreamPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12I\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects\"a\n\x0eStreamPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12>\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched\"_\n\x0cUserPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12>\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched\"\xda\x01\n\x19UserPersonalizingPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12M\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32?.haystack_ml_stack.features.UserPersonalizingPWatched.DataEntry\x1a]\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched:\x02\x38\x01\"i\n\x0bUserPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12I\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects\"\xe3\x01\n\x18UserPersonalizingPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12L\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32>.haystack_ml_stack.features.UserPersonalizingPSelect.DataEntry\x1ah\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12J\n\x05value\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects:\x02\x38\x01\"\xa2\x01\n\x16StreamSimilarityScores\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12J\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32<.haystack_ml_stack.features.StreamSimilarityScores.DataEntry\x1a+\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\xd0\x01\n\x13GlobalPlaylistStats\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12G\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x39.haystack_ml_stack.features.GlobalPlaylistStats.DataEntry\x1a_\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x41\n\x05value\x18\x02 \x01(\x0b\x32\x32.haystack_ml_stack.features.PlaylistStatsForGlobal:\x02\x38\x01\"\xca\x01\n\x11UserPlaylistStats\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x45\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x37.haystack_ml_stack.features.UserPlaylistStats.DataEntry\x1a]\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.PlaylistStatsForUser:\x02\x38\x01\"T\n\x0eGlobalChannels\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x31\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32#.haystack_ml_stack.features.Channelb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x66\x65\x61tures.proto\x12\x1ahaystack_ml_stack.features\"7\n\x12\x45ntryContextCounts\x12\x10\n\x08\x61ttempts\x18\x01 \x01(\x05\x12\x0f\n\x07watched\x18\x02 \x01(\x05\"_\n\x0cSelectCounts\x12\x15\n\rtotal_selects\x18\x01 \x01(\x05\x12!\n\x19total_selects_and_watched\x18\x02 \x01(\x05\x12\x15\n\rtotal_browsed\x18\x03 \x01(\x05\"\xf3\x02\n\x14\x45ntryContextPWatched\x12@\n\x08\x61utoplay\x18\x01 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12\x41\n\tsel_thumb\x18\x02 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12\x43\n\x0b\x63hoose_next\x18\x03 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12@\n\x08\x63h_swtch\x18\x04 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\x12O\n\x17launch_first_in_session\x18\x05 \x01(\x0b\x32..haystack_ml_stack.features.EntryContextCounts\"\x85\x02\n\x0fPositionPSelect\x12;\n\tfirst_pos\x18\x01 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12<\n\nsecond_pos\x18\x02 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12;\n\tthird_pos\x18\x03 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\x12:\n\x08rest_pos\x18\x04 \x01(\x0b\x32(.haystack_ml_stack.features.SelectCounts\"\xa9\x01\n\x1f\x42rowsedDebiasedPositionPSelects\x12\x44\n\x0fup_to_4_browsed\x18\x01 \x01(\x0b\x32+.haystack_ml_stack.features.PositionPSelect\x12@\n\x0b\x61ll_browsed\x18\x02 \x01(\x0b\x32+.haystack_ml_stack.features.PositionPSelect\"\xb8\x01\n\x16PlaylistStatsForGlobal\x12\x15\n\rwatched_count\x18\x01 \x01(\x05\x12\x19\n\x11not_watched_count\x18\x02 \x01(\x05\x12\x1b\n\x13\x63\x61pped_watched_secs\x18\x03 \x01(\x02\x12\x1f\n\x17\x63\x61pped_not_watched_secs\x18\x04 \x01(\x02\x12\x14\n\x0cwatched_secs\x18\x05 \x01(\x02\x12\x18\n\x10not_watched_secs\x18\x06 \x01(\x02\"\xc5\x01\n\x14PlaylistStatsForUser\x12\x12\n\ntotal_days\x18\x01 \x01(\x05\x12\x12\n\nstart_days\x18\x02 \x01(\x05\x12\x13\n\x0b\x61\x63tive_days\x18\x03 \x01(\x05\x12\x15\n\rtotal_watched\x18\x04 \x01(\x02\x12\x1c\n\x14\x63\x61pped_total_watched\x18\x05 \x01(\x02\x12 \n\x18\x63\x61pped_total_log_watched\x18\x06 \x01(\x02\x12\x19\n\x11\x66irst_active_date\x18\x07 \x01(\t\"^\n\x07\x43hannel\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x16\n\x0e\x63\x61tegory_group\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\x05\x12\x19\n\x11required_features\x18\x04 \x03(\t\"k\n\rStreamPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12I\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects\"a\n\x0eStreamPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12>\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched\"_\n\x0cUserPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12>\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched\"\xda\x01\n\x19UserPersonalizingPWatched\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12M\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32?.haystack_ml_stack.features.UserPersonalizingPWatched.DataEntry\x1a]\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.EntryContextPWatched:\x02\x38\x01\"i\n\x0bUserPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12I\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects\"\xe3\x01\n\x18UserPersonalizingPSelect\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12L\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32>.haystack_ml_stack.features.UserPersonalizingPSelect.DataEntry\x1ah\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12J\n\x05value\x18\x02 \x01(\x0b\x32;.haystack_ml_stack.features.BrowsedDebiasedPositionPSelects:\x02\x38\x01\"\xa2\x01\n\x16StreamSimilarityScores\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12J\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32<.haystack_ml_stack.features.StreamSimilarityScores.DataEntry\x1a+\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\xd0\x01\n\x13GlobalPlaylistStats\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12G\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x39.haystack_ml_stack.features.GlobalPlaylistStats.DataEntry\x1a_\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x41\n\x05value\x18\x02 \x01(\x0b\x32\x32.haystack_ml_stack.features.PlaylistStatsForGlobal:\x02\x38\x01\"\xca\x01\n\x11UserPlaylistStats\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x45\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x37.haystack_ml_stack.features.UserPlaylistStats.DataEntry\x1a]\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.haystack_ml_stack.features.PlaylistStatsForUser:\x02\x38\x01\"T\n\x0eGlobalChannels\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x31\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32#.haystack_ml_stack.features.Channel\"0\n\x0fStreamEmbedding\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"\x83\x01\n\rHistoryStates\x12\x12\n\nstream_ids\x18\x01 \x03(\t\x12\x14\n\x0c\x65vent_age_ms\x18\x02 \x03(\x04\x12\x17\n\x0fseconds_watched\x18\x03 \x03(\x02\x12\x17\n\x0f\x63ontext_browsed\x18\x04 \x01(\x0c\x12\x16\n\x0esnapshot_at_ms\x18\x06 \x01(\x03\"]\n\x0bUserHistory\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x37\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32).haystack_ml_stack.features.HistoryStatesJ\x04\x08\x03\x10\x04\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -87,4 +87,10 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_USERPLAYLISTSTATS_DATAENTRY']._serialized_end=2935
   _globals['_GLOBALCHANNELS']._serialized_start=2937
   _globals['_GLOBALCHANNELS']._serialized_end=3021
+  _globals['_STREAMEMBEDDING']._serialized_start=3023
+  _globals['_STREAMEMBEDDING']._serialized_end=3071
+  _globals['_HISTORYSTATES']._serialized_start=3074
+  _globals['_HISTORYSTATES']._serialized_end=3205
+  _globals['_USERHISTORY']._serialized_start=3207
+  _globals['_USERHISTORY']._serialized_end=3300
 # @@protoc_insertion_point(module_scope)

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack/generated/v1/features_pb2.pyi

@@ -218,3 +218,33 @@ class GlobalChannels(_message.Message):
     version: int
     data: _containers.RepeatedCompositeFieldContainer[Channel]
     def __init__(self, version: _Optional[int] = ..., data: _Optional[_Iterable[_Union[Channel, _Mapping]]] = ...) -> None: ...
+
+class StreamEmbedding(_message.Message):
+    __slots__ = ()
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    DATA_FIELD_NUMBER: _ClassVar[int]
+    version: int
+    data: bytes
+    def __init__(self, version: _Optional[int] = ..., data: _Optional[bytes] = ...) -> None: ...
+
+class HistoryStates(_message.Message):
+    __slots__ = ()
+    STREAM_IDS_FIELD_NUMBER: _ClassVar[int]
+    EVENT_AGE_MS_FIELD_NUMBER: _ClassVar[int]
+    SECONDS_WATCHED_FIELD_NUMBER: _ClassVar[int]
+    CONTEXT_BROWSED_FIELD_NUMBER: _ClassVar[int]
+    SNAPSHOT_AT_MS_FIELD_NUMBER: _ClassVar[int]
+    stream_ids: _containers.RepeatedScalarFieldContainer[str]
+    event_age_ms: _containers.RepeatedScalarFieldContainer[int]
+    seconds_watched: _containers.RepeatedScalarFieldContainer[float]
+    context_browsed: bytes
+    snapshot_at_ms: int
+    def __init__(self, stream_ids: _Optional[_Iterable[str]] = ..., event_age_ms: _Optional[_Iterable[int]] = ..., seconds_watched: _Optional[_Iterable[float]] = ..., context_browsed: _Optional[bytes] = ..., snapshot_at_ms: _Optional[int] = ...) -> None: ...
+
+class UserHistory(_message.Message):
+    __slots__ = ()
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    DATA_FIELD_NUMBER: _ClassVar[int]
+    version: int
+    data: HistoryStates
+    def __init__(self, version: _Optional[int] = ..., data: _Optional[_Union[HistoryStates, _Mapping]] = ...) -> None: ...

haystack_ml_stack-0.4.13/src/haystack_ml_stack/history.py

@@ -0,0 +1,38 @@
+"""Compact history v1: shared URL and state decoding for serving and replay."""
+
+STREAM_URL_PREFIX = "http://haystack.tv/id/"
+# Wire codes are append-only. Missing and unsupported contexts are distinct.
+CONTEXTS = ("", "__unsupported__", "autoplay", "sel thumb", "pres nxt",
+            "swpe nxt", "ch swtch", "launch", "pl refrsh")
+CONTEXT_CODES = {context: code for code, context in enumerate(CONTEXTS)}
+CONTEXT_CODES.update({"null": 0, "__missing__": 0})
+
+
+def history_stream_urls(record):
+    """Rebuild exact vector keys; ':' explicitly escapes a full URL."""
+    if record is None:
+        return []
+    return [value[1:] if value.startswith(":") else STREAM_URL_PREFIX + value
+            for value in record.data.stream_ids]
+
+
+def history_entries(record):
+    """Yield URL, timestamp ms, seconds, context, browsed, watched (over 25 s, not browsed)."""
+    if record is None:
+        return
+    data = record.data
+    count = len(data.stream_ids)
+    if not (record.version == 1 and count <= 256
+            and len(data.event_age_ms) == len(data.seconds_watched)
+            == len(data.context_browsed) == count):
+        raise ValueError("Invalid compact history version or column lengths")
+    for url, age, seconds, flags in zip(
+        history_stream_urls(record), data.event_age_ms,
+        data.seconds_watched, data.context_browsed,
+    ):
+        context = flags & 127
+        if context >= len(CONTEXTS):
+            raise ValueError("Unknown compact history context code")
+        browsed = bool(flags & 128)
+        yield (url, data.snapshot_at_ms - age, seconds, CONTEXTS[context],
+               browsed, seconds > 25 and not browsed)

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-ml-stack
-Version: 0.4.12
+Version: 0.4.13
 Summary: Functions related to Haystack ML
 Author-email: Oscar Vega <oscar@haystack.tv>
 License: MIT
@@ -8,6 +8,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: protobuf==6.33.2
 Requires-Dist: orjson==3.11.7
+Requires-Dist: numpy>=1.26
 Provides-Extra: server
 Requires-Dist: pydantic==2.5.0; extra == "server"
 Requires-Dist: cachetools==5.5.2; extra == "server"

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ src/haystack_ml_stack/app.py
 src/haystack_ml_stack/cache.py
 src/haystack_ml_stack/dynamo.py
 src/haystack_ml_stack/exceptions.py
+src/haystack_ml_stack/history.py
 src/haystack_ml_stack/model_store.py
 src/haystack_ml_stack/settings.py
 src/haystack_ml_stack/utils.py
@@ -20,5 +21,6 @@ src/haystack_ml_stack/generated/__init__.py
 src/haystack_ml_stack/generated/v1/__init__.py
 src/haystack_ml_stack/generated/v1/features_pb2.py
 src/haystack_ml_stack/generated/v1/features_pb2.pyi
+tests/test_history_fetch.py
 tests/test_serializers.py
 tests/test_utils.py

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/src/haystack_ml_stack.egg-info/requires.txt

@@ -1,5 +1,6 @@
 protobuf==6.33.2
 orjson==3.11.7
+numpy>=1.26
 
 [server]
 pydantic==2.5.0

haystack_ml_stack-0.4.13/tests/test_history_fetch.py

@@ -0,0 +1,226 @@
+import asyncio
+import base64
+import datetime
+import json
+
+import aiobotocore.session
+import newrelic.agent
+import pytest
+from fastapi import BackgroundTasks, Response
+from starlette.requests import Request
+
+from haystack_ml_stack import FeatureRegistryId, SerializerRegistry
+from haystack_ml_stack.app import create_app
+from haystack_ml_stack.cache import make_features_cache
+from haystack_ml_stack.dynamo import _build_cache_key, set_all_features
+from haystack_ml_stack._kafka import default_serialization
+from haystack_ml_stack.settings import Settings
+
+EMBEDDING = "EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512"
+HISTORY = "HISTORY#5M#256"
+CANDIDATE = "http://haystack.tv/id/candidate"
+WATCHED = "https://example.com/watch,video#clip"
+NO_VECTOR = "http://haystack.tv/id/no-vector"
+
+
+class FakeDynamo:
+    """batch_get_item over in-memory items."""
+
+    def __init__(self, items=()):
+        self.items = {(i["pk"]["S"], i["sk"]["S"]): i for i in items}
+        self.calls = 0
+        self.batches = []
+        self.active = 0
+        self.max_active = 0
+
+    async def batch_get_item(self, RequestItems):
+        self.calls += 1
+        ((table, request),) = RequestItems.items()
+        keys = [(k["pk"]["S"], k["sk"]["S"]) for k in request["Keys"]]
+        self.batches.append(keys)
+        self.active += 1
+        self.max_active = max(self.max_active, self.active)
+        try:
+            await asyncio.sleep(0)
+            return {"Responses": {table: [self.items[k] for k in keys if k in self.items]}}
+        finally:
+            self.active -= 1
+
+
+class FakeSession:
+    def __init__(self, client):
+        self.client = client
+
+    def create_client(self, *args, **kwargs):
+        client = self.client
+
+        class Context:
+            async def __aenter__(self):
+                return client
+
+            async def __aexit__(self, *exc):
+                return False
+
+        return Context()
+
+
+def item(pk, sk, value):
+    return {
+        "pk": {"S": pk},
+        "sk": {"S": sk},
+        "value": {"B": value},
+        "version": {"S": "v1"},
+        "cache_ttl_in_seconds": {"N": "3600"},
+    }
+
+
+@pytest.mark.parametrize("candidate_features", [[EMBEDDING], []])
+def test_history_vectors_reach_the_model_outside_the_logged_user(candidate_features):
+    history = SerializerRegistry[FeatureRegistryId("USER", HISTORY, "v1")].serialize(
+        {
+            "version": 1,
+            "data": {
+                "snapshot_at_ms": 1_758_499_200_000,
+                "stream_urls": [WATCHED, NO_VECTOR],
+                "event_at_ms": [1_758_400_000_000, 1_758_450_000_000],
+                "seconds_watched": [120.0, 0.0],
+                "contexts": ["sel thumb", "sel thumb"],
+                "browsed": [False, True],
+            },
+        }
+    )
+    vector = SerializerRegistry[FeatureRegistryId("STREAM", EMBEDDING, "v1")].serialize(
+        {"version": 1, "data": [0.01] * 512}
+    )
+    client = FakeDynamo(
+        [
+            item("USER#u1", HISTORY, history),
+            item(f"STREAM#{CANDIDATE}", EMBEDDING, vector),
+            item(f"STREAM#{WATCHED}", EMBEDDING, vector),
+        ]
+    )
+    seen = {}
+
+    def preprocess(user, streams, playlist, params):
+        seen["user"] = dict(user)
+        seen["history_streams"] = params["history_streams"]
+        return streams
+
+    model = {
+        "preprocess": preprocess,
+        "predict": lambda streams, params: {s["streamUrl"]: 0.5 for s in streams},
+        "params": {},
+        "stream_features": candidate_features,
+        "user_features": [HISTORY],
+        "history": {"user_feature": HISTORY, "stream_features": [EMBEDDING]},
+    }
+    events = []
+    original_session = aiobotocore.session.get_session
+    original_record = newrelic.agent.record_custom_event
+    aiobotocore.session.get_session = lambda: FakeSession(client)
+    newrelic.agent.record_custom_event = lambda name, meta: events.append(meta)
+    try:
+        app = create_app(Settings(), preloaded_model=model)
+        score = next(r for r in app.routes if getattr(r, "path", "") == "/score")
+        body = json.dumps({"user": {"userid": "u1"}, "streams": [{"streamUrl": CANDIDATE}]}).encode()
+
+        async def receive():
+            return {"type": "http.request", "body": body, "more_body": False}
+
+        async def call():
+            scope = {"type": "http", "method": "POST", "path": "/score",
+                     "headers": [], "query_string": b""}
+            async with app.router.lifespan_context(app):
+                return await score.endpoint(Request(scope, receive), Response(), BackgroundTasks())
+
+        output = asyncio.run(call())
+    finally:
+        aiobotocore.session.get_session = original_session
+        newrelic.agent.record_custom_event = original_record
+
+    assert output == {CANDIDATE: 0.5}
+    assert [s["streamUrl"] for s in seen["history_streams"]] == [WATCHED, NO_VECTOR]
+    assert len(seen["history_streams"][0][EMBEDDING].data) == 1024  # 512 float16
+    assert EMBEDDING not in seen["history_streams"][1]
+    assert set(seen["user"]) == {"userid", HISTORY}
+    # The retrieval metrics count both reads: the candidate's vector and the history
+    # record, then the vectors of the history's two streams.
+    (event,) = events
+    expected = (4, 3, 1) if candidate_features else (3, 2, 1)
+    assert (event["cache_misses"], event["stream_cache_misses"], event["user_cache_misses"]) == expected
+    assert event["retrieval_success"] == 1
+
+
+@pytest.mark.parametrize("cache_state", ["cold", "history", "all"])
+def test_parallel_history_fetch_and_shared_vector_assignment(cache_state):
+    import orjson
+
+    candidates = [f"http://haystack.tv/id/c{i}" for i in range(100)]
+    urls = candidates[:50] + [f"http://haystack.tv/id/h{i}" for i in range(206)]
+    history_serializer = SerializerRegistry[FeatureRegistryId("USER", HISTORY, "v1")]
+    vector_serializer = SerializerRegistry[FeatureRegistryId("STREAM", EMBEDDING, "v1")]
+    history = history_serializer.serialize({
+        "version": 1,
+        "data": {"snapshot_at_ms": 1_758_499_200_000,
+                 "stream_urls": urls, "event_at_ms": [1_758_400_000_000] * 256,
+                 "seconds_watched": [30.] * 256, "contexts": ["pres nxt"] * 256,
+                 "browsed": [False] * 256},
+    })
+    vector = vector_serializer.serialize({"version": 1, "data": [.01] * 512})
+    rows = [item("USER#u1", HISTORY, history)] + [
+        item("STREAM#" + url, EMBEDDING, vector) for url in dict.fromkeys(candidates + urls)
+    ]
+    client = FakeDynamo(rows)
+    user_cache, stream_cache = make_features_cache(1000), make_features_cache(1000)
+    def cache_value(cache, kind, entity, feature, value):
+        cache[_build_cache_key(kind, entity, feature, "--")] = {
+            "value": value, "cache_ttl_in_seconds": 3600,
+            "inserted_at": datetime.datetime.utcnow(),
+        }
+    if cache_state != "cold":
+        cache_value(user_cache, "USER", "u1", HISTORY, history_serializer.deserialize(history))
+    if cache_state == "all":
+        for url in dict.fromkeys(candidates + urls):
+            cache_value(stream_cache, "STREAM", url, EMBEDDING, vector_serializer.deserialize(vector))
+    user, streams, history_streams = {"userid": "u1"}, [{"streamUrl": u} for u in candidates], []
+    meta = asyncio.run(set_all_features(
+        user=user, streams=streams, stream_features=[EMBEDDING], user_features=[HISTORY],
+        history={"user_feature": HISTORY, "stream_features": [EMBEDDING]},
+        history_streams=history_streams, stream_features_cache=stream_cache,
+        user_features_cache=user_cache, features_table="features", cache_sep="--", dynamo_client=client,
+    ))
+    assert meta.success
+    assert [s["streamUrl"] for s in history_streams] == urls
+    assert all(s[EMBEDDING].data == vector_serializer.deserialize(vector).data for s in streams + history_streams)
+    assert set(user) == {"userid", HISTORY}
+    # Both a candidate and its history reference receive the one fetched object.
+    assert streams[0][EMBEDDING] is history_streams[0][EMBEDDING]
+    batches = [len(batch) for batch in client.batches]
+    assert batches == {"cold": [100, 1, 100, 100, 6], "history": [100, 100, 100, 6], "all": []}[cache_state]
+    assert client.max_active == {"cold": 3, "history": 4, "all": 0}[cache_state]
+    keys = [key for batch in client.batches for key in batch]
+    assert len(keys) == len(set(keys))  # no repeat lookups across either stage
+    assert meta.stream_cache_misses == (0 if cache_state == "all" else 306)
+    # Exercise the real Kafka encoder; vectors of history stay outside user/streams.
+    logged = orjson.loads(orjson.dumps({"user": user, "streams": streams}, default=default_serialization))
+    assert set(logged["user"]) == {"userid", HISTORY}
+    restored = history_serializer.deserialize(base64.b64decode(logged["user"][HISTORY]["proto"]))
+    assert restored == user[HISTORY]
+
+
+def test_missing_history_is_cached_only_after_a_successful_read():
+    client = FakeDynamo()
+    user_cache, stream_cache = make_features_cache(10), make_features_cache(10)
+    async def run():
+        for _ in range(2):
+            history_streams = []
+            meta = await set_all_features(
+                user={"userid": "u1"}, streams=[{"streamUrl": CANDIDATE}],
+                stream_features=[], user_features=[HISTORY],
+                history={"user_feature": HISTORY, "stream_features": [EMBEDDING]},
+                history_streams=history_streams, stream_features_cache=stream_cache,
+                user_features_cache=user_cache, features_table="features", cache_sep="--", dynamo_client=client,
+            )
+            assert meta.success and history_streams == []
+    asyncio.run(run())
+    assert client.calls == 1

{haystack_ml_stack-0.4.12 → haystack_ml_stack-0.4.13}/tests/test_serializers.py

@@ -1,6 +1,12 @@
+import math
+import struct
+
+import pytest
+
 from google.protobuf.json_format import ParseDict as ProtoParseDict
 from haystack_ml_stack import SerializerRegistry
 from haystack_ml_stack.generated.v1 import features_pb2 as features_pb2_v1
+from haystack_ml_stack.history import history_entries, history_stream_urls
 
 
 def test_v0_serializers():
@@ -211,3 +217,89 @@ def test_user_playlist_stats_serializer_backward_compatibility():
     assert result.data["sports"].total_days == 30
     assert result.data["sports"].capped_total_log_watched == 0.0
     assert result.data["sports"].first_active_date == ""
+
+
+def test_stream_embedding_serializer():
+    values = [math.sin(0.37 * (i + 1)) for i in range(512)]
+    serializer = SerializerRegistry[
+        ("STREAM", "EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512", "v1")
+    ]
+    message = serializer.deserialize(serializer.serialize({"version": 1, "data": values}))
+    vector = struct.unpack("<512e", message.data)
+    norm = math.sqrt(sum(x * x for x in values))
+    assert message.version == 1
+    assert all(abs(a - b / norm) < 1e-3 for a, b in zip(vector, values))
+
+
+def test_stream_embedding_normalizes_before_float16_storage():
+    import numpy as np
+    values = [math.sin(0.37 * (i + 1)) * (1 + i % 7) for i in range(512)]
+    first = np.asarray(values, dtype=np.float32)
+    expected = (first / np.linalg.norm(first)).astype("<f2")
+    serializer = SerializerRegistry[
+        ("STREAM", "EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512", "v1")
+    ]
+    message = serializer.deserialize(serializer.serialize({"version": 1, "data": values}))
+    assert message.data == expected.tobytes()
+    assert len(message.data) == 1024
+    rounded = first.astype(np.float16).astype(np.float32)
+    assert message.data != (rounded / np.linalg.norm(rounded)).astype("<f2").tobytes()
+
+
+@pytest.mark.parametrize("count", [511, 513])
+def test_stream_embedding_requires_512_coordinates(count):
+    serializer = SerializerRegistry[("STREAM", "EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512", "v1")]
+    with pytest.raises(AssertionError, match="512 coordinates"):
+        serializer.serialize({"version": 1, "data": [1.] * count})
+
+
+def test_user_history_serializer():
+    data = {
+        "snapshot_at_ms": 1758499200000,
+        "stream_urls": ["http://haystack.tv/id/a", "https://example.com/a,b#c",
+                        "http://haystack.tv/id/:escape", "relative/path", "http://haystack.tv/id/e"],
+        "event_at_ms": [1758400000001 + i for i in range(5)],
+        "seconds_watched": [25.7, 3600.0, 30.0, 0.0, 25.0],
+        "contexts": ["pres nxt", "null", "sel thumb", "unsupported", "__missing__"],
+        "browsed": [False, False, True, False, False],
+    }
+    serializer = SerializerRegistry[("USER", "HISTORY#5M#256", "v1")]
+    message = serializer.deserialize(
+        serializer.serialize({"version": 1, "data": data})
+    )
+    assert message.version == 1
+    assert message.data.snapshot_at_ms == data["snapshot_at_ms"]
+    assert list(message.data.stream_ids) == ["a", ":https://example.com/a,b#c",
+                                            ":http://haystack.tv/id/:escape", ":relative/path", "e"]
+    assert history_stream_urls(message) == data["stream_urls"]
+    rows = list(history_entries(message))
+    assert [r[1] for r in rows] == data["event_at_ms"]
+    assert [r[2] for r in rows] == pytest.approx(data["seconds_watched"])
+    assert [r[3] for r in rows] == ["pres nxt", "", "sel thumb", "__unsupported__", ""]
+    assert [r[4] for r in rows] == data["browsed"]
+    assert [r[5] for r in rows] == [True, True, False, False, False]
+
+
+def test_empty_history_keeps_snapshot_inside_data():
+    serializer = SerializerRegistry[("USER", "HISTORY#5M#256", "v1")]
+    data = dict(snapshot_at_ms=1758499200000, stream_urls=[], event_at_ms=[],
+                seconds_watched=[], contexts=[], browsed=[])
+    message = serializer.deserialize(serializer.serialize({"version": 1, "data": data}))
+    assert message.data.snapshot_at_ms == data["snapshot_at_ms"]
+    assert history_stream_urls(message) == []
+    assert list(history_entries(message)) == []
+
+
+@pytest.mark.parametrize("value", [0.0, float("nan"), float("inf")])
+def test_invalid_vector_norm_has_a_clear_error(value):
+    serializer = SerializerRegistry[("STREAM", "EMBEDDING#GEMINI_2_TITLE_TRANSCRIPT#512", "v1")]
+    with pytest.raises(ValueError, match="finite, nonzero"):
+        serializer.serialize({"version": 1, "data": [value] * 512})
+
+
+def test_history_decoder_rejects_misaligned_columns():
+    message = features_pb2_v1.UserHistory(version=1)
+    message.data.snapshot_at_ms = 1000
+    message.data.stream_ids.append("a")
+    with pytest.raises(ValueError, match="column lengths"):
+        list(history_entries(message))

haystack_ml_stack-0.4.12/src/haystack_ml_stack/_version.py

@@ -1 +0,0 @@
-__version__ = "0.4.12"