haystack-ml-stack 0.1.1__tar.gz → 0.2.0__tar.gz

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-ml-stack
-Version: 0.1.1
+Version: 0.2.0
 Summary: Functions related to Haystack ML
 Author-email: Oscar Vega <oscar@haystack.tv>
 License: MIT
@@ -12,6 +12,7 @@ Requires-Dist: cloudpickle==2.2.1
 Requires-Dist: aioboto3==12.0.0
 Requires-Dist: fastapi==0.104.1
 Requires-Dist: pydantic-settings==2.2
+Requires-Dist: newrelic==11.1.0
 
 # Haystack ML Stack
 

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "haystack-ml-stack"
-version = "0.1.1"
+version = "0.2.0"
 description = "Functions related to Haystack ML"
 readme = "README.md"
 authors = [{ name = "Oscar Vega", email = "oscar@haystack.tv" }]
@@ -16,6 +16,7 @@ dependencies = [
     "cloudpickle==2.2.1",
     "aioboto3==12.0.0",
     "fastapi==0.104.1",
-    "pydantic-settings==2.2"
+    "pydantic-settings==2.2",
+    "newrelic==11.1.0"
 ]
 license = { text = "MIT" }

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack/__init__.py

@@ -1,4 +1,4 @@
 from .app import create_app
 
 __all__ = ["create_app"]
-__version__ = "0.1.1"
+__version__ = "0.2.0"

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack/app.py

@@ -4,13 +4,15 @@ import random
 import sys
 from http import HTTPStatus
 from typing import Any, Dict, List, Optional
+import time
 
 import aiobotocore.session
 from fastapi import FastAPI, HTTPException, Request, Response
 from fastapi.encoders import jsonable_encoder
 
+
 from .cache import make_features_cache
-from .dynamo import set_stream_features
+from .dynamo import set_stream_features, FeatureRetrievalMeta
 from .model_store import download_and_load_model
 from .settings import Settings
 
@@ -23,6 +25,8 @@ logging.basicConfig(
 
 logger = logging.getLogger(__name__)
 
+import newrelic.agent
+
 
 def create_app(
     settings: Optional[Settings] = None,
@@ -96,11 +100,15 @@ def create_app(
 
         try:
             data = await request.json()
-        except Exception:
+        except Exception as e:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST, detail="Invalid JSON payload"
-            )
-
+            ) from e
+        query_params = {}
+        for k in request.query_params.keys():
+            values = request.query_params.getlist(k)
+            # flatten single-element lists
+            query_params[k] = values[0] if len(values) == 1 else values
         user = data.get("user", {})
         streams: List[Dict[str, Any]] = data.get("streams", [])
         playlist = data.get("playlist", {})
@@ -112,9 +120,11 @@ def create_app(
         # Feature fetch (optional based on model)
         model = state["model"]
         stream_features = model.get("stream_features", []) or []
+        retrieval_meta = FeatureRetrievalMeta(
+            cache_misses=0, retrieval_ms=0, success=True, cache_delay_minutes=0
+        )
         if stream_features:
-            logger.info("Fetching stream features for user %s", user.get("userid", ""))
-            await set_stream_features(
+            retrieval_meta = await set_stream_features(
                 aio_session=state["session"],
                 streams=streams,
                 stream_features=stream_features,
@@ -124,24 +134,52 @@ def create_app(
                 cache_sep=cfg.cache_separator,
             )
 
+        random_number = random.random()
+        userid = user.get("userid", "")
         # Sampling logs
-        if random.random() < cfg.logs_fraction:
+        if random_number < cfg.logs_fraction:
             logger.info("User %s streams: %s", user.get("userid", ""), streams)
 
         # Synchronous model execution (user code)
         try:
+            preprocess_start = time.perf_counter_ns()
             model_input = model["preprocess"](
-                user, streams, playlist, model.get("params")
+                user,
+                streams,
+                playlist,
+                {**model.get("params"), "query_params": query_params},
+            )
+            predict_start = time.perf_counter_ns()
+            model_output = model["predict"](
+                model_input, {**model.get("params"), "query_params": query_params}
             )
-            model_output = model["predict"](model_input, model.get("params"))
+            predict_end = time.perf_counter_ns()
         except Exception as e:
             logger.error("Model prediction failed: %s", e)
             raise HTTPException(
                 status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                 detail="Model prediction failed",
-            )
-
+            ) from e
+
+        newrelic.agent.record_custom_event(
+            "Inference",
+            {
+                "cache_misses": retrieval_meta.cache_misses,
+                "retrieval_success": int(retrieval_meta.success),
+                "cache_delay_minutes": retrieval_meta.cache_delay_minutes,
+                "retrieval_ms": retrieval_meta.retrieval_ms,
+                "preprocess_ms": (predict_start - preprocess_start) * 1e-6,
+                "predict_ms": (predict_end - predict_start) * 1e-6,
+                "total_scores": len(model_output),
+            },
+        )
         if model_output:
+            if random_number < cfg.logs_fraction:
+                logger.info(
+                    "User %s - model output %s",
+                    userid,
+                    model_output,
+                )
             return jsonable_encoder(model_output)
 
         raise HTTPException(

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack/dynamo.py

@@ -1,11 +1,23 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, NamedTuple
 import logging
+import time
+import datetime
 
 import aiobotocore.session
+import newrelic.agent
+
 
 logger = logging.getLogger(__name__)
 
 
+class FeatureRetrievalMeta(NamedTuple):
+    cache_misses: int
+    retrieval_ms: float
+    success: bool
+    cache_delay_minutes: float
+
+
+@newrelic.agent.function_trace()
 async def async_batch_get(
     dynamo_client, table_name: str, keys: List[Dict[str, Any]]
 ) -> List[Dict[str, Any]]:
@@ -53,6 +65,7 @@ async def async_batch_get(
     return all_items
 
 
+@newrelic.agent.function_trace()
 def parse_dynamo_item(item: Dict[str, Any]) -> Dict[str, Any]:
     """Parse a DynamoDB attribute map (low-level) to Python types."""
     out: Dict[str, Any] = {}
@@ -76,6 +89,7 @@ def parse_dynamo_item(item: Dict[str, Any]) -> Dict[str, Any]:
     return out
 
 
+@newrelic.agent.function_trace()
 async def set_stream_features(
     *,
     streams: List[Dict[str, Any]],
@@ -85,25 +99,45 @@ async def set_stream_features(
     stream_pk_prefix: str,
     cache_sep: str,
     aio_session: aiobotocore.session.Session | None = None,
-) -> None:
+) -> FeatureRetrievalMeta:
+    time_start = time.perf_counter_ns()
     """Fetch missing features for streams from DynamoDB and fill them into streams."""
     if not streams or not stream_features:
-        return
+        return FeatureRetrievalMeta(
+            cache_misses=0,
+            retrieval_ms=(time.perf_counter_ns() - time_start) * 1e-6,
+            success=True,
+            cache_delay_minutes=0,
+        )
 
     cache_miss: Dict[str, Dict[str, Any]] = {}
+    cache_delay_obj: dict[str, float] = {f: 0 for f in stream_features}
+    now = datetime.datetime.utcnow()
     for f in stream_features:
         for s in streams:
             key = f"{s['streamUrl']}{cache_sep}{f}"
-            cached = features_cache.get(key)
-            if cached is not None:
-                s[f] = cached["value"]
+            if key in features_cache:
+                # Only set if value is not None
+                cached = features_cache.get(key)
+                if cached["value"] is not None:
+                    s[f] = cached["value"]
+                    cache_delay_obj[f] = max(
+                        cache_delay_obj[f], (now - cached["updated_at"]).total_seconds()
+                    )
             else:
                 cache_miss[key] = s
+    valid_cache_delays = list(v for v in cache_delay_obj.values() if v > 0)
+    cache_delay = min(valid_cache_delays) if valid_cache_delays else 0
 
     if not cache_miss:
-        return
-
-    logger.info("Cache miss for %d items", len(cache_miss))
+        return FeatureRetrievalMeta(
+            cache_misses=0,
+            retrieval_ms=(time.perf_counter_ns() - time_start) * 1e-6,
+            success=True,
+            cache_delay_minutes=cache_delay / 60,
+        )
+    cache_misses = len(cache_miss)
+    logger.info("Cache miss for %d items", cache_misses)
 
     # Prepare keys
     keys = []
@@ -118,8 +152,14 @@ async def set_stream_features(
             items = await async_batch_get(dynamodb, features_table, keys)
         except Exception as e:
             logger.error("DynamoDB batch_get failed: %s", e)
-            return
-
+            return FeatureRetrievalMeta(
+                cache_misses=cache_misses,
+                retrieval_ms=(time.perf_counter_ns() - time_start) * 1e-6,
+                success=False,
+                cache_delay_minutes=cache_delay / 60,
+            )
+
+    updated_keys = set()
     for item in items:
         stream_url = item["pk"]["S"].removeprefix(stream_pk_prefix)
         feature_name = item["sk"]["S"]
@@ -129,6 +169,20 @@ async def set_stream_features(
         features_cache[cache_key] = {
             "value": parsed.get("value"),
             "cache_ttl_in_seconds": int(parsed.get("cache_ttl_in_seconds", -1)),
+            "updated_at": datetime.datetime.fromisoformat(parsed.get("updated_at")),
         }
         if cache_key in cache_miss:
             cache_miss[cache_key][feature_name] = parsed.get("value")
+            updated_keys.add(cache_key)
+
+    # Save keys that were not found in DynamoDB with None value
+    if len(updated_keys) < len(cache_miss):
+        missing_keys = set(cache_miss.keys()) - updated_keys
+        for k in missing_keys:
+            features_cache[k] = {"value": None, "cache_ttl_in_seconds": 300}
+    return FeatureRetrievalMeta(
+        cache_misses=cache_misses,
+        retrieval_ms=(time.perf_counter_ns() - time_start) * 1e-6,
+        success=True,
+        cache_delay_minutes=cache_delay / 60,
+    )

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack/model_store.py

@@ -4,10 +4,12 @@ from typing import Any, Dict
 
 import aiobotocore.session
 import cloudpickle
+import newrelic.agent
 
 logger = logging.getLogger(__name__)
 
 
+@newrelic.agent.function_trace()
 async def download_and_load_model(
     s3_url: str, aio_session: aiobotocore.session.Session | None = None
 ) -> Dict[str, Any]:
@@ -33,4 +35,4 @@ async def download_and_load_model(
 
     with open(local_path, "rb") as f:
         model: Dict[str, Any] = cloudpickle.load(f)
-    return model
+    return model

haystack_ml_stack-0.2.0/src/haystack_ml_stack/utils.py

@@ -0,0 +1,276 @@
+import pandas as pd
+import numpy as np
+import typing as _t
+
+
+def stream_tags_cleanup(
+    stream, user_favorite_tags: list[str], user_favorite_authors: list[str]
+) -> dict:
+    stream_tags = stream.get("haystackTags", [])
+    is_favorite_tag = (
+        any(stream_tag in user_favorite_tags for stream_tag in stream_tags)
+        if user_favorite_tags is not None
+        else False
+    )
+    is_favorite_author = (
+        stream.get("author", None) in user_favorite_authors
+        if user_favorite_authors is not None
+        else False
+    )
+    return {
+        "IS_FAVORITE_TAG": is_favorite_tag,
+        "IS_FAVORITE_AUTHOR": is_favorite_author,
+    }
+
+
+def browsed_count_cleanups(
+    stream,
+    position_debiasing: _t.Literal["4_browsed", "all_browsed"] = "4_browsed",
+) -> dict:
+    position_alias_mapping = {
+        "0": "1ST_POS",
+        "1": "2ND_POS",
+        "2": "3RD_POS",
+        "3+": "REST_POS",
+    }
+    if position_debiasing == "4_browsed":
+        suffix = "_UP_TO_4_BROWSED"
+    elif position_debiasing == "all_browsed":
+        suffix = ""
+    else:
+        raise ValueError(f"Unexpected position debiasing '{position_debiasing}'.")
+    browsed_count_obj = stream.get("PSELECT#24H", {}).get(position_debiasing, {})
+    total_selects = 0
+    total_browsed = 0
+    total_selects_and_watched = 0
+    feats = {}
+    for position in position_alias_mapping.keys():
+        pos_counts = browsed_count_obj.get(position, {})
+        total_browsed += pos_counts.get("total_browsed", 0)
+        total_selects += pos_counts.get("total_selects", 0)
+        total_selects_and_watched += pos_counts.get("total_selects_and_watched", 0)
+    if position_debiasing == "4_browsed":
+        suffix = "_UP_TO_4_BROWSED"
+    elif position_debiasing == "all_browsed":
+        suffix = ""
+    else:
+        raise ValueError("Should not be here.")
+    feats[f"STREAM_24H_TOTAL_BROWSED{suffix}"] = total_browsed
+    feats[f"STREAM_24H_TOTAL_SELECTS{suffix}"] = total_selects
+    feats[f"STREAM_24H_TOTAL_SELECTS_AND_WATCHED{suffix}"] = total_selects_and_watched
+    return feats
+
+
+def device_split_browsed_count_cleanups(
+    stream,
+    device_type: _t.Literal["TV", "MOBILE"],
+    position_debiasing: _t.Literal["4_browsed", "all_browsed"] = "4_browsed",
+) -> dict:
+    position_alias_mapping = {
+        "0": "1ST_POS",
+        "1": "2ND_POS",
+        "2": "3RD_POS",
+        "3+": "REST_POS",
+    }
+    if position_debiasing == "4_browsed":
+        suffix = "_UP_TO_4_BROWSED"
+    elif position_debiasing == "all_browsed":
+        suffix = ""
+    else:
+        raise ValueError(f"Unexpected position debiasing '{position_debiasing}'.")
+
+    _validate_device_type(device_type)
+
+    browsed_count_obj = stream.get(f"PSELECT#24H#{device_type}", {}).get(
+        position_debiasing, {}
+    )
+    total_selects = 0
+    total_browsed = 0
+    total_selects_and_watched = 0
+    feats = {}
+    for position, alias in position_alias_mapping.items():
+        pos_counts = browsed_count_obj.get(position, {})
+        total_browsed = pos_counts.get("total_browsed", 0)
+        total_selects = pos_counts.get("total_selects", 0)
+        total_selects_and_watched = pos_counts.get("total_selects_and_watched", 0)
+        feats[f"STREAM_{alias}_{device_type}_24H_TOTAL_BROWSED{suffix}"] = total_browsed
+        feats[f"STREAM_{alias}_{device_type}_24H_TOTAL_SELECTS{suffix}"] = total_selects
+        feats[f"STREAM_{alias}_{device_type}_24H_TOTAL_SELECTS_AND_WATCHED{suffix}"] = (
+            total_selects_and_watched
+        )
+    return feats
+
+
+def watched_count_cleanups(stream, entry_contexts: list[str] = None) -> dict:
+    if entry_contexts is None:
+        entry_contexts = [
+            "autoplay",
+            "choose next",
+            "ch swtch",
+            "sel thumb",
+            "launch first in session",
+        ]
+    _validate_pwatched_entry_context(entry_contexts)
+
+    counts_obj = stream.get(f"PWATCHED#24H", {})
+    feats = {}
+    for entry_context in entry_contexts:
+        attempts = counts_obj.get(entry_context, {}).get("attempts", 0)
+        watched = counts_obj.get(entry_context, {}).get("watched", 0)
+        context_key = entry_context if "launch" not in entry_context else "launch"
+        context_key = context_key.upper().replace(" ", "_")
+        feats[f"STREAM_{context_key}_24H_TOTAL_WATCHED"] = watched
+        feats[f"STREAM_{context_key}_24H_TOTAL_ATTEMPTS"] = attempts
+    return feats
+
+
+def device_watched_count_cleanups(
+    stream, device_type: str, entry_contexts: list[str] = None
+) -> dict:
+    if entry_contexts is None:
+        entry_contexts = [
+            "autoplay",
+            "choose next",
+            "ch swtch",
+            "sel thumb",
+            "launch first in session",
+        ]
+
+    _validate_pwatched_entry_context(entry_contexts)
+    _validate_device_type(device_type)
+
+    counts_obj = stream.get(f"PWATCHED#24H#{device_type}", {})
+    feats = {}
+    for entry_context in entry_contexts:
+        attempts = counts_obj.get(entry_context, {}).get("attempts", 0)
+        watched = counts_obj.get(entry_context, {}).get("watched", 0)
+        context_key = entry_context if "launch" not in entry_context else "launch"
+        context_key = context_key.upper().replace(" ", "_")
+        feats["features"][f"STREAM_{context_key}_{device_type}_24H_TOTAL_WATCHED"] = (
+            watched
+        )
+        feats["features"][f"STREAM_{context_key}_{device_type}_24H_TOTAL_ATTEMPTS"] = (
+            attempts
+        )
+    return feats
+
+
+def generic_beta_adjust_features(
+    data: pd.DataFrame,
+    prefix: str,
+    pwatched_beta_params: dict,
+    pselect_beta_params: dict,
+    pslw_beta_params: dict,
+    use_low_sample_flags: bool = False,
+    low_sample_threshold: int = 3,
+    use_attempt_features: bool = False,
+    max_attempt_cap: int = 100,
+    debiased_pselect: bool = True,
+    use_logodds: bool = False,
+) -> pd.DataFrame:
+    pwatched_features = {}
+    for context, (alpha, beta) in pwatched_beta_params.items():
+        total_watched = data[f"{prefix}_{context}_TOTAL_WATCHED"].fillna(0)
+        total_attempts = data[f"{prefix}_{context}_TOTAL_ATTEMPTS"].fillna(0)
+        pwatched_features[f"{prefix}_{context}_ADJ_PWATCHED"] = (
+            total_watched + alpha
+        ) / (total_attempts + alpha + beta)
+        if use_low_sample_flags:
+            pwatched_features[f"{prefix}_{context}_LOW_SAMPLE"] = total_attempts.le(
+                low_sample_threshold
+            ).astype(int)
+        if use_attempt_features:
+            pwatched_features[f"{prefix}_{context}_ATTEMPTS"] = total_attempts.clip(
+                upper=max_attempt_cap
+            )
+
+    pselect_features = {}
+    debias_suffix = "_UP_TO_4_BROWSED" if debiased_pselect else ""
+    for key, (alpha, beta) in pselect_beta_params.items():
+        total_selects = data[f"{prefix}_{key}_TOTAL_SELECTS{debias_suffix}"].fillna(0)
+        total_browsed = data[f"{prefix}_{key}_TOTAL_BROWSED{debias_suffix}"].fillna(0)
+        pselect_features[f"{prefix}_{key}_ADJ_PSELECT{debias_suffix}"] = (
+            total_selects + alpha
+        ) / (total_selects + total_browsed + alpha + beta)
+        if use_low_sample_flags:
+            pselect_features[f"{prefix}_{key}_PSELECT_LOW_SAMPLE{debias_suffix}"] = (
+                (total_selects + total_browsed).le(low_sample_threshold).astype(int)
+            )
+        if use_attempt_features:
+            pselect_features[f"{prefix}_{key}_PSELECT_ATTEMPTS{debias_suffix}"] = (
+                total_selects + total_browsed
+            ).clip(upper=max_attempt_cap)
+        total_slw = data[
+            f"{prefix}_{key}_TOTAL_SELECTS_AND_WATCHED{debias_suffix}"
+        ].fillna(0)
+        pslw_alpha, pslw_beta = pslw_beta_params[key]
+        pselect_features[f"{prefix}_{key}_ADJ_PSLW{debias_suffix}"] = (
+            total_slw + pslw_alpha
+        ) / (total_selects + total_browsed + pslw_alpha + pslw_beta)
+        pselect_features[f"{prefix}_{key}_PSelNotW{debias_suffix}"] = (
+            pselect_features[f"{prefix}_{key}_ADJ_PSELECT{debias_suffix}"]
+            - pselect_features[f"{prefix}_{key}_ADJ_PSLW{debias_suffix}"]
+        )
+
+    adjusted_feats = pd.DataFrame({**pwatched_features, **pselect_features})
+    if use_logodds:
+        adjusted_feats = adjusted_feats.pipe(
+            lambda x: x.assign(
+                **x[
+                    [
+                        c
+                        for c in x.columns
+                        if "PSELECT" in c
+                        or "PSLW" in c
+                        or "PWATCHED" in c
+                        or "PSelNotW" in c
+                    ]
+                ]
+                .clip(lower=0.001)
+                .pipe(prob_to_logodds)
+            )
+        )
+    return adjusted_feats
+
+
+def prob_to_logodds(prob: float) -> float:
+    return np.log(prob) - np.log(1 - prob)
+
+
+def scale_preds(
+    preds: pd.Series,
+    original_mean: float,
+    original_std: float,
+    target_mean: float,
+    target_std: float,
+) -> pd.Series:
+    z_score = (preds - original_mean) / original_std
+    return z_score * target_std + target_mean
+
+
+def sigmoid(x: float) -> float:
+    return 1 / (1 + np.exp(-x))
+
+
+def generic_logistic_predict(
+    data: pd.DataFrame, coeffs: pd.Series, intercept: float
+) -> pd.Series:
+    return ((data[coeffs.index] * coeffs).sum(axis=1) + intercept).pipe(sigmoid)
+
+
+def _validate_device_type(device_type: str):
+    if device_type not in ("TV", "MOBILE"):
+        raise ValueError(f"Invalid device type '{device_type}")
+
+
+def _validate_pwatched_entry_context(entry_contexts: list[str]):
+    valid_contexts = [
+        "autoplay",
+        "choose next",
+        "ch swtch",
+        "sel thumb",
+        "launch first in session",
+    ]
+    invalid_contexts = [c for c in entry_contexts if c not in valid_contexts]
+    if invalid_contexts:
+        raise ValueError(f"Invalid entry contexts found: {invalid_contexts}")

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-ml-stack
-Version: 0.1.1
+Version: 0.2.0
 Summary: Functions related to Haystack ML
 Author-email: Oscar Vega <oscar@haystack.tv>
 License: MIT
@@ -12,6 +12,7 @@ Requires-Dist: cloudpickle==2.2.1
 Requires-Dist: aioboto3==12.0.0
 Requires-Dist: fastapi==0.104.1
 Requires-Dist: pydantic-settings==2.2
+Requires-Dist: newrelic==11.1.0
 
 # Haystack ML Stack
 

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ src/haystack_ml_stack/cache.py
 src/haystack_ml_stack/dynamo.py
 src/haystack_ml_stack/model_store.py
 src/haystack_ml_stack/settings.py
+src/haystack_ml_stack/utils.py
 src/haystack_ml_stack.egg-info/PKG-INFO
 src/haystack_ml_stack.egg-info/SOURCES.txt
 src/haystack_ml_stack.egg-info/dependency_links.txt

{haystack_ml_stack-0.1.1 → haystack_ml_stack-0.2.0}/src/haystack_ml_stack.egg-info/requires.txt

@@ -4,3 +4,4 @@ cloudpickle==2.2.1
 aioboto3==12.0.0
 fastapi==0.104.1
 pydantic-settings==2.2
+newrelic==11.1.0