datarobot-moderations 11.1.22__py3-none-any.whl → 11.2.1__py3-none-any.whl

datarobot_dome/drum_integration.py

@@ -22,6 +22,7 @@ from typing import Optional
 
 import numpy as np
 import pandas as pd
+import yaml
 from openai.types.chat import ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletion
 from openai.types.chat.chat_completion import Choice
@@ -257,11 +258,6 @@ def run_user_score_function(filtered_df, model, pipeline, drum_score_fn, **kwarg
     return valid_predictions_df, none_predictions_df, score_latency
 
 
-def _auto_generate_association_ids(num_rows):
-    _logger.info(f"Auto generating {num_rows} association ids")
-    return [str(uuid.uuid4()) for _ in range(num_rows)]
-
-
 def guard_score_wrapper(data, model, pipeline, drum_score_fn, **kwargs):
     """
     Score wrapper function provided by the moderation library.  DRUM will invoke this
@@ -291,7 +287,7 @@ def guard_score_wrapper(data, model, pipeline, drum_score_fn, **kwargs):
         and association_id_column_name not in data.columns
         and pipeline.auto_generate_association_ids
     ):
-        data[association_id_column_name] = _auto_generate_association_ids(data.shape[0])
+        data[association_id_column_name] = pipeline.generate_association_ids(data.shape[0])
 
     # ==================================================================
     # Step 1: Prescore Guards processing
@@ -721,7 +717,7 @@ def guard_chat_wrapper(
         if association_id:
             data[association_id_column_name] = [association_id]
         elif pipeline.auto_generate_association_ids:
-            data[association_id_column_name] = _auto_generate_association_ids(1)
+            data[association_id_column_name] = pipeline.generate_association_ids(1)
             association_id = data[association_id_column_name].tolist()[0]
 
     # ==================================================================
@@ -849,63 +845,18 @@ def guard_chat_wrapper(
     )
 
 
-def run_vdb_score_function(
-    input_df: pd.DataFrame, model, pipeline, drum_score_fn, **kwargs
-) -> pd.DataFrame:
-    """
-    A wrapper to execute vdb's `score` method.  Wrapper is useful to calculate the
-    latency of the `score` method and handle any exceptional conditions
-    Returns:
-        predictions_df: DataFrame obtained as a return value from user's `score`
-            method
-    """
-    start_time = time.time()
-
-    try:
-        predictions_df = drum_score_fn(input_df, model, **kwargs)
-    except Exception as e:
-        title = "Failed to execute vdb score function"
-        message = f"Exception: {e}"
-        _logger.error(title + " " + message)
-        pd.set_option("display.max_columns", None)
-        _logger.error(input_df)
-        pipeline.send_event_sync(
-            title, message, ModerationEventTypes.MODERATION_MODEL_SCORING_ERROR
-        )
-        raise
-
-    score_latency = time.time() - start_time
-    pipeline.record_score_latency(score_latency)
-    return predictions_df
-
-
-def vdb_score_wrapper(data: pd.DataFrame, model, pipeline: VDBPipeline, drum_score_fn, **kwargs):
-    """
-    Run on each prediction, and takes care of running the "score" function as well
-    as collecting the metrics for the VDB pipeline.
-    """
-    _logger.debug(data)
-
-    # clear/allocate memory for reporting metrics
-    pipeline.get_new_metrics_payload()
-
-    # NOTE: no "pre-score" calculation on the DataFrame for the predictions
-
-    # perform the main "score" function for this model
-    predictions_df = run_vdb_score_function(data, model, pipeline, drum_score_fn, **kwargs)
-
-    # loop through all the metrics scoring for the pipeline with predictions_df that has citations
-    for scorer in pipeline.scorers():
-        value = scorer.score(predictions_df)
-        pipeline.record_aggregate_value(scorer.name, value)
-
-    pipeline.report_custom_metrics()
-    return predictions_df
+def vdb_init(model_dir: str = os.getcwd()):
+    """Initializes a VDB pipeline."""
+    config = {}
 
+    config_file = os.path.join(model_dir, MODERATION_CONFIG_FILE_NAME)
+    if not os.path.exists(config_file):
+        _logger.info(f"No config file ({config_file}) found")
+    else:
+        with open(config_file) as fp:
+            config = yaml.safe_load(fp)
 
-def vdb_init():
-    """Initializes a VDB pipeline."""
-    return VDBPipeline()
+    return VDBPipeline(config)
 
 
 def init(model_dir: str = os.getcwd()):
@@ -992,7 +943,7 @@ class VdbModerationPipeline(ModerationPipeline):
 
     def score(self, data: pd.DataFrame, model, drum_score_fn, **kwargs):
         """Calls the VDB score function."""
-        return vdb_score_wrapper(data, model, self._pipeline, drum_score_fn, **kwargs)
+        return self._pipeline.score(data, model, drum_score_fn, **kwargs)
 
 
 def moderation_pipeline_factory(
@@ -1007,7 +958,7 @@ def moderation_pipeline_factory(
             return LlmModerationPipeline(pipeline)
 
     if target_type in TargetType.vdb():
-        pipeline = vdb_init()
+        pipeline = vdb_init(model_dir=model_dir)
         if pipeline:
             return VdbModerationPipeline(pipeline)
 

datarobot_dome/metrics/citation_metrics.py

@@ -29,6 +29,13 @@ class CitationTokenCountScorer(MetricScorer):
     BASELINE_VALUE = 0
     INPUT_COLUMN = CITATION_COLUMN
 
+    def score_rows(self, df: pd.DataFrame) -> list[float]:
+        column = self.input_column
+        if column not in df.columns:
+            return []
+
+        return [sum(get_token_count(v, self.encoding) for v in cell) for cell in df[column].values]
+
     def score(self, df: pd.DataFrame) -> float:
         column = self.input_column
         if column not in df.columns:
@@ -48,6 +55,19 @@ class CitationTokenAverageScorer(MetricScorer):
     BASELINE_VALUE = 0
     INPUT_COLUMN = CITATION_COLUMN
 
+    def score_rows(self, df: pd.DataFrame) -> []:
+        column = self.input_column
+        if column not in df.columns:
+            return []
+
+        averages = []
+        for cell in df[column].values:
+            total = sum(get_token_count(v, self.encoding) for v in cell)
+            count = sum(v != "" for v in cell)
+            averages.append(total / count)
+
+        return averages
+
     def score(self, df: pd.DataFrame) -> float:
         average = 0.0
         total = 0
@@ -73,6 +93,13 @@ class DocumentCountScorer(MetricScorer):
     BASELINE_VALUE = 0
     INPUT_COLUMN = CITATION_COLUMN
 
+    def score_rows(self, df: pd.DataFrame) -> list[float]:
+        column = self.input_column
+        if column not in df.columns:
+            return []
+
+        return [sum(bool(v) for v in cell) for cell in df[column].values]
+
     def score(self, df: pd.DataFrame) -> float:
         column = self.input_column
         if column not in df.columns:
@@ -90,6 +117,13 @@ class DocumentAverageScorer(MetricScorer):
     BASELINE_VALUE = 0
     INPUT_COLUMN = CITATION_COLUMN
 
+    def score_rows(self, df: pd.DataFrame) -> list[float]:
+        column = self.input_column
+        if column not in df.columns:
+            return []
+
+        return [sum(bool(v) for v in cell) for cell in df[column].values]
+
     def score(self, df: pd.DataFrame) -> float:
         column = self.input_column
         if column not in df.columns:

datarobot_dome/metrics/metric_scorer.py

@@ -61,10 +61,6 @@ class MetricScorer(ABC):
     def name(self) -> str:
         return self.config.get("name", self.NAME)
 
-    @property
-    def per_prediction(self) -> bool:
-        return self.config.get("per_prediction", False)
-
     @property
     def input_column(self) -> str:
         return self.config.get("input_column", self.INPUT_COLUMN)
@@ -75,4 +71,8 @@ class MetricScorer(ABC):
 
     @abstractmethod
     def score(self, df: pd.DataFrame) -> float:
-        pass
+        pass  # pragma: no cover
+
+    @abstractmethod
+    def score_rows(self, df: pd.DataFrame) -> list[float]:
+        pass  # pragma: no cover

datarobot_dome/pipeline/llm_pipeline.py

@@ -12,10 +12,7 @@
 import logging
 import math
 import os
-from datetime import datetime
-from datetime import timezone
 
-import numpy as np
 import yaml
 from datarobot.enums import CustomMetricAggregationType
 from datarobot.enums import CustomMetricDirectionality
@@ -199,44 +196,37 @@ class LLMPipeline(Pipeline):
         guard.set_pipeline(self)
 
         if guard.has_average_score_custom_metric():
-            self.custom_metric_map[guard.get_average_score_custom_metric_name(guard_stage)] = {
-                "metric_definition": self._get_average_score_metric_definition(guard)
-            }
+            metric_def = self._get_average_score_metric_definition(guard)
+            self.add_custom_metric(metric_def, True)
 
         if guard.has_latency_custom_metric():
-            self.custom_metric_map[guard.get_latency_custom_metric_name()] = {
-                "metric_definition": guard.get_latency_custom_metric()
-            }
+            metric_def = guard.get_latency_custom_metric()
+            self.add_custom_metric(metric_def, False)
 
         if intervention_action:
             # Enforced metric for all kinds of guards, as long as they have intervention
             # action defined - even for token count
-            self.custom_metric_map[
-                guard.get_guard_enforced_custom_metric_name(guard_stage, intervention_action)
-            ] = {
-                "metric_definition": guard.get_enforced_custom_metric(
-                    guard_stage, intervention_action
-                )
-            }
-        self.custom_metrics_no_association_ids.append(guard.get_latency_custom_metric_name())
+            metric_def = guard.get_enforced_custom_metric(guard_stage, intervention_action)
+            self.add_custom_metric(metric_def, True)
 
     def _add_default_custom_metrics(self):
         """Default custom metrics"""
-        metric_list = [
+        # These metrics do not need association id for reporting
+        for metric_def in [
             get_total_custom_metric(GuardStage.PROMPT),
             get_total_custom_metric(GuardStage.RESPONSE),
             prescore_guard_latency_custom_metric,
             postscore_guard_latency_custom_metric,
             score_latency,
-        ]
-        # Metric list so far does not need association id for reporting
-        for metric in metric_list:
-            self.custom_metrics_no_association_ids.append(metric["name"])
+        ]:
+            self.add_custom_metric(metric_def, False)
 
-        metric_list.append(get_blocked_custom_metric(GuardStage.PROMPT))
-        metric_list.append(get_blocked_custom_metric(GuardStage.RESPONSE))
-        for metric in metric_list:
-            self.custom_metric_map[metric["name"]] = {"metric_definition": metric}
+        # These metrics report with an association-id
+        for metric_def in [
+            get_blocked_custom_metric(GuardStage.PROMPT),
+            get_blocked_custom_metric(GuardStage.RESPONSE),
+        ]:
+            self.add_custom_metric(metric_def, True)
 
     def _add_guard_to_pipeline(self, guard):
         if guard.stage == GuardStage.PROMPT:
@@ -299,23 +289,6 @@ class LLMPipeline(Pipeline):
                 else (os.environ.get("TARGET_NAME").replace('"', ""))
             )
 
-    def _set_custom_metrics_individual_entry(self, metric_id, value, association_id):
-        if isinstance(value, bool):
-            _value = 1.0 if value else 0.0
-        elif isinstance(value, np.bool_):
-            _value = 1.0 if value.item() else 0.0
-        elif isinstance(value, np.generic):
-            _value = value.item()
-        else:
-            _value = value
-        return {
-            "customMetricId": str(metric_id),
-            "value": _value,
-            "associationId": str(association_id),
-            "sampleSize": 1,
-            "timestamp": str(datetime.now(timezone.utc).isoformat()),
-        }
-
     def get_enforced_column_name(self, guard, stage):
         input_column = self.get_input_column(stage)
         intervention_action = guard.get_intervention_action()
@@ -366,14 +339,14 @@ class LLMPipeline(Pipeline):
                         f"Not reporting the value with association id {association_id}"
                     )
                     continue
-                custom_metric_id = self.custom_metric_map[metric_name].get("id")
+                custom_metric_id = self.custom_metric_id_from_name(metric_name)
                 if custom_metric_id is None:
                     self._logger.warning(f"No metric id for '{metric_name}', not reporting")
                     continue
-                bucket = self._set_custom_metrics_individual_entry(
+                item = self.custom_metric_individual_payload(
                     custom_metric_id, row[column_name], association_id
                 )
-                buckets.append(bucket)
+                buckets.append(item)
         return buckets
 
     def _get_blocked_column_name_from_result_df(self, stage):
@@ -393,11 +366,11 @@ class LLMPipeline(Pipeline):
                 if math.isnan(row[blocked_column_name]):
                     # If prompt is blocked, response will be NaN, so don't report it
                     continue
-                custom_metric_id = self.custom_metric_map[blocked_metric_name].get("id")
+                custom_metric_id = self.custom_metric_id_from_name(blocked_metric_name)
                 if custom_metric_id is None:
                     self._logger.warning(f"No metric id for '{blocked_metric_name}', not reporting")
                     continue
-                bucket = self._set_custom_metrics_individual_entry(
+                bucket = self.custom_metric_individual_payload(
                     custom_metric_id, row[blocked_column_name], association_id
                 )
                 payload["buckets"].append(bucket)

datarobot_dome/pipeline/pipeline.py

@@ -14,8 +14,11 @@ import logging
 import math
 import os
 import traceback
+import uuid
 from datetime import datetime
 from datetime import timezone
+from typing import Any
+from typing import Optional
 
 import datarobot as dr
 import numpy as np
@@ -48,11 +51,8 @@ class Pipeline:
         self._model_id = None
         self.async_http_client = None
         self._custom_metrics_bulk_upload_url = None
-        self._assoc_id_specific_custom_metric_ids = list()
         self.aggregate_custom_metric = None
         self.custom_metric_map = dict()
-        # List of custom metrics names which do not need the association id while reporting
-        self.custom_metrics_no_association_ids = list()
         self.delayed_custom_metric_creation = False
         self.upload_custom_metrics_tasks = set()
 
@@ -170,12 +170,27 @@ class Pipeline:
             self.create_custom_metrics()
             self.delayed_custom_metric_creation = False
 
+    def add_custom_metric(
+        self, metric_definition: dict[str, Any], requires_association_id: bool, **kwargs
+    ) -> None:
+        """
+        Adds an entry to the `custom_metric_map`.
+
+        NOTE: the kwargs allow implementations to add their own specialized values.
+        """
+        name = metric_definition["name"]
+        self.custom_metric_map[name] = {
+            "metric_definition": metric_definition,
+            "requires_association_id": requires_association_id,
+            **kwargs,
+        }
+
     def create_custom_metrics(self):
         """
         Creates all the custom-metrics in the DR app for an active deployment.
 
-        The `custom_metric_map` and `_requires_association_id` attributes are consulted to
-        insure the appropriate data is put in place for reporting.
+        Updates the `custom_metric_map` with id's to insure the appropriate data
+        is put in place for reporting.
         """
         cleanup_metrics_list = list()
         for index, (metric_name, custom_metric) in enumerate(self.custom_metric_map.items()):
@@ -196,14 +211,11 @@ class Pipeline:
                     is_model_specific=metric_definition["isModelSpecific"],
                 )
                 custom_metric["id"] = _metric_obj.id
-                custom_metric["requires_association_id"] = self._requires_association_id(
-                    metric_name
-                )
             except ClientError as e:
                 if e.status_code == 409:
                     if "not unique for deployment" in e.json["message"]:
                         # Duplicate entry nothing to worry - just continue
-                        self._logger.error(f"Metric '{metric_name}' already exists, skipping")
+                        self._logger.warning(f"Metric '{metric_name}' already exists, skipping")
                         continue
                     elif e.json["message"].startswith("Maximum number of custom metrics reached"):
                         # Reached the limit - we can't create more
@@ -261,9 +273,6 @@ class Pipeline:
                 self._logger.error(f"Metric '{metric_name}' exists at DR but not in moderation")
                 continue
             self.custom_metric_map[metric_name]["id"] = metric["id"]
-            self.custom_metric_map[metric_name]["requires_association_id"] = (
-                self._requires_association_id(metric_name)
-            )
 
         # These are the metrics we couldn't create - so, don't track them
         for metric_name in cleanup_metrics_list:
@@ -271,12 +280,33 @@ class Pipeline:
                 self._logger.error(f"Skipping metric creation: {metric_name}")
                 del self.custom_metric_map[metric_name]
 
-    def _requires_association_id(self, metric_name):
-        return metric_name not in self.custom_metrics_no_association_ids
+    def custom_metric_id_from_name(self, name: str) -> Optional[str]:
+        """Gets the custom-metric id from the name of a custom metric."""
+        identifier = self.custom_metric_map.get(name, {}).get("id")
+        return str(identifier) if identifier else None
 
-    @property
-    def prediction_url(self):
-        return self._datarobot_url
+    def custom_metric_individual_payload(
+        self, metric_id: Any, value: Any, association_id: Any
+    ) -> dict[str, Any]:
+        """
+        Creates a dictionary for an individual custom-metric value, suitable to report
+        in the bulk upload (when surrounded by other stuff).
+        """
+        if isinstance(value, bool):
+            _value = 1.0 if value else 0.0
+        elif isinstance(value, np.bool_):
+            _value = 1.0 if value.item() else 0.0
+        elif isinstance(value, np.generic):
+            _value = value.item()
+        else:
+            _value = value
+        return {
+            "customMetricId": str(metric_id),
+            "value": _value,
+            "associationId": str(association_id),
+            "sampleSize": 1,
+            "timestamp": str(datetime.now(timezone.utc).isoformat()),
+        }
 
     @property
     def api_token(self):
@@ -285,6 +315,10 @@ class Pipeline:
     def get_association_id_column_name(self):
         return self._association_id_column_name
 
+    def generate_association_ids(self, num_rows: int) -> list[str]:
+        self._logger.info(f"Generating {num_rows} association ids")
+        return [str(uuid.uuid4()) for _ in range(num_rows)]
+
     def get_new_metrics_payload(self):
         """
         Resets the data for aggregate metrics reporting based on the `custom_metric_map`.

datarobot_dome/pipeline/vdb_pipeline.py

@@ -10,19 +10,24 @@
 #  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
 #  ---------------------------------------------------------------------------------
 import logging
+import time
 from typing import Any
+from typing import Optional
 
+import pandas as pd
 from datarobot.enums import CustomMetricAggregationType
 from datarobot.enums import CustomMetricDirectionality
 
 from datarobot_dome.constants import CUSTOM_METRIC_DESCRIPTION_SUFFIX
 from datarobot_dome.constants import LOGGER_NAME_PREFIX
+from datarobot_dome.constants import ModerationEventTypes
 from datarobot_dome.metrics.factory import MetricScorerFactory
 from datarobot_dome.metrics.metric_scorer import MetricScorer
 from datarobot_dome.metrics.metric_scorer import ScorerType
 from datarobot_dome.pipeline.pipeline import Pipeline
 
 LATENCY_NAME = "VDB Score Latency"
+DEFAULT_PER_PREDICTION = True
 
 score_latency = {
     "name": LATENCY_NAME,
@@ -37,36 +42,35 @@ score_latency = {
 
 
 class VDBPipeline(Pipeline):
-    def __init__(self):
+    def __init__(self, config: Optional[dict[str, Any]] = None):
         super().__init__()
+        metric_config = config.get("metrics", {}) if config else {}
         self._score_configs: dict[ScorerType, dict[str, Any]] = {
-            ScorerType.CITATION_TOKEN_AVERAGE: {},
-            ScorerType.CITATION_TOKEN_COUNT: {},
-            ScorerType.DOCUMENT_AVERAGE: {},
-            ScorerType.DOCUMENT_COUNT: {},
+            stype.value: metric_config.get(stype.lower().replace("_", "-"), {})
+            for stype in ScorerType
         }
         self._scorers: list[MetricScorer] = list()
         self._logger = logging.getLogger(LOGGER_NAME_PREFIX + "." + self.__class__.__name__)
         self._add_default_custom_metrics()
         self.create_custom_metrics_if_any()
         self.create_scorers()
+        self.update_custom_metric_association_ids()
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({len(self.custom_metrics)} metrics)"
 
     def _add_default_custom_metrics(self):
         """Adds the default custom metrics based on the `_score_configs` map."""
         # create a list of tuples, so we can track the scorer type
-        metric_list = [(score_latency, None)]
+        metric_list = [(score_latency, False, None)]
         for score_type, score_config in self._score_configs.items():
-            metric_config = MetricScorerFactory.custom_metric_config(score_type, score_config)
-            metric_list.append((metric_config, score_type))
+            metric_def = MetricScorerFactory.custom_metric_config(score_type, score_config)
+            per_row = score_config.get("per-prediction", DEFAULT_PER_PREDICTION)
+            metric_list.append((metric_def, per_row, score_type))
 
         # Metric list so far does not need association id for reporting
-        for metric_config, score_type in metric_list:
-            name = metric_config["name"]
-            self.custom_metrics_no_association_ids.append(name)
-            self.custom_metric_map[name] = {
-                "metric_definition": metric_config,
-                "scorer_type": score_type,
-            }
+        for metric_def, per_row, score_type in metric_list:
+            self.add_custom_metric(metric_def, per_row, scorer_type=score_type)
 
     def create_scorers(self):
         """
@@ -90,6 +94,18 @@ class VDBPipeline(Pipeline):
             scorer = MetricScorerFactory.create(score_type, score_config)
             self._scorers.append(scorer)
 
+    def update_custom_metric_association_ids(self):
+        """Update whether tracking per-prediction metrics based on deployment settings."""
+        has_assoc = bool(self._association_id_column_name)
+        for metric_name, metric_data in self.custom_metric_map.items():
+            score_type = metric_data.get("scorer_type")
+            if not score_type:
+                continue
+
+            scorer_config = self._score_configs.get(score_type, {})
+            per_assoc = scorer_config.get("per-prediction", DEFAULT_PER_PREDICTION)
+            metric_data["requires_association_id"] = has_assoc and per_assoc
+
     def scorers(self) -> list[MetricScorer]:
         """Get all scorers for this pipeline."""
         return self._scorers
@@ -109,11 +125,12 @@ class VDBPipeline(Pipeline):
         """Records aggregate latency metric value locally"""
         self.record_aggregate_value(LATENCY_NAME, latency_in_sec)
 
-    def report_custom_metrics(self):
+    def report_custom_metrics(self, individual_metrics: list[dict[str, Any]]) -> None:
         """
         Reports all the custom-metrics to DR app.
 
-        The bulk upload includes grabbing all the aggregated metrics.
+        The bulk upload includes grabbing all the aggregated metrics, plus the list of
+        individual metric payloads.
         """
         if self.delayed_custom_metric_creation:
             # Flag is not set yet, so no point reporting custom metrics
@@ -123,5 +140,96 @@ class VDBPipeline(Pipeline):
             # in "test" mode, there is not a deployment and therefore no custom_metrics
             return
 
-        payload = self.add_aggregate_metrics_to_payload({"buckets": []})
+        payload = self.add_aggregate_metrics_to_payload({"buckets": individual_metrics})
         self.upload_custom_metrics(payload)
+
+    def run_model_score(
+        self, input_df: pd.DataFrame, model, drum_score_fn, **kwargs
+    ) -> pd.DataFrame:
+        """
+        A wrapper to execute vdb's `score` method.  Wrapper is useful to calculate the
+        latency of the `score` method and handle any exceptional conditions
+        Returns:
+            predictions_df: DataFrame obtained as a return value from user's `score`
+                method
+        """
+        start_time = time.time()
+
+        try:
+            predictions_df = drum_score_fn(input_df, model, **kwargs)
+        except Exception as e:
+            title = "Failed to execute vdb score function"
+            message = f"Exception: {e}"
+            self._logger.error(title + " " + message)
+            pd.set_option("display.max_columns", None)
+            self._logger.error(input_df)
+            self.send_event_sync(
+                title, message, ModerationEventTypes.MODERATION_MODEL_SCORING_ERROR
+            )
+            raise
+
+        score_latency = time.time() - start_time
+        self.record_score_latency(score_latency)
+        return predictions_df
+
+    def score(self, data: pd.DataFrame, model, drum_score_fn, **kwargs):
+        """
+        Run on each prediction, and takes care of running the "score" function as well
+        as collecting the metrics.
+        """
+        self._logger.debug(data)
+
+        # clear/allocate memory for reporting metrics
+        self.get_new_metrics_payload()
+
+        # add the association-id if not present
+        association_id_column_name = self.get_association_id_column_name()
+        if (
+            association_id_column_name
+            and association_id_column_name not in data.columns
+            and self.auto_generate_association_ids
+        ):
+            data[association_id_column_name] = self.generate_association_ids(len(data))
+
+        # NOTE: no "pre-score" calculation on the DataFrame for the predictions
+
+        # perform the main "score" function for this model
+        predictions_df = self.run_model_score(data, model, drum_score_fn, **kwargs)
+
+        # make sure association ids get copied over
+        if (
+            association_id_column_name
+            and association_id_column_name not in predictions_df.columns
+            and association_id_column_name in data.columns
+        ):
+            predictions_df[association_id_column_name] = data[association_id_column_name]
+
+        # loop through all the metrics scoring with predictions_df that has citations
+        association_ids = (
+            []
+            if association_id_column_name not in predictions_df.columns
+            else predictions_df[association_id_column_name]
+        )
+        metric_reports = []
+        for scorer in self.scorers():
+            metric_info = self.custom_metric_map[scorer.name]
+            if metric_info.get("requires_association_id", False) and len(association_ids):
+                values = scorer.score_rows(predictions_df)
+                if not values:
+                    self.logger.debug(f"No {scorer} values found")
+                    continue
+
+                # assign back to the dataframe, so consumer has it
+                predictions_df[scorer.name] = values
+                metric_id = metric_info.get("id")
+                for association_id, value in zip(association_ids, values):
+                    metric_reports.append(
+                        self.custom_metric_individual_payload(metric_id, value, association_id)
+                    )
+                continue
+
+            value = scorer.score(predictions_df)
+            self.record_aggregate_value(scorer.name, value)
+
+        self.report_custom_metrics(metric_reports)
+        return predictions_df

{datarobot_moderations-11.1.22.dist-info → datarobot_moderations-11.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datarobot-moderations
-Version: 11.1.22
+Version: 11.2.1
 Summary: DataRobot Monitoring and Moderation framework
 License: DataRobot Tool and Utility Agreement
 Author: DataRobot
@@ -19,10 +19,9 @@ Requires-Dist: deepeval (==2.7.9)
 Requires-Dist: langchain (>=0.1.12)
 Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
 Requires-Dist: langchain-openai (>=0.1.7)
-Requires-Dist: llama-index (>=0.12.9)
+Requires-Dist: llama-index (>=0.12.49)
 Requires-Dist: llama-index-embeddings-azure-openai (>=0.1.6)
 Requires-Dist: llama-index-llms-bedrock-converse (>=0.1.6)
-Requires-Dist: llama-index-llms-fireworks (>=0.1.5)
 Requires-Dist: llama-index-llms-langchain (>=0.1.3)
 Requires-Dist: llama-index-llms-vertex (>=0.1.5)
 Requires-Dist: nemoguardrails (>=0.9.0)

{datarobot_moderations-11.1.22.dist-info → datarobot_moderations-11.2.1.dist-info}/RECORD

@@ -2,7 +2,7 @@ datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,58
 datarobot_dome/async_http_client.py,sha256=wkB4irwvnchNGzO1bk2C_HWM-GOSB3AUn5TXKl-X0ZI,9649
 datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
 datarobot_dome/constants.py,sha256=vM2_JkXbn4dkWARCqxNfLriSo0E05LDXVrwNktptpuc,10416
-datarobot_dome/drum_integration.py,sha256=DGtf2XfVb8uxNWOFyJwB6l7QObFLaDQFN13vYhA68Rs,42293
+datarobot_dome/drum_integration.py,sha256=HresblJwlCk_sRnWReWQWeZMg5rYzKTA2hjmy1Rcn6U,40553
 datarobot_dome/guard.py,sha256=1INYx17n9ToiB5bzI-jIReUUuqkK_ucxpOx4jQLts6g,33264
 datarobot_dome/guard_executor.py,sha256=AOI8MZeZETHMoFgBePe0wa2vE9d2975MYQnEDHLZL7s,35462
 datarobot_dome/guard_helpers.py,sha256=ajxm-w7MS7eN5DMMO-jbbzjcOYMZ-LvhO53n2NI5_Fk,16773
@@ -10,15 +10,15 @@ datarobot_dome/guards/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzI
 datarobot_dome/guards/guard_llm_mixin.py,sha256=VovlpNZjWIGamF4SSvLF5lzOFyApH5IoOiB_qtCmRg0,12216
 datarobot_dome/llm.py,sha256=L02OvTrflmD34-FrfXebfF-zzKTeuin7fpne1Cl5psg,5719
 datarobot_dome/metrics/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/metrics/citation_metrics.py,sha256=q0hTMWuk6wy_jqk2UjFPON3kU94HN3W2vxr9giJ8O8E,3544
+datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5QIBnKQ6shLnw4,4652
 datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
-datarobot_dome/metrics/metric_scorer.py,sha256=mGxW3NNP93LpbpOiX3MeYyd0YEEjTPE8WVYMGS4SWoY,2516
+datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
 datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/pipeline/llm_pipeline.py,sha256=fOp_OJnQMDUJH-LKv12kEqli-EqfHjAiSTFqtxzMkhM,19942
-datarobot_dome/pipeline/pipeline.py,sha256=YrGR3uS7kY9dd30B97cMESBbVNsze25RSoIcjEMqCb0,16251
-datarobot_dome/pipeline/vdb_pipeline.py,sha256=WTOGn1qe_ZvEcdlvHgeXxl2xTqp7GjfL13c6S-FmAfM,5146
+datarobot_dome/pipeline/llm_pipeline.py,sha256=g7PAiLOMADr2DQFrtg2NrUj4u_tcvnoiJXrBR8xWsmY,18789
+datarobot_dome/pipeline/pipeline.py,sha256=7UmvrZtNxTGewpgM4cf2oThHPoJSarEU1Dyp7xEsASU,17401
+datarobot_dome/pipeline/vdb_pipeline.py,sha256=q3c_Z-hGUqhH6j6n8VpS3wZiBIkWgpRDsBnyJyZhiw4,9855
 datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
 datarobot_dome/streaming.py,sha256=6nYvh6SoxPRLfO6GGdEoHsQuyLP9oX1lDMe8IeGo4lw,17801
-datarobot_moderations-11.1.22.dist-info/METADATA,sha256=tL9-6ZvQXyVLxDbx5EvMi4x3dlsxJJy_ussBOSoPPVQ,4794
-datarobot_moderations-11.1.22.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datarobot_moderations-11.1.22.dist-info/RECORD,,
+datarobot_moderations-11.2.1.dist-info/METADATA,sha256=fEyM5I3z0qS9dT2ofZ3J8UAN_ybm2SBF_46mifEHIpA,4742
+datarobot_moderations-11.2.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datarobot_moderations-11.2.1.dist-info/RECORD,,