aiverify-moonshot 0.4.0__py3-none-any.whl

moonshot/src/metrics/metric.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import validate_call
+
+from moonshot.src.configs.env_variables import EnvVariables
+from moonshot.src.metrics.metric_interface import MetricInterface
+from moonshot.src.storage.storage import Storage
+from moonshot.src.utils.import_modules import get_instance
+
+
+class Metric:
+    cache_name = "cache"
+    cache_extension = "json"
+
+    @classmethod
+    def load(cls, met_id: str) -> Metric:
+        """
+        Retrieves a metric instance by its ID.
+
+        This method attempts to load a metric instance using the provided ID. If the metric instance is found,
+        it is returned. If the metric instance does not exist, a RuntimeError is raised.
+
+        Args:
+            met_id (str): The unique identifier of the metric to be retrieved.
+
+        Returns:
+            Metric: The retrieved metric instance.
+
+        Raises:
+            RuntimeError: If the metric instance does not exist.
+        """
+        metric_instance = get_instance(
+            met_id,
+            Storage.get_filepath(EnvVariables.METRICS.name, met_id, "py"),
+        )
+        if metric_instance:
+            return metric_instance()
+        else:
+            raise RuntimeError(f"Unable to get defined metric instance - {met_id}")
+
+    @staticmethod
+    @validate_call
+    def delete(met_id: str) -> bool:
+        """
+        Deletes a metric identified by its unique metric ID.
+
+        This method attempts to delete the metric with the given ID from the storage. If the deletion is successful,
+        it returns True. If an exception occurs during the deletion process, it prints an error message and re-raises
+        the exception.
+
+        Args:
+            met_id (str): The unique identifier of the metric to be deleted.
+
+        Returns:
+            bool: True if the metric was successfully deleted.
+
+        Raises:
+            Exception: If an error occurs during the deletion process.
+        """
+        try:
+            Storage.delete_object(EnvVariables.METRICS.name, met_id, "py")
+            return True
+
+        except Exception as e:
+            print(f"Failed to delete metric: {str(e)}")
+            raise e
+
+    @staticmethod
+    def get_cache_information() -> dict:
+        """
+        Retrieves cache information from the storage.
+
+        This method attempts to read the cache information from the storage and return it as a dictionary.
+        If the cache information does not exist or an error occurs, it returns an empty dictionary.
+
+        Returns:
+            dict: A dictionary containing the cache information or an empty dictionary if an error occurs
+            or if the cache information does not exist.
+
+        Raises:
+            Exception: If there's an error during the retrieval process, it is logged and an
+            empty dictionary is returned.
+        """
+        try:
+            # Retrieve cache information from the storage and return it as a dictionary
+            cache_info = Storage.read_object(
+                EnvVariables.METRICS.name, Metric.cache_name, Metric.cache_extension
+            )
+            return cache_info if cache_info else {}
+        except Exception:
+            print(
+                f"No previous cache information because {Metric.cache_name} is not found."
+            )
+            return {}
+
+    @staticmethod
+    def write_cache_information(cache_info: dict) -> None:
+        """
+        Writes the updated cache information to the storage.
+
+        Args:
+            cache_info (dict): The cache information to be written.
+        """
+        try:
+            Storage.create_object(
+                obj_type=EnvVariables.METRICS.name,
+                obj_id=Metric.cache_name,
+                obj_info=cache_info,
+                obj_extension=Metric.cache_extension,
+            )
+        except Exception as e:
+            print(f"Failed to write cache information: {str(e)}")
+            raise e
+
+    @staticmethod
+    def get_available_items() -> tuple[list[str], list[dict]]:
+        """
+        Retrieves a list of available metric names and their corresponding information.
+
+        This method scans the storage for metric objects, filters out any system or cache-related entries,
+        and compiles a list of metric names and their detailed information. It also checks the cache for
+        any updates and writes back to the cache if necessary.
+
+        Returns:
+            tuple[list[str], list[dict]]: A tuple containing two lists, one with the names of the metrics
+                                           and the other with the corresponding metric information dictionaries.
+        """
+        try:
+            retn_mets = []
+            retn_mets_ids = []
+            met_cache_info = Metric.get_cache_information()
+            cache_needs_update = False  # Initialize a flag to track cache updates
+            mets = Storage.get_objects(EnvVariables.METRICS.name, "py")
+
+            for met in mets:
+                if "__" in met or MetricInterface.config_name in met:
+                    continue
+
+                met_name = Path(met).stem
+                met_info, cache_updated = Metric._get_or_update_metrics_info(
+                    met_name, met_cache_info
+                )
+                if cache_updated:
+                    cache_needs_update = True  # Set the flag if any cache was updated
+
+                retn_mets.append(met_info)
+                retn_mets_ids.append(met_name)
+
+            if cache_needs_update:  # Check the flag after the loop
+                Metric.write_cache_information(met_cache_info)
+
+            return retn_mets_ids, retn_mets
+
+        except Exception as e:
+            print(f"Failed to get available metrics: {str(e)}")
+            raise e
+
+    @staticmethod
+    def _get_or_update_metrics_info(
+        met_name: str, met_cache_info: dict
+    ) -> tuple[dict, bool]:
+        """
+        Retrieves or updates the metric information from the cache.
+
+        This method checks if the metric information is already available in the cache and if the file hash matches
+        the one stored in the cache. If it does, the information is retrieved from the cache. If not, the metric
+        information is read from the storage, the cache is updated with the new information and the new file hash,
+        and a flag is set to indicate that the cache has been updated.
+
+        Args:
+            met_name (str): The name of the metric.
+            met_cache_info (dict): A dictionary containing the cached metric information.
+
+        Returns:
+            tuple[dict, bool]: A tuple containing the dictionary with the metric information
+                               and a boolean indicating whether the cache was updated or not.
+        """
+        file_hash = Storage.get_file_hash(EnvVariables.METRICS.name, met_name, "py")
+        cache_updated = False
+
+        if met_name in met_cache_info and file_hash == met_cache_info[met_name]["hash"]:
+            met_metadata = met_cache_info[met_name].copy()
+            met_metadata.pop("hash", None)
+        else:
+            met_metadata = Metric.load(met_name).get_metadata()  # type: ignore ; ducktyping
+            met_cache_info[met_name] = met_metadata.copy()
+            met_cache_info[met_name]["hash"] = file_hash
+            cache_updated = True
+
+        return met_metadata, cache_updated

moonshot/src/metrics/metric_interface.py

@@ -0,0 +1,95 @@
+from abc import abstractmethod
+from typing import Any
+
+from moonshot.src.configs.env_variables import EnvVariables
+from moonshot.src.storage.storage import Storage
+from moonshot.src.utils.timeit import timeit
+
+
+class MetricInterface:
+    config_name = "metrics_config"
+    config_extension = "json"
+
+    @abstractmethod
+    @timeit
+    def get_metadata(self) -> dict | None:
+        """
+        Abstract method to retrieve metadata from a metric class.
+
+        Returns:
+            dict | None: Returns a dictionary containing the metadata of the metric class, or None if the operation was
+            unsuccessful.
+        """
+        pass
+
+    @abstractmethod
+    async def get_results(
+        self, prompts: Any, predicted_results: Any, targets: Any, *args, **kwargs
+    ) -> dict:
+        """
+        Abstract method to compute metrics based on model predictions.
+
+        This method is intended to be implemented by subclasses to calculate specific
+        metrics (e.g., accuracy, BLEU score) based on the inputs provided.
+        It should handle any necessary preprocessing of inputs, computation of the metric, and formatting of the output.
+
+        Args:
+            prompts (Any): The input prompts used to generate the predictions.
+            The exact type and format will depend on the specific implementation and the nature of the
+            task (e.g., text, images).
+
+            predicted_results (Any): The model's predictions in response to the prompts.
+            The type and format should align with the prompts and the expected output of the model.
+
+            targets (Any): The ground truth targets against which the predicted results will be evaluated.
+
+            *args: Variable length argument list for additional parameters that might be required for specific metric
+            computations.
+
+            **kwargs: Arbitrary keyword arguments for additional options or configuration settings that might be
+            required for specific metric computations.
+
+        Returns:
+            dict: A dictionary containing the computed metric(s). The keys should clearly indicate the metric being
+            reported, and the values should be the corresponding metric values.
+        """
+        pass
+
+    def get_metrics_configuration(self, met_id: str) -> dict:
+        """
+        Retrieves the configuration for a specific metric by its identifier.
+
+        Args:
+            met_id (str): The identifier for the metric configuration to retrieve.
+
+        Returns:
+            dict: The metric configuration as a dictionary. Returns an empty dict if the configuration is not found.
+
+        Raises:
+            Exception: If reading the metrics configuration fails or if the configuration cannot be created.
+        """
+        try:
+            obj_results = Storage.read_object(
+                EnvVariables.METRICS.name, "metrics_config", "json"
+            )
+            return obj_results.get(met_id, {})
+
+        except Exception as e:
+            print(f"[MetricInterface] Failed to read metrics configuration: {str(e)}")
+            print("Attempting to create empty metrics configuration...")
+            try:
+                Storage.create_object(
+                    obj_type=EnvVariables.METRICS.name,
+                    obj_id="metrics_config",
+                    obj_info={},
+                    obj_extension="json",
+                )
+                # After creation, attempt to read it again to ensure it was created successfully
+                obj_results = Storage.read_object(
+                    EnvVariables.METRICS.name, "metrics_config", "json"
+                )
+                return obj_results.get(met_id, {})
+            except Exception as e:
+                raise Exception(
+                    f"[MetricInterface] Failed to create metrics configuration: {str(e)}"
+                )

moonshot/src/prompt_templates/prompt_template.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+
+from jinja2 import Template
+
+from moonshot.src.configs.env_variables import EnvVariables
+from moonshot.src.storage.storage import Storage
+
+
+class PromptTemplate:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_all_prompt_template_names() -> list[str]:
+        """
+        Retrieve the names of all prompt templates available.
+
+        Returns:
+            list: A list of prompt template names.
+        """
+
+        filepaths = []
+        prompt_template_files = Storage.get_objects(
+            EnvVariables.PROMPT_TEMPLATES.name, "json"
+        )
+        for prompt_template in prompt_template_files:
+            filepaths.append(Path(prompt_template).stem)
+        return filepaths
+
+    @staticmethod
+    def get_all_prompt_template_details() -> list[dict]:
+        """
+        Retrieve details of all prompt templates available.
+
+        Returns:
+            list[dict]: A list of dictionaries containing the details of each prompt template.
+        """
+        list_of_pt_names = PromptTemplate.get_all_prompt_template_names()
+        list_of_pt_contents = []
+
+        for pt_name in list_of_pt_names:
+            pt_contents = Storage.read_object(
+                EnvVariables.PROMPT_TEMPLATES.name, pt_name, "json"
+            )
+            id = pt_name
+            name = pt_contents["name"]
+            description = pt_contents["description"]
+            template = pt_contents["template"]
+            list_of_pt_contents.append(
+                {
+                    "id": id,
+                    "name": name,
+                    "description": description,
+                    "template": template,
+                }
+            )
+        return list_of_pt_contents
+
+    @staticmethod
+    def delete(pt_id: str) -> bool:
+        """
+        Deletes a prompt template identified by its unique ID.
+
+        This method attempts to delete the prompt template with the given ID from the storage.
+        If the deletion is successful, it returns True. If an exception occurs during the deletion
+        process, it prints an error message and re-raises the exception.
+
+        Args:
+            pt_id (str): The unique identifier of the prompt template to be deleted.
+
+        Returns:
+            bool: True if the prompt template was successfully deleted.
+
+        Raises:
+            Exception: If an error occurs during the deletion process.
+        """
+        try:
+            Storage.delete_object(EnvVariables.PROMPT_TEMPLATES.name, pt_id, "json")
+            return True
+
+        except Exception as e:
+            print(f"Failed to delete prompt template: {str(e)}")
+            raise e
+
+    @staticmethod
+    def process_prompt_pt(user_prompt: str, prompt_template_name: str) -> str:
+        """
+        Process a user prompt using a specified prompt template.
+
+        Args:
+            user_prompt (str): The user prompt to process.
+            prompt_template (str): The name of the prompt template to use.
+
+        Returns:
+            str: The processed user prompt based on the template.
+        """
+
+        prompt_template_file = Storage.read_object(
+            EnvVariables.PROMPT_TEMPLATES.name, prompt_template_name, "json"
+        )
+        template = prompt_template_file["template"]
+        jinja_template = Template(template)
+        return jinja_template.render({"prompt": user_prompt})