levelapp 0.1.15__py3-none-any.whl

levelapp/repository/firestore.py

@@ -0,0 +1,291 @@
+"""levelapp/repository/firestore.py"""
+from typing import List, Dict, Any, Type, TYPE_CHECKING
+from pydantic import ValidationError
+
+from google.cloud import firestore_v1
+from google.cloud.firestore_v1 import DocumentSnapshot
+from google.api_core.exceptions import ClientError, ServerError, NotFound, InvalidArgument, DeadlineExceeded
+from google.auth.exceptions import DefaultCredentialsError
+
+from levelapp.core.base import BaseRepository, Model
+from levelapp.aspects import logger
+
+
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
+
+class FirestoreRepository(BaseRepository):
+    """
+    Firestore implementation of BaseRepository.
+    (Uses hierarchical path: {user_id}/{collection_id}/{document_id}
+    """
+
+    def __init__(self, config: "WorkflowConfig | None"):
+        if config:
+            self.config = config
+            self.project_id: str | Any = config.repository.project_id
+            self.database_name: str | Any = config.repository.database_name
+        else:
+            self.project_id: str | Any = None
+            self.database_name: str | Any = '(default)'
+
+        self.client: firestore_v1.Client | None = None
+
+    def connect(self) -> None:
+        """
+        Connects to Firestore, prioritizing the project ID passed to the constructor.
+        """
+        try:
+            import google.auth
+            credentials, default_project_id = google.auth.default()
+
+            if not credentials:
+                raise ValueError(
+                    "Failed to obtain credentials. "
+                    "Please set GOOGLE_APPLICATION_CREDENTIALS "
+                    "or run 'gcloud auth application-default login'."
+                )
+
+            project_id = self.project_id if self.project_id else default_project_id
+
+            self.client = firestore_v1.Client(
+                project=project_id,
+                credentials=credentials,
+                database=self.database_name
+            )
+
+            if not self.client:
+                raise ValueError("Failed to initialize Firestore client")
+
+            logger.info(
+                f"Successfully connected to Firestore. "
+                f"Project: '{self.client.project}', "
+                f"Scope: '{self.client.SCOPE}'"
+            )
+
+        except (ClientError, ServerError, DefaultCredentialsError, ValueError) as e:
+            logger.error(f"Failed to initialize Firestore client:\n{e}")
+
+    def close(self) -> None:
+        if self.client:
+            self.client.close()
+
+    def retrieve_document(
+            self,
+            collection_id: str,
+            section_id: str,
+            sub_collection_id: str,
+            document_id: str,
+            model_type: Type[Model]
+    ) -> Model | None:
+        """
+        Retrieves a document from Firestore.
+
+        Args:
+            collection_id (str): User reference.
+            section_id (str): Section reference.
+            sub_collection_id (str): Collection reference.
+            document_id (str): Document reference.
+            model_type (Type[Model]): Pydantic model for parsing.
+
+        Returns:
+            An instance of the provide Pydantic model.
+        """
+        if not self.client:
+            logger.error("Client connection lost")
+            return None
+
+        try:
+            doc_ref = (
+                self.client
+                .collection(collection_id)
+                .document(section_id)
+                .collection(sub_collection_id)
+                .document(document_id)
+            )
+            snapshot: DocumentSnapshot = doc_ref.get()
+
+            if not snapshot.exists:
+                logger.warning(f"Document '{document_id}' does not exist in Firestore")
+                return None
+
+            data = snapshot.to_dict()
+            return model_type.model_validate(data)
+
+        except NotFound as e:
+            logger.warning(f"Failed to retrieve Firestore document <ID:{document_id}>:\n{e}")
+            return None
+
+        except InvalidArgument as e:
+            logger.error(f"Invalid argument in document path <{sub_collection_id}/{sub_collection_id}/{document_id}>:\n{e}")
+            return None
+
+        except DeadlineExceeded as e:
+            logger.error(f"Request to retrieved document <ID:{document_id}> timout:\n{e}")
+            return None
+
+        except ValidationError as e:
+            logger.exception(f"Failed to parse the retrieved document <ID:{document_id}>:\n{e}")
+            return None
+
+        except Exception as e:
+            logger.exception(f"Failed to retrieve Firestore document <ID:{document_id}>:\n{e}")
+            return None
+
+    def store_document(
+            self,
+            collection_id: str,
+            section_id: str,
+            sub_collection_id: str,
+            document_id: str,
+            data: Model
+    ) -> None:
+        """
+        Stores a document in Firestore.
+
+        Args:
+            collection_id (str): Collection reference.
+            section_id (str): Section reference.
+            sub_collection_id (str): Sub-collection reference.
+            document_id (str): Document reference.
+            data (Model): An instance of the Pydantic model containing the data.
+        """
+        if not self.client:
+            logger.error("Client connection lost")
+
+        try:
+            doc_ref = (
+                self.client
+                .collection(collection_id)
+                .document(section_id)
+                .collection(sub_collection_id)
+                .document(document_id)
+            )
+            data = data.model_dump()
+            doc_ref.set(data)
+
+        except NotFound as e:
+            logger.warning(f"Failed to store Firestore document <ID:{document_id}>:\n{e}")
+            return None
+
+        except InvalidArgument as e:
+            logger.error(f"Invalid argument in document path <{sub_collection_id}/{sub_collection_id}/{document_id}>:\n{e}")
+            return None
+
+        except DeadlineExceeded as e:
+            logger.error(f"Request to retrieved document <ID:{document_id}> timout:\n{e}")
+            return None
+
+        except ValidationError as e:
+            logger.exception(f"Failed to parse the retrieved document <ID:{document_id}>:\n{e}")
+            return None
+
+        except Exception as e:
+            logger.exception(f"Failed to retrieve Firestore document <ID:{document_id}>:\n{e}")
+            return None
+
+    def query_collection(
+            self,
+            collection_id: str,
+            section_id: str,
+            sub_collection_id: str,
+            filters: Dict[str, Any],
+            model_type: Type[Model]
+    ) -> List[Model]:
+        """
+        Queries a collection with specified filters.
+
+        Args:
+            collection_id (str): Collection reference.
+            section_id (str): Section reference.
+            sub_collection_id (str): Sub-collection reference.
+            filters (Dict[str, Any]): A dictionary of key-value pairs to filter the query.
+            model_type (Type [Model]): The class to deserialize the documents into.
+
+        Returns:
+            A list of deserialized models that match the query.
+        """
+        if not self.client:
+            logger.error("Client connection lost")
+            return []
+
+        try:
+            collection_ref = self.client.collection('users', collection_id, sub_collection_id)
+            query = collection_ref
+
+            for key, value in filters.items():
+                query = query.where(key, "==", value)
+
+            results = []
+            for doc in query.stream():
+                if doc.exists and doc.to_dict():
+                    results.append(model_type.model_validate(doc.to_dict()))
+
+            return results
+
+        except NotFound as e:
+            logger.warning(f"Collection for user '{collection_id}' not found:\n{e}")
+            return []
+
+        except InvalidArgument as e:
+            logger.error(f"Invalid query argument for user '{collection_id}':\n{e}")
+            return []
+
+        except DeadlineExceeded as e:
+            logger.error(f"Query for user '{collection_id}' timed out:\n{e}")
+            return []
+
+        except ValidationError as e:
+            logger.exception(f"Failed to parse a document from query results:\n{e}")
+            return []
+
+        except Exception as e:
+            logger.exception(f"An unexpected error occurred during collection query:\n{e}")
+            return []
+
+    def delete_document(
+            self,
+            collection_id: str,
+            section_id: str,
+            sub_collection_id: str,
+            document_id: str
+    ) -> bool:
+        """
+        Deletes a document from Firestore.
+
+        Fields:
+            collection_id (str): Collection reference.
+            section_id (str): Section reference.
+            sub_collection_id (str): Sub-collection reference.
+            document_id (str): Document reference.
+
+        Returns:
+            True if the document was deleted successfully, False otherwise.
+        """
+        if not self.client:
+            logger.error("Client connection lost")
+            return False
+
+        try:
+            doc_ref = self.client.collection(
+                collection_id,
+                section_id,
+                sub_collection_id
+            ).document(document_id)
+            doc_ref.delete()
+            logger.info(f"Document '{document_id}' deleted successfully.")
+            return True
+
+        except NotFound as e:
+            logger.warning(f"Failed to delete document. Document '{document_id}' not found:\n{e}")
+            return False
+        except InvalidArgument as e:
+            logger.error(f"Invalid argument in document path <{collection_id}/{sub_collection_id}/{document_id}>:\n{e}")
+            return False
+        except DeadlineExceeded as e:
+            logger.error(f"Request to delete document <ID:{document_id}> timed out:\n{e}")
+            return False
+        except Exception as e:
+            logger.exception(f"Failed to delete Firestore document <ID:{document_id}>:\n{e}")
+            return False

levelapp/simulator/__init__.py

@@ -0,0 +1,3 @@
+from .simulator import ConversationSimulator
+
+__all__ = ['ConversationSimulator']

levelapp/simulator/schemas.py

@@ -0,0 +1,116 @@
+"""
+levelapp/simulator/schemas.py
+
+Defines Pydantic models for simulator-related data structures,
+including test configurations, batch metadata, and evaluation results.
+"""
+from enum import Enum
+from uuid import UUID, uuid4
+from datetime import datetime
+
+from typing import Dict, Any, List
+from pydantic import BaseModel, Field, computed_field
+
+from levelapp.evaluator.evaluator import JudgeEvaluationResults
+
+
+class InteractionLevel(str, Enum):
+    """Enum representing the type of interaction."""
+    INITIAL = "initial"
+    INTERMEDIATE = "intermediate"
+    FINAL = "final"
+
+
+class Interaction(BaseModel):
+    """Represents a single interaction within a conversation."""
+    id: UUID = Field(default_factory=uuid4, description="Interaction identifier")
+    user_message_path: str = Field(..., description="Path of the user message in the request payload")
+    user_message: str = Field(..., description="The user's query message")
+    reference_reply: str = Field(..., description="The preset reference message")
+    interaction_type: InteractionLevel = Field(default=InteractionLevel.INITIAL, description="Type of interaction")
+    reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
+    guardrail_flag: Any = Field(default=False, description="Flag for guardrail signaling")
+    request_payload: Dict[str, Any] = Field(default_factory=dict, description="Additional request payload")
+
+
+class ConversationScript(BaseModel):
+    """Represents a basic conversation with multiple interactions."""
+    id: UUID = Field(default_factory=uuid4, description="Conversation identifier")
+    interactions: List[Interaction] = Field(default_factory=list, description="List of interactions")
+    description: str = Field(default="no-description", description="A short description of the conversation")
+    details: Dict[str, str] = Field(default_factory=dict, description="Conversation details")
+    variable_request_schema: bool = Field(default=False, description="The payload schema changes for each request")
+    uuid_field: str | None = Field(default=None, description="field that requires a UUID value")
+
+
+class ScriptsBatch(BaseModel):
+    id: UUID = Field(default_factory=uuid4, description="Batch identifier")
+    scripts: List[ConversationScript] = Field(default_factory=list, description="List of conversation scripts")
+
+
+# ---- Interaction Details Models ----
+class InteractionResults(BaseModel):
+    """Represents metadata extracted from a VLA interaction."""
+    generated_reply: str | None = "No response"
+    generated_metadata: Dict[str, Any] | None = {}
+    guardrail_flag: Any | None = False
+    interaction_type: str | None = ""
+
+
+class InteractionEvaluationResults(BaseModel):
+    """Model representing the evaluation result of an interaction."""
+    judge_evaluations: Dict[str, JudgeEvaluationResults] | None = Field(default_factory=dict)
+    metadata_evaluation: Dict[str, float] | None = Field(default_factory=dict)
+    guardrail_flag: int | None = Field(default=None)
+    errors: Dict[str, Any] | None = Field(default_factory=dict)
+
+
+class SingleInteractionResults(BaseModel):
+    """Represents metadata extracted from VLA interaction."""
+    conversation_id: str = Field(description="Conversation identifier")
+    user_message: str = Field(default="", description="The user's query message")
+    generated_reply: str = Field(default="Interaction request failed", description="The generated reply message")
+    reference_reply: str = Field(default="", description="The preset reference message")
+    generated_metadata: Dict[str, Any] = Field(default_factory=dict, description="Extracted metadata")
+    reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
+    guardrail_details: bool | None = Field(default=None, description="Flag for guardrail signaling")
+    evaluation_results: InteractionEvaluationResults = Field(default_factory=InteractionEvaluationResults)
+    response_content: Dict[str, Any] = Field(default_factory=dict, description="Response content")
+    errors: Dict[str, Any] = Field(default_factory=dict, description="Captured errors")
+
+
+class SingleAttemptResults(BaseModel):
+    attempt_nbr: int = Field(default=1, description="The attempt number")
+    attempt_id: str = Field(default=None, description="The attempt ID")
+    script_id: str = Field(default=None, description="The script ID")
+    total_duration: float = Field(default=None, description="Total duration")
+    interaction_results: List[SingleInteractionResults] = Field(default_factory=list)
+    evaluation_verdicts: Dict[str, List[str]] = Field(default_factory=dict)
+    average_scores: Dict[str, float] = Field(default_factory=dict)
+    interaction_summaries: List[str] = Field(default_factory=list)
+
+
+class AllAttemptsResults(BaseModel):
+    script_id: str = Field(default=None, description="The script ID")
+    attempts: List[SingleAttemptResults] = Field(default_factory=list)
+    average_scores: Dict[str, float] = Field(default_factory=dict)
+
+
+class SimulationResults(BaseModel):
+    # Collected data
+    started_at: datetime = datetime.now()
+    finished_at: datetime
+    # Collected Results
+    evaluation_summary: Dict[str, Any] | None = Field(default_factory=dict, description="Evaluation result")
+    average_scores: Dict[str, Any] | None = Field(default_factory=dict, description="Average scores")
+    script_results: List[AllAttemptsResults] | None = Field(default_factory=list, description="detailed results")
+
+    @computed_field
+    @property
+    def batch_id(self) -> str:
+        return str(uuid4())
+
+    @computed_field
+    @property
+    def elapsed_time(self) -> float:
+        return (self.finished_at - self.started_at).total_seconds()