levelapp 0.1.0__py3-none-any.whl

levelapp/workflow/base.py

@@ -0,0 +1,113 @@
+import asyncio
+
+from abc import ABC, abstractmethod
+from pydantic import ValidationError
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+from levelapp.core.base import BaseProcess
+from levelapp.simulator.schemas import ScriptsBatch
+from levelapp.simulator.simulator import ConversationSimulator
+from levelapp.aspects.loader import DataLoader
+from levelapp.workflow.schemas import WorkflowContext
+
+
+class BaseWorkflow(ABC):
+    """Abstract base class for evaluation workflows."""
+
+    def __init__(self, name: str, context: WorkflowContext) -> None:
+        self.name = name
+        self.context = context
+        self.process: BaseProcess | None = None
+        self._input_data: Any | None = None
+        self._results: Any | None = None
+        self._initialized: bool = False
+
+    def setup(self) -> None:
+        """Validate and initialize workflow-specific settings."""
+        if self._initialized:
+            return
+        self.process = self._setup_process(context=self.context)
+        self._initialized = True
+
+    def load_data(self) -> None:
+        """Load and preprocess input data."""
+        if not self._initialized:
+            raise RuntimeError(f"[{self.name}] Workflow not initialized. Call setup() first.")
+        self._input_data = self._load_input_data(context=self.context)
+
+    def execute(self) -> None:
+        """Run the workflow evaluation steps."""
+        if not self._input_data:
+            raise RuntimeError(f"[{self.name}] No reference data available.")
+
+        if asyncio.iscoroutinefunction(self.process.run):
+            self._results = asyncio.run(self.process.run(**self._input_data))
+        else:
+            self._results = self.process.run(**self._input_data)
+
+    async def aexecute(self) -> None:
+        if not self._input_data:
+            raise RuntimeError(f"[{self.name}] No reference data available.")
+
+        if asyncio.iscoroutinefunction(self.process.run):
+            self._results = await self.process.run(**self._input_data)
+        else:
+            loop = asyncio.get_running_loop()
+            func = partial(self.process.run, **self._input_data)
+            self._results = await loop.run_in_executor(None, func)
+
+    def collect_results(self) -> Any:
+        """Return unified results structure."""
+        return self._results
+
+    @abstractmethod
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _load_input_data(self, context: WorkflowContext) -> Any:
+        raise NotImplementedError
+
+
+class SimulatorWorkflow(BaseWorkflow):
+    def __init__(self, context: WorkflowContext) -> None:
+        super().__init__(name="ConversationSimulator", context=context)
+
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        simulator = ConversationSimulator()
+        simulator.setup(
+            repository=context.repository,
+            evaluators=context.evaluators,
+            endpoint_config=context.endpoint_config,
+        )
+        return simulator
+
+    def _load_input_data(self, context: WorkflowContext) -> Any:
+        loader = DataLoader()
+        reference_data_path = context.inputs.get("reference_data_path", "no-path-provided")
+        file_path = Path(reference_data_path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"[{self.name}] Reference data file not found.")
+
+        evaluation_params = context.inputs.get("evaluation_params", {})
+        data_config = loader.load_raw_data(path=reference_data_path)
+        try:
+            scripts_batch = ScriptsBatch.model_validate(data_config)
+        except ValidationError as e:
+            raise RuntimeError(f"[{self.name}] Validation error: {e}")
+
+        return {"test_batch": scripts_batch, "attempts": evaluation_params.get("attempts", 1)}
+
+
+class ComparatorWorkflow(BaseWorkflow):
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        pass
+
+    def _load_input_data(self, context: WorkflowContext) -> Any:
+        pass
+
+    def __init__(self, context: WorkflowContext) -> None:
+        super().__init__(name="MetadataComparator", context=context)

levelapp/workflow/factory.py

@@ -0,0 +1,51 @@
+from typing import Callable, Dict
+from levelapp.workflow.schemas import WorkflowType, RepositoryType, EvaluatorType, WorkflowConfig, WorkflowContext
+from levelapp.core.base import BaseRepository, BaseEvaluator
+from levelapp.workflow.base import BaseWorkflow
+
+from levelapp.repository.firestore import FirestoreRepository
+from levelapp.evaluator.evaluator import JudgeEvaluator, MetadataEvaluator
+
+
+class MainFactory:
+    """Central factory for repositories, evaluators, and workflows."""
+
+    _repository_map: dict[RepositoryType, Callable[[WorkflowConfig], BaseRepository]] = {
+        RepositoryType.FIRESTORE: lambda cfg: FirestoreRepository(),
+    }
+
+    _evaluator_map: dict[EvaluatorType, Callable[[WorkflowConfig], BaseEvaluator]] = {
+        EvaluatorType.JUDGE: lambda cfg: JudgeEvaluator(),
+        EvaluatorType.REFERENCE: lambda cfg: MetadataEvaluator(),
+        # Next is the RAG evaluator..
+    }
+
+    _workflow_map: dict[WorkflowType, Callable[["WorkflowContext"], BaseWorkflow]] = {}
+
+    @classmethod
+    def create_repository(cls, config: WorkflowConfig) -> BaseRepository:
+        fn = cls._repository_map.get(config.repository)
+        if not fn:
+            raise NotImplementedError(f"Repository {config.repository} not implemented")
+        return fn(config)
+
+    @classmethod
+    def create_evaluator(cls, config: WorkflowConfig) -> Dict[EvaluatorType, BaseEvaluator]:
+        evaluators: dict[EvaluatorType, BaseEvaluator] = {}
+        for ev in config.evaluators:
+            fn = cls._evaluator_map.get(ev)
+            if not fn:
+                raise NotImplementedError(f"Evaluator {config.evaluators} not implemented")
+            evaluators[ev] = fn(config)
+        return evaluators
+
+    @classmethod
+    def create_workflow(cls, wf_type: WorkflowType, context: "WorkflowContext") -> BaseWorkflow:
+        fn = cls._workflow_map.get(wf_type)
+        if not fn:
+            raise NotImplementedError(f"Workflow {wf_type} not implemented")
+        return fn(context)
+
+    @classmethod
+    def register_workflow(cls, wf_type: WorkflowType, builder: Callable[["WorkflowContext"], BaseWorkflow]) -> None:
+        cls._workflow_map[wf_type] = builder

levelapp/workflow/registration.py

@@ -0,0 +1,6 @@
+from levelapp.workflow.factory import MainFactory
+from levelapp.workflow.schemas import WorkflowType
+from levelapp.workflow.base import SimulatorWorkflow, ComparatorWorkflow
+
+MainFactory.register_workflow(WorkflowType.SIMULATOR, lambda ctx: SimulatorWorkflow(ctx))
+MainFactory.register_workflow(WorkflowType.COMPARATOR, lambda ctx: ComparatorWorkflow(ctx))

levelapp/workflow/schemas.py

@@ -0,0 +1,121 @@
+from enum import Enum
+from pydantic import BaseModel
+from dataclasses import dataclass
+from typing import List, Dict, Any
+
+from levelapp.config.endpoint import EndpointConfig
+from levelapp.core.base import BaseRepository, BaseEvaluator
+from levelapp.aspects import DataLoader
+
+
+class ExtendedEnum(Enum):
+    @classmethod
+    def list(cls):
+        return [e.value for e in cls]
+
+
+class WorkflowType(ExtendedEnum):
+    SIMULATOR = "SIMULATOR"
+    COMPARATOR = "COMPARATOR"
+    ASSESSOR = "ASSESSOR"
+
+
+class RepositoryType(ExtendedEnum):
+    FIRESTORE = "FIRESTORE"
+    FILESYSTEM = "FILESYSTEM"
+
+
+class EvaluatorType(ExtendedEnum):
+    JUDGE = "JUDGE"
+    REFERENCE = "REFERENCE"
+    RAG = "RAG"
+
+
+class WorkflowConfig:
+    """Configuration for a workflow, loaded from JSON/YAML via DataLoader."""
+
+    # Class-level constant
+    _fields_list: List[str] = [
+        "project_name",
+        "evaluation_params",
+        "workflow",
+        "repository",
+        "evaluators",
+        "reference_data",
+    ]
+
+    def __init__(
+        self,
+        workflow: WorkflowType,
+        repository: RepositoryType,
+        evaluators: List[EvaluatorType],
+        endpoint_config: EndpointConfig,
+        inputs: Dict[str, Any],
+    ):
+        self.workflow = workflow
+        self.repository = repository
+        self.evaluators = evaluators
+        self.endpoint_config = endpoint_config
+        self.inputs = inputs
+
+    @classmethod
+    def load(cls, path: str | None = None) -> "WorkflowConfig":
+        """Load and validate workflow configuration from a file."""
+        loader = DataLoader()
+        config_dict = loader.load_raw_data(path=path)
+        model_config: BaseModel = loader.create_dynamic_model(data=config_dict, model_name="WorkflowConfiguration")
+
+        cls._check_fields(model_config)
+        cls._check_values(model_config)
+
+        workflow = WorkflowType(model_config.workflow)
+        repository = RepositoryType(model_config.repository)
+
+        if isinstance(model_config.evaluators, str):
+            print(f"evaluators: {model_config.evaluators}")
+            evaluators = [EvaluatorType(model_config.evaluators)]
+        else:
+            evaluators = [EvaluatorType(e) for e in model_config.evaluators]
+
+        evaluation_params = model_config.evaluation_params.model_dump()
+        reference_data_path = getattr(model_config.reference_data, "path", None)
+        endpoint_config = EndpointConfig.model_validate(model_config.endpoint_configuration.model_dump())
+
+        return cls(
+            workflow=workflow,
+            repository=repository,
+            evaluators=evaluators,
+            endpoint_config=endpoint_config,
+            inputs={'reference_data_path': reference_data_path, 'evaluation_params': evaluation_params},
+        )
+
+    @classmethod
+    def _check_fields(cls, config: BaseModel) -> None:
+        for field_name in cls._fields_list:
+            if field_name not in config.model_fields:
+                raise ValueError(f"[WorkflowConfig] Field '{field_name}' missing in configuration")
+
+    @staticmethod
+    def _check_values(config: BaseModel) -> None:
+        if config.workflow not in WorkflowType.list():
+            raise ValueError(f"[WorkflowConfig] Unsupported workflow type '{config.workflow}'")
+        if config.repository not in RepositoryType.list():
+            raise ValueError(f"[WorkflowConfig] Unsupported repository type '{config.repository}'")
+
+        evals = config.evaluators
+        if isinstance(evals, str):
+            evals = [evals]
+
+        for e in evals:
+            if e not in EvaluatorType.list():
+                raise ValueError(f"[WorkflowConfig] Unsupported evaluator type '{config.evaluators}'")
+
+
+@dataclass(frozen=True)
+class WorkflowContext:
+    """Immutable data holder for workflow execution context."""
+    config: WorkflowConfig
+    repository: BaseRepository
+    evaluators: Dict[str, BaseEvaluator]
+    endpoint_config: EndpointConfig
+    inputs: Dict[str, Any]

levelapp-0.1.0.dist-info/METADATA

@@ -0,0 +1,254 @@
+Metadata-Version: 2.4
+Name: levelapp
+Version: 0.1.0
+Summary: LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]
+Project-URL: Homepage, https://github.com/levelapp-org
+Project-URL: Repository, https://github.com/levelapp-org/levelapp-framework
+Project-URL: Documentation, https://levelapp.readthedocs.io
+Project-URL: Issues, https://github.com/levelapp-org/levelapp-framework/issues
+Author-email: KadriSof <kadrisofyen@gmail.com>
+License-File: LICENSE
+Keywords: ai,evaluation,framework,llm,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.12
+Requires-Dist: arrow>=1.3.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: numpy>=2.3.2
+Requires-Dist: openai>=1.99.9
+Requires-Dist: pandas-stubs==2.3.0.250703
+Requires-Dist: pandas>=2.3.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: python-dotenv>=1.1.1
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: rapid>=0.0.3
+Requires-Dist: rapidfuzz>=3.13.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: tenacity>=9.1.2
+Provides-Extra: dev
+Requires-Dist: arrow>=1.3.0; extra == 'dev'
+Requires-Dist: httpx>=0.28.1; extra == 'dev'
+Requires-Dist: numpy>=2.3.2; extra == 'dev'
+Requires-Dist: openai>=1.99.9; extra == 'dev'
+Requires-Dist: pandas-stubs==2.3.0.250703; extra == 'dev'
+Requires-Dist: pandas>=2.3.1; extra == 'dev'
+Requires-Dist: pydantic>=2.11.7; extra == 'dev'
+Requires-Dist: python-dotenv>=1.1.1; extra == 'dev'
+Requires-Dist: pyyaml>=6.0.2; extra == 'dev'
+Requires-Dist: rapid>=0.0.3; extra == 'dev'
+Requires-Dist: rapidfuzz>=3.13.0; extra == 'dev'
+Requires-Dist: requests>=2.32.4; extra == 'dev'
+Requires-Dist: tenacity>=9.1.2; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# LevelApp: AI/LLM Evaluation Framework for Regression Testing
+
+[![PyPI version](https://badge.fury.io/py/levelapp.svg)](https://badge.fury.io/py/levelapp)  
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)  
+[![Python Version](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/)
+
+## Overview
+
+LevelApp is an evaluation framework designed for regression testing (black-box) of already built LLM-based systems in production or testing phases. It focuses on assessing the performance and reliability of AI/LLM applications through simulation and comparison modules. Powered by Norma.
+
+Key benefits:
+- Configuration-driven: Minimal coding required; define evaluations via YAML files.
+- Supports LLM-as-a-judge for qualitative assessments and quantitative metrics for metadata evaluation.
+- Modular architecture for easy extension to new workflows, evaluators, and repositories.
+
+## Features
+
+- **Simulator Module**: Evaluates dialogue systems by simulating conversations using predefined scripts. It uses an LLM as a judge to score replies against references and supports metrics (e.g., Exact, Embedded, Token-based, Fuzzy) for comparing extracted metadata to ground truth.
+- **Comparator Module**: Evaluates metadata extraction from JSON outputs (e.g., from legal/financial document processing with LLMs) by comparing against reference/ground-truth data.
+- **Configuration-Based Workflow**: Users provide YAML configs for endpoints, parameters, data sources, and metrics, reducing the need for custom code.
+- **Supported Workflows**: SIMULATOR, COMPARATOR, ASSESSOR (coming soon!).
+- **Repositories**: FIRESTORE, FILESYSTEM, MONGODB.
+- **Evaluators**: JUDGE, REFERENCE, RAG.
+- **Metrics**: Exact, Levenshtein, and more (see docs for full list).
+- **Data Sources**: Local or remote JSON for conversation scripts.
+
+## Installation
+
+Install LevelApp via pip:
+
+```bash
+pip install levelapp
+```
+
+### Prerequisites
+- Python 3.12 or higher.
+- API keys for LLM providers (e.g., OpenAI, Anthropic) if using external clients—store in a `.env` file.
+- Optional: Google Cloud credentials for Firestore repository.
+- Dependencies are automatically installed, including `openai`, `pydantic`, `numpy`, etc. (see `pyproject.toml` for full list).
+
+## Configuration
+
+LevelApp uses a YAML configuration file to define the evaluation setup. Create a `workflow_config.yaml` with the following structure:
+
+```yaml
+project_name: "test-project"
+evaluation_params:
+  attempts: 1  # Number of simulation attempts.
+
+workflow: SIMULATOR  # SIMULATOR, COMPARATOR, ASSESSOR.
+repository: FIRESTORE  # FIRESTORE, FILESYSTEM, MONGODB.
+evaluators: # JUDGE, REFERENCE, RAG.
+  - JUDGE
+  - REFERENCE
+
+endpoint_configuration:
+  base_url: "http://127.0.0.1:8000"
+  url_path: ''
+  api_key: "<API-KEY>"
+  bearer_token: "<BEARER-TOKEN>"
+  model_id: "meta-llama/Meta-Llama-3.1-8B-Instruct"
+  payload_path: "../../src/data/payload_example_1.yaml"
+  default_request_payload_template:
+    prompt: "${user_message}"
+    details: "${request_payload}"  # Rest of the request payload data.
+  default_response_payload_template:
+    agent_reply: "${agent_reply}"
+    guardrail_flag: "${guardrail_flag}"
+    generated_metadata: "${generated_metadata}"
+
+reference_data:
+  source: LOCAL  # LOCAL or REMOTE.
+  path: "../../src/data/conversation_example_1.json"
+
+metrics_map:
+  field_1: EXACT
+  field_2: LEVENSHTEIN
+```
+
+- **Endpoint Configuration**: Define how to interact with your LLM-based system (base URL, auth, payload templates).
+- **Placeholders**: For the request payload, change the field names (e.g., 'prompt' to 'message') according to your API specs. For the response payload, change the place holders values (e.g., `${agent_reply}` to `${generated_reply}`).
+- **Secrets**: Store API keys in `.env` and load via `python-dotenv` (e.g., `API_KEY=your_key_here`).
+
+For conversation scripts (used in Simulator), provide a JSON file with this schema:
+
+```json
+{
+  "id": "1fa6f6ed-3cfe-4c0b-b389-7292f58879d4",
+  "scripts": [
+    {
+      "id": "65f58cec-d55d-4a24-bf16-fa8327a3aa6b",
+      "interactions": [
+        {
+          "id": "e99a2898-6a79-4a20-ac85-dfe977ea9935",
+          "user_message": "Hello, I would like to book an appointment with a doctor.",
+          "reference_reply": "Sure, I can help with that. Could you please specify the type of doctor you need to see?",
+          "interaction_type": "initial",
+          "reference_metadata": {},
+          "generated_metadata": {},
+          "guardrail_flag": false,
+          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+        },
+        {
+          "id": "fe5c539a-d0a1-40ee-97bd-dbe456703ccc",
+          "user_message": "I need to see a cardiologist.",
+          "reference_reply": "When would you like to schedule your appointment?",
+          "interaction_type": "intermediate",
+          "reference_metadata": {},
+          "generated_metadata": {},
+          "guardrail_flag": false,
+          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+        },
+        {
+          "id": "2cfdbd1c-a065-48bb-9aa9-b958342154b1",
+          "user_message": "I would like to book it for next Monday morning.",
+          "reference_reply": "We have an available slot at 10 AM next Monday. Does that work for you?",
+          "interaction_type": "intermediate",
+          "reference_metadata": {
+            "appointment_type": "Cardiology",
+            "date": "next Monday",
+            "time": "10 AM"
+          },
+          "generated_metadata": {
+            "appointment_type": "Cardiology",
+            "date": "next Monday",
+            "time": "morning"
+          },
+          "guardrail_flag": false,
+          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+        },
+        {
+          "id": "f4f2dd35-71d7-4b75-ba2b-93a4f546004a",
+          "user_message": "Yes, please book it for 10 AM then.",
+          "reference_reply": "Your appointment with the cardiologist is booked for 10 AM next Monday. Is there anything else I can help you with?",
+          "interaction_type": "final",
+          "reference_metadata": {},
+          "generated_metadata": {},
+          "guardrail_flag": false,
+          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+        }
+      ],
+      "description": "A conversation about booking a doctor appointment.",
+      "details": {
+        "context": "Booking a doctor appointment"
+      }
+    }
+  ]
+}
+```
+
+- **Fields**: Include user messages, reference/references replies, metadata for comparison, guardrail flags, and request payloads.
+
+## Usage Example
+
+To run an evaluation:
+
+1. Prepare your YAML config and JSON data files.
+2. Use the following Python script:
+
+```python
+if __name__ == "__main__":
+    from levelapp.workflow.schemas import WorkflowConfig
+    from levelapp.core.session import EvaluationSession
+
+    # Load configuration from YAML
+    config = WorkflowConfig.load(path="../data/workflow_config.yaml")
+
+    # Run evaluation session
+    with EvaluationSession(session_name="sim-test", workflow_config=config) as session:
+        session.run()
+        results = session.workflow.collect_results()
+        print("Results:", results)
+
+    stats = session.get_stats()
+    print(f"session stats:\n{stats}")
+```
+
+- This loads the config, runs the specified workflow (e.g., Simulator), collects results, and prints stats.
+
+For more examples, see the `examples/` directory.
+
+## Documentation
+
+Detailed docs are in the `docs/` directory, including API references and advanced configuration.
+
+## Contributing
+
+Contributions are welcome! Please follow these steps:
+- Fork the repository on GitHub.
+- Create a feature branch (`git checkout -b feature/new-feature`).
+- Commit changes (`git commit -am 'Add new feature'`).
+- Push to the branch (`git push origin feature/new-feature`).
+- Open a pull request.
+
+Report issues via GitHub Issues. Follow the code of conduct (if applicable).
+
+## Acknowledgments
+
+- Powered by Norma.
+- Thanks to contributors and open-source libraries like Pydantic, NumPy, and OpenAI SDK.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENCE](LICENCE) file for details.
+
+---

levelapp-0.1.0.dist-info/RECORD

@@ -0,0 +1,46 @@
+levelapp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp/aspects/__init__.py,sha256=_OaPcjTWBizqcUdDVj5aYue7lG9ytjQGLhPvReriKnU,326
+levelapp/aspects/loader.py,sha256=xWpcWtS25zbVhZ0UnIJEcQA9klajKk10TLK4j1IStH0,9543
+levelapp/aspects/logger.py,sha256=MJ9HphyHYkTE5-ajA_WuMUTM0qQzd0WIP243vF-pj3M,1698
+levelapp/aspects/monitor.py,sha256=lXGgxxeKZNkL5XbC_0ybES75gaRA9SuspHLHAEDbMR8,21981
+levelapp/aspects/sanitizer.py,sha256=zUqgb76tXJ8UUYtHp0Rz7q9PZjAHpSpHPPFfGTjjQNg,5229
+levelapp/clients/__init__.py,sha256=bbSFkYIODxEFatET4CFsKGC4m7Ng_tUkTGNcVBMa9h0,4169
+levelapp/clients/anthropic.py,sha256=Bxp-HffcIPLwM9BLcTR7n-D8ZXYVWCmbr2oH33fKV04,4030
+levelapp/clients/ionos.py,sha256=GFkLSeu8epFZV44GbNO3h1fRCKcfxscHMTFY0kPfh3o,4267
+levelapp/clients/mistral.py,sha256=e1NRvP9qN7O2zWAzBbgdQmmUDHQfCRLtVKDJCrh0DNA,3777
+levelapp/clients/openai.py,sha256=J9srwepIGMDasUyeUvL9ETxxyCyOyUsL01YOdEAyF8w,3656
+levelapp/comparator/__init__.py,sha256=ynmc0mrx-JbcCqLH-z4hOVezqGocDbDQGqgbhWy2xzI,187
+levelapp/comparator/comparator.py,sha256=yk0FWREnWKhIbXlsYpieqPJPqrlWXzyBMjVSznGqKY8,8183
+levelapp/comparator/extractor.py,sha256=vJ9iEoWAtXo2r9r7X72uUQPKW3UZE9Kx3uIjCufEp9k,3910
+levelapp/comparator/schemas.py,sha256=lUAQzEyStidt2ePQgV2zq-An5MLBrVSw6t8fB0FQKJs,1803
+levelapp/comparator/scorer.py,sha256=x5tU28SQekCNXmtK6wxr9ht7MZkqI78eYmZA6srkp4o,9167
+levelapp/comparator/utils.py,sha256=Eu48nDrNzFr0lwAJJS0aNhKsAWQ72syTEWYMNYfg764,4331
+levelapp/config/__init__.py,sha256=9oaajE5zW-OVWOszUzMAG6nHDSbLQWa3KT6bVoSvzRA,137
+levelapp/config/endpoint.py,sha256=ll34rZ0KRmUwI81EWJ3HX9i6pziq2YrQb84kv4ErymI,7649
+levelapp/config/prompts.py,sha256=crjOk01weLz5_IdF6dDZWPfSmiKNL8SgnbThyf4Jz2o,1345
+levelapp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp/core/base.py,sha256=oh4OkKgwGxmw_jgjX6wrBoK0KPc1JvCMZfbZP_mGmIg,12453
+levelapp/core/session.py,sha256=0pE8iUNyA6__RYGJlfaqSr0MtOT0vYIwr496KvC1VlA,7885
+levelapp/evaluator/__init__.py,sha256=K-P75Q1FXXLCNqH1wyhT9sf4y2R9a1qR5449AXEsY1k,109
+levelapp/evaluator/evaluator.py,sha256=JlqszyfOvA8JbgHmQNhMSBkminF95_AjXe14jg_OO7s,9908
+levelapp/metrics/__init__.py,sha256=1y4gDLOu2Jz4QVIgPH-v9YMgaWOFr263tYLUTiFJ-fc,1965
+levelapp/metrics/embedding.py,sha256=wvlT8Q5DjDT6GrAIFtc5aFbA_80hDLUXMP4RbSpSwHE,115
+levelapp/metrics/exact.py,sha256=Kb13nD2OVLrl3iYHaXrxDfrxDuhW0SMVvLAEXPaJtlY,6235
+levelapp/metrics/fuzzy.py,sha256=Rg8ashzMxtQwKO-z_LLzdj2PDIRqL4CBw6PGRf9IBrI,2598
+levelapp/metrics/token.py,sha256=yQi9hxT_fXTGjLiCCemDxQ4Uk2zD-wQYtSnDlI2AuuY,3521
+levelapp/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp/repository/__init__.py,sha256=hNmFRZ7kKJN1mMlOHeW9xf0j9Q7gqTXYJ3hMCzk9to4,79
+levelapp/repository/firestore.py,sha256=6HuBCgcBBg-E8QHr06SVDXKY0QBhsWPOhmO7Ps3BWMs,10060
+levelapp/simulator/__init__.py,sha256=8Dz8g7rbpBZX3WoknVmMVoWm_VT72ZL9BABOF1xFpqs,83
+levelapp/simulator/schemas.py,sha256=YD6yu3UDXNoApX3R9VpCnpe3sWF7Gs8TNNUgsnqTSXg,4026
+levelapp/simulator/simulator.py,sha256=GhoJYcl9GS9vfK58uLwPiI5_cFExFb_Xw_EjElesmvY,17120
+levelapp/simulator/utils.py,sha256=qx0JdV1ZDQdTRVKa9xfq278ASrE44GBXSnJZJuhICqo,7365
+levelapp/workflow/__init__.py,sha256=X73ulbq3Vk_Vsug1eU2m7LsBCmbR7ehQlwUVCh4Rbb0,142
+levelapp/workflow/base.py,sha256=4LFrJyGnuSUJSZ1oOVI_uF1cwEpbYeFFZXHPSh3aEGo,4289
+levelapp/workflow/factory.py,sha256=gW5wLv_kJfzrSTQeSLFoUunk87rw0NzW4wmJLIfcwX8,2228
+levelapp/workflow/registration.py,sha256=33tPCyzP-XeI1vefjA3PmVdBl7x4wtsVGg05Bh_j7os,363
+levelapp/workflow/schemas.py,sha256=Q4TmjqTz_tKzMPXD_fJ2hc7K3IVsn_U56BzHG94WwIA,4090
+levelapp-0.1.0.dist-info/METADATA,sha256=6XNgxGHG8oNc-yjg82ZAaGi_UVvrnOalIpUmS5viaW8,10242
+levelapp-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+levelapp-0.1.0.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp-0.1.0.dist-info/RECORD,,

levelapp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any