levelapp 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

levelapp/repository/firestore.py

@@ -1,7 +1,5 @@
 """levelapp/repository/firestore.py"""
-import google.auth
-
-from typing import List, Dict, Any, Type
+from typing import List, Dict, Any, Type, TYPE_CHECKING
 from pydantic import ValidationError
 
 from google.cloud import firestore_v1
@@ -13,15 +11,25 @@ from levelapp.core.base import BaseRepository, Model
 from levelapp.aspects import logger
 
 
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
+
 class FirestoreRepository(BaseRepository):
     """
     Firestore implementation of BaseRepository.
     (Uses hierarchical path: {user_id}/{collection_id}/{document_id}
     """
 
-    def __init__(self, project_id: str | Any = None, database_name: str | Any = '(default)'):
-        self.project_id = project_id
-        self.database_name = database_name
+    def __init__(self, config: "WorkflowConfig | None"):
+        if config:
+            self.config = config
+            self.project_id: str | Any = config.repository.project_id
+            self.database_name: str | Any = config.repository.database_name
+        else:
+            self.project_id: str | Any = None
+            self.database_name: str | Any = '(default)'
+
         self.client: firestore_v1.Client | None = None
 
     def connect(self) -> None:
@@ -29,6 +37,7 @@ class FirestoreRepository(BaseRepository):
         Connects to Firestore, prioritizing the project ID passed to the constructor.
         """
         try:
+            import google.auth
             credentials, default_project_id = google.auth.default()
 
             if not credentials:

levelapp/simulator/schemas.py

@@ -8,8 +8,8 @@ from enum import Enum
 from uuid import UUID, uuid4
 from datetime import datetime
 
-from typing import Optional, Dict, Any, List
-from pydantic import BaseModel, Field, computed_field
+from typing import Dict, Any, List
+from pydantic import BaseModel, Field, computed_field, field_validator
 
 from levelapp.evaluator.evaluator import JudgeEvaluationResults
 
@@ -25,11 +25,11 @@ class Interaction(BaseModel):
     """Represents a single interaction within a conversation."""
     id: UUID = Field(default_factory=uuid4, description="Interaction identifier")
     user_message: str = Field(..., description="The user's query message")
-    generated_reply: str = Field(..., description="The agent's reply message")
+    # generated_reply: str = Field(..., description="The agent's reply message")
     reference_reply: str = Field(..., description="The preset reference message")
-    interaction_type: InteractionLevel = Field(..., description="Type of interaction")
-    reference_metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Expected metadata")
-    generated_metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Extracted metadata")
+    interaction_type: InteractionLevel = Field(default=InteractionLevel.INITIAL, description="Type of interaction")
+    reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
+    # generated_metadata: Dict[str, Any] = Field(default_factory=dict, description="Extracted metadata")
     guardrail_flag: bool = Field(default=False, description="Flag for guardrail signaling")
     request_payload: Dict[str, Any] = Field(default_factory=dict, description="Additional request payload")
 
@@ -38,7 +38,7 @@ class ConversationScript(BaseModel):
     """Represents a basic conversation with multiple interactions."""
     id: UUID = Field(default_factory=uuid4, description="Conversation identifier")
     interactions: List[Interaction] = Field(default_factory=list, description="List of interactions")
-    description: str = Field(..., description="A short description of the conversation")
+    description: str = Field(default="no-description", description="A short description of the conversation")
     details: Dict[str, str] = Field(default_factory=dict, description="Conversation details")
 
 
@@ -58,32 +58,26 @@ class InteractionResults(BaseModel):
 
 class InteractionEvaluationResults(BaseModel):
     """Model representing the evaluation result of an interaction."""
-    judge_evaluations: Dict[str, JudgeEvaluationResults] = Field(default_factory=dict)
-    metadata_evaluation: Dict[str, float] = Field(default_factory=dict)
+    judge_evaluations: Dict[str, JudgeEvaluationResults] | None = Field(default_factory=dict)
+    metadata_evaluation: Dict[str, float] | None = Field(default_factory=dict)
     guardrail_flag: int = Field(default=0)
 
 
 class SimulationResults(BaseModel):
-    # Initial data
-    project_id: str = Field(default_factory=uuid4, description="Project identifier")
-    user_id: str = Field(default_factory=uuid4, description="User identifier")
-    batch_id: str = Field(default_factory=uuid4, description="Batch identifier")
     # Collected data
     started_at: datetime = datetime.now()
     finished_at: datetime
     # Collected Results
     evaluation_summary: Dict[str, Any] | None = Field(default_factory=dict, description="Evaluation result")
     average_scores: Dict[str, Any] | None = Field(default_factory=dict, description="Average scores")
+    interaction_results: List[Dict[str, Any]] | None = Field(default_factory=list, description="detailed results")
+
+    @computed_field
+    @property
+    def batch_id(self) -> str:
+        return str(uuid4())
 
     @computed_field
     @property
     def elapsed_time(self) -> float:
         return (self.finished_at - self.started_at).total_seconds()
-
-
-class TestResults(BaseModel):
-    api_host: str = Field(..., alias="apiHost")
-    ionos_model_name: str = Field(..., alias="ionosModelName")
-    test_name: str = Field(..., alias="testName")
-    test_type: str = Field(..., alias="testType")
-    batch_details: Optional[SimulationResults] = Field(..., alias="results")

levelapp/simulator/simulator.py

@@ -1,6 +1,7 @@
 """
 'simulators/service.py': Service layer to manage conversation simulation and evaluation.
 """
+import json
 import time
 import asyncio
 
@@ -23,7 +24,7 @@ from levelapp.simulator.utils import (
     summarize_verdicts,
 )
 from levelapp.aspects import logger
-from levelapp.workflow.schemas import EvaluatorType
+from levelapp.core.schemas import EvaluatorType
 
 
 class ConversationSimulator(BaseProcess):
@@ -33,6 +34,7 @@ class ConversationSimulator(BaseProcess):
         self,
         repository: BaseRepository | None = None,
         evaluators: Dict[EvaluatorType, BaseEvaluator] | None = None,
+        providers: List[str] | None = None,
         endpoint_config: EndpointConfig | None = None,
     ):
         """
@@ -47,6 +49,7 @@ class ConversationSimulator(BaseProcess):
 
         self.repository = repository
         self.evaluators = evaluators
+        self.providers = providers
         self.endpoint_config = endpoint_config
 
         self._url: str | None = None
@@ -60,7 +63,8 @@ class ConversationSimulator(BaseProcess):
     def setup(
             self,
             repository: BaseRepository,
-            evaluators: Dict[str, BaseEvaluator],
+            evaluators: Dict[EvaluatorType, BaseEvaluator],
+            providers: List[str],
             endpoint_config: EndpointConfig,
     ) -> None:
         """
@@ -69,6 +73,7 @@ class ConversationSimulator(BaseProcess):
         Args:
             repository (BaseRepository): Repository object for storing simulation results.
             evaluators (Dict[str, BaseEvaluator]): List of evaluator objects for evaluating interactions.
+            providers (List[str]): List of LLM provider names.
             endpoint_config (EndpointConfig): Configuration object for VLA.
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.setup.__name__}]"
@@ -76,6 +81,11 @@ class ConversationSimulator(BaseProcess):
 
         self.repository = repository
         self.evaluators = evaluators
+        self.providers = providers
+
+        if not self.providers:
+            logger.warning(f"{_LOG} No LLM providers were provided. The Judge Evaluation process will not be executed.")
+
         self.endpoint_config = endpoint_config
 
         self._url = endpoint_config.full_url
@@ -83,17 +93,27 @@ class ConversationSimulator(BaseProcess):
         self._headers = endpoint_config.headers
 
     def get_evaluator(self, name: EvaluatorType) -> BaseEvaluator:
+        """
+        Retrieve an evaluator by name.
+
+        Args:
+            name (EvaluatorType): Name of evaluator.
+
+        Returns:
+            An evaluator object.
+        """
         _LOG: str = f"[{self._CLASS_NAME}][{self.get_evaluator.__name__}]"
 
         if name not in self.evaluators:
             raise KeyError(f"{_LOG} Evaluator {name} not registered.")
+
         return self.evaluators[name]
 
     async def run(
         self,
         test_batch: ScriptsBatch,
         attempts: int = 1,
-    ) -> Dict[str, Any]:
+    ) -> Any:
         """
         Run a batch test for the given batch name and details.
 
@@ -119,9 +139,10 @@ class ConversationSimulator(BaseProcess):
             finished_at=finished_at,
             evaluation_summary=self.verdict_summaries,
             average_scores=results.get("average_scores", {}),
+            interaction_results=results.get("results")
         )
 
-        return {"results": results, "status": "COMPLETE"}
+        return results.model_dump_json(indent=2)
 
     async def simulate_conversation(self, attempts: int = 1) -> Dict[str, Any]:
         """
@@ -161,10 +182,11 @@ class ConversationSimulator(BaseProcess):
                 verdicts=verdicts, judge=judge
             )
 
-        return {"scripts": results, "average_scores": overall_average_scores}
+        return {"results": results, "average_scores": overall_average_scores}
 
     async def simulate_single_scenario(
-        self, script: ConversationScript, attempts: int = 1
+        self, script: ConversationScript,
+            attempts: int = 1
     ) -> Dict[str, Any]:
         """
         Simulate a single scenario with the given number of attempts, concurrently.
@@ -183,19 +205,18 @@ class ConversationSimulator(BaseProcess):
         all_attempts_verdicts: Dict[str, List[str]] = defaultdict(list)
 
         async def simulate_attempt(attempt_number: int) -> Dict[str, Any]:
-            logger.info(f"{_LOG} Running attempt: {attempt_number + 1}/{attempts}")
+            logger.info(f"{_LOG} Running attempt: {attempt_number + 1}/{attempts}\n---")
             start_time = time.time()
 
             collected_scores: Dict[str, List[Any]] = defaultdict(list)
             collected_verdicts: Dict[str, List[str]] = defaultdict(list)
 
-            initial_interaction_results = await self.simulate_interactions(
+            interaction_results = await self.simulate_interactions(
                 script=script,
                 evaluation_verdicts=collected_verdicts,
                 collected_scores=collected_scores,
             )
 
-            logger.info(f"{_LOG} collected_scores: {collected_scores}\n---")
             single_attempt_scores = calculate_average_scores(collected_scores)
 
             for target, scores in single_attempt_scores.items():
@@ -215,7 +236,7 @@ class ConversationSimulator(BaseProcess):
                 "attempt": attempt_number + 1,
                 "script_id": script.id,
                 "total_duration": elapsed_time,
-                "interaction_results": initial_interaction_results,
+                "interaction_results": interaction_results,
                 "evaluation_verdicts": collected_verdicts,
                 "average_scores": single_attempt_scores,
             }
@@ -228,10 +249,6 @@ class ConversationSimulator(BaseProcess):
         for judge_, verdicts_ in all_attempts_verdicts.items():
             self.evaluation_verdicts[judge_].extend(verdicts_)
 
-        logger.info(
-            f"{_LOG} average scores:\n{average_scores}\n---"
-        )
-
         return {
             "script_id": script.id,
             "attempts": attempt_results,
@@ -314,8 +331,6 @@ class ConversationSimulator(BaseProcess):
                 reference_guardrail=reference_guardrail_flag,
             )
 
-            logger.info(f"{_LOG} Evaluation results:\n{evaluation_results.model_dump()}\n")
-
             self.store_evaluation_results(
                 results=evaluation_results,
                 evaluation_verdicts=evaluation_verdicts,
@@ -323,9 +338,7 @@ class ConversationSimulator(BaseProcess):
             )
 
             elapsed_time = time.time() - start_time
-            logger.info(
-                f"{_LOG} Interaction simulation complete in {elapsed_time:.2f} seconds.\n---"
-            )
+            logger.info(f"{_LOG} Interaction simulation complete in {elapsed_time:.2f} seconds.\n---")
 
             result = {
                 "user_message": user_message,
@@ -368,50 +381,106 @@ class ConversationSimulator(BaseProcess):
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.evaluate_interaction.__name__}]"
 
-        judge_evaluator = self.evaluators.get(EvaluatorType.JUDGE)
-        metadata_evaluator = self.evaluators.get(EvaluatorType.REFERENCE)
+        judge_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.JUDGE, None)
+        metadata_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.REFERENCE, None)
 
-        if not judge_evaluator:
-            raise ValueError(f"{_LOG} No Judge Evaluator found.")
+        evaluation_results = InteractionEvaluationResults()
 
-        openai_eval_task = judge_evaluator.async_evaluate(
-            generated_data=generated_reply,
-            reference_data=reference_reply,
-            user_input=user_input,
-            provider="openai"
-        )
+        if judge_evaluator and self.providers:
+            await self._judge_evaluation(
+                user_input=user_input,
+                generated_reply=generated_reply,
+                reference_reply=reference_reply,
+                providers=self.providers,
+                judge_evaluator=judge_evaluator,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"[{_LOG}] Judge evaluation skipped (no evaluator or no providers).")
 
-        ionos_eval_task = judge_evaluator.async_evaluate(
-            provider="ionos",
-            user_input=user_input,
-            generated_data=generated_reply,
-            reference_data=reference_reply,
-        )
+        if metadata_evaluator and reference_metadata:
+            self._metadata_evaluation(
+                metadata_evaluator=metadata_evaluator,
+                generated_metadata=generated_metadata,
+                reference_metadata=reference_metadata,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"[{_LOG}] Metadata evaluation skipped (no evaluator or no reference metadata).")
 
-        openai_judge_evaluation, ionos_judge_evaluation = await asyncio.gather(
-            openai_eval_task, ionos_eval_task
-        )
+        evaluation_results.guardrail_flag = 1 if generated_guardrail == reference_guardrail else 0
 
-        if not metadata_evaluator:
-            raise ValueError(f"{_LOG} No Metadata Evaluator found.")
+        return evaluation_results
 
-        metadata_evaluation = {}
-        if reference_metadata:
-            metadata_evaluation = metadata_evaluator.evaluate(
-                generated_data=generated_metadata,
-                reference_data=reference_metadata,
+    async def _judge_evaluation(
+            self,
+            user_input: str,
+            generated_reply: str,
+            reference_reply: str,
+            providers: List[str],
+            judge_evaluator: BaseEvaluator,
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run LLM-as-a-judge evaluation using multiple providers (async).
+
+        Args:
+            user_input (str): The user input message.
+            generated_reply (str): The generated agent reply.
+            reference_reply (str): The reference agent reply.
+            providers (List[str]): List of judge provider names.
+            judge_evaluator (BaseEvaluator): Evaluator instance.
+            evaluation_results (InteractionEvaluationResults): Results container (Pydantic model).
+
+        Returns:
+            None
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][judge_evaluation]"
+
+        tasks = {
+            provider: judge_evaluator.async_evaluate(
+                generated_data=generated_reply,
+                reference_data=reference_reply,
+                user_input=user_input,
+                provider=provider,
             )
+            for provider in providers
+        }
 
-        guardrail_flag = 1 if generated_guardrail == reference_guardrail else 0
+        results = await asyncio.gather(*tasks.values(), return_exceptions=True)
 
-        return InteractionEvaluationResults(
-            judge_evaluations={
-                openai_judge_evaluation.provider: openai_judge_evaluation,
-                ionos_judge_evaluation.provider: ionos_judge_evaluation
-            },
-            metadata_evaluation=metadata_evaluation,
-            guardrail_flag=guardrail_flag,
-        )
+        for provider, result in zip(tasks.keys(), results):
+            if isinstance(result, Exception):
+                logger.error(f"{_LOG} Provider '{provider}' failed to perform Judge Evaluation.")
+                continue
+
+            evaluation_results.judge_evaluations[provider] = result
+
+    def _metadata_evaluation(
+            self,
+            metadata_evaluator: BaseEvaluator,
+            generated_metadata: Dict[str, Any],
+            reference_metadata: Dict[str, Any],
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run metadata evaluation using the provided evaluator.
+
+        Args:
+            metadata_evaluator (BaseEvaluator): Evaluator for metadata comparison.
+            generated_metadata (Dict[str, Any]): The generated metadata.
+            reference_metadata (Dict[str, Any]): The reference metadata.
+            evaluation_results (InteractionEvaluationResults): Results container.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][metadata_evaluation]"
+
+        try:
+            evaluation_results.metadata_evaluation = metadata_evaluator.evaluate(
+                generated_data=generated_metadata,
+                reference_data=reference_metadata,
+            )
+        except Exception as e:
+            logger.error(f"[{_LOG}] Metadata evaluation failed:\n{e}", exc_info=e)
 
     @staticmethod
     def store_evaluation_results(
@@ -428,7 +497,7 @@ class ConversationSimulator(BaseProcess):
             collected_scores (Dict[str, List[Any]]): The collected scores.
         """
         for provider in results.judge_evaluations.keys():
-            evaluation_verdicts[f"{provider}_verdicts_summary"].append(
+            evaluation_verdicts[f"{provider}"].append(
                 results.judge_evaluations.get(provider, "").justification
             )
 

levelapp/simulator/utils.py

@@ -1,17 +1,18 @@
 """
 'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
 """
+import ast
 import json
-
 import httpx
-import arrow
 
 from uuid import UUID
-from typing import Dict, Any, Optional, List, Union
+from string import Template
+from typing import Any, Dict, List, Union
 
-from openai import OpenAI
 from pydantic import ValidationError
 
+from levelapp.clients import ClientRegistry
+from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
 from levelapp.simulator.schemas import InteractionResults
 from levelapp.aspects import MonitoringAspect, MetricType, logger
 
@@ -48,7 +49,14 @@ def extract_interaction_details(
             missing_keys = required_keys - response_dict.keys()
             logger.warning(f"[extract_interaction_details] Missing data: {missing_keys}]")
 
-        return InteractionResults.model_validate(response_dict)
+        output = {}
+        for k, v in template.items():
+            output[k] = Template(v).safe_substitute(response_dict)
+
+        raw_value = output.get("generated_metadata", {})
+        output["generated_metadata"] = ast.literal_eval(raw_value) if isinstance(raw_value, str) else raw_value
+
+        return InteractionResults.model_validate(output)
 
     except json.JSONDecodeError as e:
         logger.error(f"[extract_interaction_details] Failed to extract details:\n{e}")
@@ -64,7 +72,7 @@ async def async_interaction_request(
         url: str,
         headers: Dict[str, str],
         payload: Dict[str, Any],
-) -> Optional[httpx.Response]:
+) -> httpx.Response | None:
     """
     Perform an asynchronous interaction request.
 
@@ -74,7 +82,7 @@ async def async_interaction_request(
         payload (Dict[str, Any]): The payload to send in the request.
 
     Returns:
-        Optional[httpx.Response]: The response from the interaction request, or None if an error occurred.
+        httpx.Response: The response from the interaction request, or None if an error occurred.
     """
     try:
         async with httpx.AsyncClient(timeout=180) as client:
@@ -92,42 +100,6 @@ async def async_interaction_request(
     return None
 
 
-def parse_date_value(raw_date_value: Optional[str], default_date_value: Optional[str] = "") -> str:
-    """
-    Cleans and parses a dehumanized relative date string to ISO format.
-
-    Args:
-        raw_date_value (Optional[str]): The raw date value to parse.
-        default_date_value (Optional[str]): The default value to return if parsing fails. Defaults to an empty string.
-
-    Returns:
-        str: The parsed date in ISO format, or the default value if parsing fails.
-    """
-    if not raw_date_value:
-        logger.info(f"[parse_date_value] No raw value provided. returning default: '{default_date_value}'")
-        return default_date_value
-
-    clean = raw_date_value.replace("{{", "").replace("}}", "").replace("_", " ").strip()
-    clean += 's' if not clean.endswith('s') else clean
-
-    try:
-        arw = arrow.utcnow()
-        parsed_date = arw.dehumanize(clean).utcnow().format('YYYY-MM-DD')
-        return parsed_date
-
-    except arrow.parser.ParserError as e:
-        logger.error(f"[parse_date_value] Failed to parse date: '{clean}'\nParserError: {str(e)}", exc_info=True)
-        return default_date_value
-
-    except ValueError as e:
-        logger.error(f"[parse_date_value] Invalid date value: '{clean}'\nValueError: {str(e)}", exc_info=True)
-        return default_date_value
-
-    except Exception as e:
-        logger.error(f"[parse_date_value] Unexpected error.\nException: {str(e)}", exc_info=True)
-        return default_date_value
-
-
 @MonitoringAspect.monitor(
     name="average_calc",
     category=MetricType.SCORING,
@@ -157,45 +129,35 @@ def calculate_average_scores(scores: Dict[str, Union[List[float], float]]) -> Di
 
 
 @MonitoringAspect.monitor(name="summarization", category=MetricType.API_CALL)
-def summarize_verdicts(verdicts: List[str], judge: str, max_bullets: int = 5) -> List[str]:
-    """
-    Summarize the justifications for each judge.
-
-    Args:
-        verdicts (List[str]): A list of justifications.
-        judge (str): The judge or evaluator (provider) name for context.
-        max_bullets (int): The maximum number of bullets allowed per judge.
-
-    Returns:
-        List[str]: The summarized justifications.
-    """
-    if not verdicts:
-        return []
-
-    prompt = f"""
-    You are reviewing evaluation justifications from LL judges about replies generated by a virtual leasing agent.\n
-    Each justification contains the judge's assessment of how well the agent's response matched the expected reply.\n
-    Your task is to identify and summarize only the **negative points**, such as errors, misunderstandings,
-    missing information, or failure to meet expectations.\n
-    Return up to {max_bullets} bullet points. Be concise and start each point with '- '\n\n
-    ---
-    - Judge: {judge}
-    - Justifications:\n{chr(10).join(verdicts)}\n
-    """
-
-    client = OpenAI()
+def summarize_verdicts(
+        verdicts: List[str],
+        judge: str,
+        max_bullets: int = 5
+) -> List[str]:
+    client_registry = ClientRegistry()
+    client = client_registry.get(provider=judge)
 
     try:
-        result = client.chat.completions.create(
-            model="gpt-4o-mini",
-            temperature=0,
-            messages=[{"role": "user", "content": prompt}]
-        ).choices[0].message.content
+        verdicts = chr(10).join(verdicts)
+        prompt = SUMMARIZATION_PROMPT_TEMPLATE.format(max_bullets=max_bullets, judge=judge, verdicts=verdicts)
+        response = client.call(message=prompt)
+        parsed = client.parse_response(response=response)
+        striped = parsed.get("output", "").strip("")
+        bullet_points = [point.strip() for point in striped.split("- ") if point.strip()]
 
-        bullet_points = [point.strip() for point in result.split('- ') if point.strip()]
-
-        return bullet_points
+        return bullet_points[:max_bullets]
 
     except Exception as e:
         logger.error(f"[summarize_justifications] Error during summarization: {str(e)}", exc_info=True)
         return []
+
+
+# if __name__ == '__main__':
+#     template = {'generated_reply': '${agent_reply}', 'generated_metadata': '${generated_metadata}'}
+#     response_dict = {
+#         'agent_reply': "I'd be happy to help you book something for 10 AM.",
+#         'generated_metadata': {'appointment_type': 'Cardiology', 'date': 'next Monday', 'time': '10 AM'}
+#     }
+#
+#     result = extract_interaction_details(response_dict, template)
+#     print(f"result: {result.model_dump()}")

levelapp/workflow/__init__.py

@@ -1,5 +1,6 @@
 from . import registration
-from .schemas import WorkflowType
+from .base import BaseWorkflow
+from .config import WorkflowConfig
 from .factory import MainFactory
 
-__all__ = ["WorkflowType", "MainFactory"]
+__all__ = ["BaseWorkflow", "WorkflowConfig", "MainFactory"]