evalvault 1.67.0__py3-none-any.whl → 1.68.1__py3-none-any.whl

evalvault/adapters/outbound/dataset/base.py

@@ -3,6 +3,7 @@
 import json
 from abc import ABC, abstractmethod
 from pathlib import Path
+from typing import Any
 
 from evalvault.domain.entities.dataset import Dataset
 
@@ -118,6 +119,45 @@ class BaseDatasetLoader(ABC):
         # Fall back to pipe-separated format
         return [ctx.strip() for ctx in contexts_str.split("|")]
 
+    def _parse_metadata_cell(self, raw: Any) -> dict[str, Any]:
+        if raw is None or (isinstance(raw, float) and str(raw) == "nan"):
+            return {}
+        text = str(raw).strip()
+        if not text:
+            return {}
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise ValueError("Invalid metadata JSON") from exc
+        if not isinstance(parsed, dict):
+            raise ValueError("metadata must be a JSON object")
+        return parsed
+
+    def _parse_summary_tags_cell(self, raw: Any) -> list[str]:
+        if raw is None or (isinstance(raw, float) and str(raw) == "nan"):
+            return []
+        if isinstance(raw, list):
+            return [str(item).strip().lower() for item in raw if str(item).strip()]
+        text = str(raw).strip()
+        if not text:
+            return []
+        if text.startswith("["):
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return [str(item).strip().lower() for item in parsed if str(item).strip()]
+        delimiter = "," if "," in text else "|" if "|" in text else None
+        parts = text.split(delimiter) if delimiter else [text]
+        return [part.strip().lower() for part in parts if part.strip()]
+
+    def _parse_summary_intent_cell(self, raw: Any) -> str | None:
+        if raw is None or (isinstance(raw, float) and str(raw) == "nan"):
+            return None
+        text = str(raw).strip()
+        return text or None
+
     def _get_default_name(self, file_path: Path) -> str:
         """Get default dataset name from file path.
 

evalvault/adapters/outbound/dataset/csv_loader.py

@@ -123,12 +123,28 @@ class CSVDatasetLoader(BaseDatasetLoader):
                 else None
             )
 
+            metadata = {}
+            if "metadata" in df.columns:
+                try:
+                    metadata = self._parse_metadata_cell(row["metadata"])
+                except ValueError as exc:
+                    raise ValueError(f"Test case {row['id']}: {exc}") from exc
+            if "summary_tags" in df.columns:
+                tags = self._parse_summary_tags_cell(row["summary_tags"])
+                if tags:
+                    metadata["summary_tags"] = tags
+            if "summary_intent" in df.columns:
+                intent = self._parse_summary_intent_cell(row["summary_intent"])
+                if intent:
+                    metadata["summary_intent"] = intent
+
             test_case = TestCase(
                 id=str(row["id"]),
                 question=str(row["question"]),
                 answer=str(row["answer"]),
                 contexts=contexts,
                 ground_truth=ground_truth,
+                metadata=metadata,
             )
             test_cases.append(test_case)
 

evalvault/adapters/outbound/dataset/excel_loader.py

@@ -96,12 +96,28 @@ class ExcelDatasetLoader(BaseDatasetLoader):
                 else None
             )
 
+            metadata = {}
+            if "metadata" in df.columns:
+                try:
+                    metadata = self._parse_metadata_cell(row["metadata"])
+                except ValueError as exc:
+                    raise ValueError(f"Test case {row['id']}: {exc}") from exc
+            if "summary_tags" in df.columns:
+                tags = self._parse_summary_tags_cell(row["summary_tags"])
+                if tags:
+                    metadata["summary_tags"] = tags
+            if "summary_intent" in df.columns:
+                intent = self._parse_summary_intent_cell(row["summary_intent"])
+                if intent:
+                    metadata["summary_intent"] = intent
+
             test_case = TestCase(
                 id=str(row["id"]),
                 question=str(row["question"]),
                 answer=str(row["answer"]),
                 contexts=contexts,
                 ground_truth=ground_truth,
+                metadata=metadata,
             )
             test_cases.append(test_case)
 

evalvault/config/settings.py

@@ -424,9 +424,13 @@ def apply_profile(settings: Settings, profile_name: str) -> Settings:
     """
     from evalvault.config.model_config import get_model_config
 
+    normalized = profile_name.strip() if isinstance(profile_name, str) else profile_name
+    if not normalized:
+        return settings
+
     try:
         model_config = get_model_config()
-        profile = model_config.get_profile(profile_name)
+        profile = model_config.get_profile(normalized)
 
         # LLM 설정 적용 (모델명과 provider만)
         settings.llm_provider = profile.llm.provider
@@ -449,9 +453,16 @@ def apply_profile(settings: Settings, profile_name: str) -> Settings:
         elif profile.embedding.provider == "vllm":
             settings.vllm_embedding_model = profile.embedding.model
 
-    except FileNotFoundError:
-        # 설정 파일이 없으면 프로필 무시
-        pass
+    except FileNotFoundError as exc:
+        raise ValueError(
+            "Model profile config not found. Create 'config/models.yaml' or 'evalvault.yaml' "
+            f"to use profile '{normalized}'."
+        ) from exc
+    except KeyError as exc:
+        available = ", ".join(sorted(model_config.profiles.keys()))
+        raise ValueError(
+            f"Unknown profile '{normalized}'. Available profiles: {available}"
+        ) from exc
 
     return settings
 

evalvault/domain/entities/stage.py

@@ -60,18 +60,16 @@ class StageEvent:
 
     @classmethod
     def from_dict(cls, payload: dict[str, Any]) -> StageEvent:
-        if "run_id" not in payload:
-            raise ValueError("StageEvent requires 'run_id'")
-        if "stage_type" not in payload:
-            raise ValueError("StageEvent requires 'stage_type'")
+        run_id = _require_str(payload, "run_id")
+        stage_type = _normalize_stage_type(payload)
 
         trace_payload = payload.get("trace") or {}
         input_ref = _parse_payload_ref(payload.get("input_ref"))
         output_ref = _parse_payload_ref(payload.get("output_ref"))
 
         return cls(
-            run_id=str(payload["run_id"]),
-            stage_type=str(payload["stage_type"]),
+            run_id=run_id,
+            stage_type=stage_type,
             stage_id=str(payload.get("stage_id") or uuid4()),
             stage_name=_optional_str(payload.get("stage_name")),
             parent_stage_id=_optional_str(payload.get("parent_stage_id")),
@@ -187,6 +185,24 @@ def _parse_datetime(value: Any) -> datetime | None:
     raise ValueError("Invalid datetime value")
 
 
+def _require_str(payload: dict[str, Any], key: str) -> str:
+    if key not in payload:
+        raise ValueError(f"StageEvent requires '{key}'")
+    value = str(payload.get(key, "")).strip()
+    if not value:
+        raise ValueError(f"StageEvent requires non-empty '{key}'")
+    return value
+
+
+def _normalize_stage_type(payload: dict[str, Any]) -> str:
+    if "stage_type" not in payload:
+        raise ValueError("StageEvent requires 'stage_type'")
+    value = str(payload.get("stage_type", "")).strip()
+    if not value:
+        raise ValueError("StageEvent requires non-empty 'stage_type'")
+    return value.lower()
+
+
 @overload
 def _ensure_dict(value: None, *, allow_none: Literal[True]) -> None: ...
 

{evalvault-1.67.0.dist-info → evalvault-1.68.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.67.0
+Version: 1.68.1
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.67.0.dist-info → evalvault-1.68.1.dist-info}/RECORD

@@ -118,9 +118,9 @@ evalvault/adapters/outbound/cache/__init__.py,sha256=LcsKzxnx1AnAwS07iSCdws11CfE
 evalvault/adapters/outbound/cache/hybrid_cache.py,sha256=AVhctQVOIbQWwvn_K0kxSq3lkhucuM7tezmSkPDbCrA,12711
 evalvault/adapters/outbound/cache/memory_cache.py,sha256=jvjIgXp7YRj08_AzBFaJ58jjXNzUlYbG_zX6fQJP4C0,3533
 evalvault/adapters/outbound/dataset/__init__.py,sha256=SDFnjmieEgz0uH5MpdXx8pmjnIMjRLkMFmFioMxCju0,1183
-evalvault/adapters/outbound/dataset/base.py,sha256=9ExRuKa3ZHdkeyK2SiBrWxZFA7DGo98ox9FVKizPFA8,3923
-evalvault/adapters/outbound/dataset/csv_loader.py,sha256=Zb-FhWxyauvcdkxLhFJWSAgSjkaKaMFbkATg-W2OGzc,4870
-evalvault/adapters/outbound/dataset/excel_loader.py,sha256=Am6T23MQVLs_HoQ_r2T-x7j5chFNXtP1opTuaodFanY,3769
+evalvault/adapters/outbound/dataset/base.py,sha256=4rxpQgxpFty0G5XRv1SP-XJ9mpZ9YO6PAMDgp71JiJQ,5547
+evalvault/adapters/outbound/dataset/csv_loader.py,sha256=xHg2QadMvLfHTHzeex6WxXmagLJog3LN-ui6dFxD8HY,5595
+evalvault/adapters/outbound/dataset/excel_loader.py,sha256=MUl-63r1s1GjVVmDgdag1DpMJvIVX_agGx20NQzEZN8,4494
 evalvault/adapters/outbound/dataset/json_loader.py,sha256=4wG7APg1LLADPxJ-wQZo2zBcvVX12sqo9VUIb-0Kww4,4923
 evalvault/adapters/outbound/dataset/loader_factory.py,sha256=32sjGuW2Yta12lpKy4DLH4I5B4Pi-YuHTvGG1Pr4VAk,1361
 evalvault/adapters/outbound/dataset/method_input_loader.py,sha256=d7pB4OPvvr-q-Y5DlvjX3X719jCCQ2vRDfT_ov0dUFU,3833
@@ -207,7 +207,7 @@ evalvault/config/langfuse_support.py,sha256=DEzVMfMGGf1V45W_2oUG-NCDfsYI4UUdnYJI
 evalvault/config/model_config.py,sha256=KlzDbGyDLeOGE7ElekFFk5YjjT5u8i6KO2B4EyZkLnI,3542
 evalvault/config/phoenix_support.py,sha256=e6RPWd6Qb7KU6Q8pLaYTpJGWULtvEEU6B0xHWyVyOH0,13604
 evalvault/config/secret_manager.py,sha256=YjPMuNqeBrAR2BzCJvsBNUExaU4TBSFyZ8kVYZZifqA,4172
-evalvault/config/settings.py,sha256=xvoNma4CHAd8R_nF0DL4MUWXBWCR5M0C68NPSPLT5JQ,18285
+evalvault/config/settings.py,sha256=DY170XUoMo8yQx8_CJjPt96QsGg7tyTx5wJ-ptcfdY0,18766
 evalvault/config/playbooks/improvement_playbook.yaml,sha256=9F9WVVCydFfz6zUuGYzZ4PKdW1LLtcBKVF36T7xT764,26965
 evalvault/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/domain/entities/__init__.py,sha256=wszRJ1Imdc5NJ1bQPC2udk-mAgFdlw4uZV5IPNjLpHQ,3669
@@ -228,7 +228,7 @@ evalvault/domain/entities/prompt.py,sha256=lQlRnHEKY69GWTC-cUIu0DMuPfJ9UWm6Sm4KT
 evalvault/domain/entities/prompt_suggestion.py,sha256=Ep_XSjdYUj7pFSCMyeeZKs8yTnp74AVx05Zqr7829PE,1243
 evalvault/domain/entities/rag_trace.py,sha256=sZgnkG4fK6KOe3Np6TYAZ_tPnsRbOmucDSQns35U1n4,11868
 evalvault/domain/entities/result.py,sha256=OaGHMDLWMW2O4fNVuVTUvWFVBQ1iu93OD_oI3NumrCQ,10697
-evalvault/domain/entities/stage.py,sha256=UqS59sjoMs_bhMupNtvagbIx8QgHgFjWoRPhJ3uJP2s,7426
+evalvault/domain/entities/stage.py,sha256=KyR-v3tyusPJ7pfTXtHE2_23tVvNSRU9Q1RT-R5akXg,7914
 evalvault/domain/metrics/__init__.py,sha256=Ros3CWg5in1xlEdMa0WUSG602SBVkxw2Zbro-XUlmxU,1214
 evalvault/domain/metrics/analysis_registry.py,sha256=JZpBrBs7-JExHKYuEML6Vg_uYLm-WniBE3BfiU5OtJg,7641
 evalvault/domain/metrics/confidence.py,sha256=AX4oeN28OvmMkwD0pT-jskkOlXh87C1pe2W9P1sF69g,17224
@@ -339,8 +339,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.67.0.dist-info/METADATA,sha256=8KLerbvqXFjDF3iio6CNKbIr5O3YmnnNg5C91hmXqdo,26159
-evalvault-1.67.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.67.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.67.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.67.0.dist-info/RECORD,,
+evalvault-1.68.1.dist-info/METADATA,sha256=BRG7UFXRx1fT_JDFqSsdOuB_nk_LVnaNSNYzVyYWyyU,26159
+evalvault-1.68.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.68.1.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.68.1.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.68.1.dist-info/RECORD,,