meta-evaluator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meta_evaluator/__init__.py +19 -0
- meta_evaluator/annotator/exceptions.py +60 -0
- meta_evaluator/annotator/interface/__init__.py +7 -0
- meta_evaluator/annotator/interface/streamlit_app.py +815 -0
- meta_evaluator/annotator/interface/streamlit_session_manager.py +316 -0
- meta_evaluator/annotator/launcher/__init__.py +7 -0
- meta_evaluator/annotator/launcher/entry_point.py +26 -0
- meta_evaluator/annotator/launcher/streamlit_launcher.py +248 -0
- meta_evaluator/common/__init__.py +1 -0
- meta_evaluator/common/async_utils.py +32 -0
- meta_evaluator/common/error_constants.py +12 -0
- meta_evaluator/common/models.py +26 -0
- meta_evaluator/data/__init__.py +20 -0
- meta_evaluator/data/dataloader.py +219 -0
- meta_evaluator/data/eval_data.py +913 -0
- meta_evaluator/data/exceptions.py +177 -0
- meta_evaluator/data/serialization.py +26 -0
- meta_evaluator/eval_task/__init__.py +8 -0
- meta_evaluator/eval_task/eval_task.py +299 -0
- meta_evaluator/eval_task/exceptions.py +28 -0
- meta_evaluator/eval_task/serialization.py +20 -0
- meta_evaluator/judge/__init__.py +5 -0
- meta_evaluator/judge/async_evaluator.py +716 -0
- meta_evaluator/judge/enums.py +34 -0
- meta_evaluator/judge/exceptions.py +129 -0
- meta_evaluator/judge/judge.py +646 -0
- meta_evaluator/judge/models.py +304 -0
- meta_evaluator/judge/serialization.py +22 -0
- meta_evaluator/judge/sync_evaluator.py +667 -0
- meta_evaluator/meta_evaluator/__init__.py +5 -0
- meta_evaluator/meta_evaluator/base.py +712 -0
- meta_evaluator/meta_evaluator/exceptions.py +345 -0
- meta_evaluator/meta_evaluator/judge.py +1067 -0
- meta_evaluator/meta_evaluator/scoring.py +1134 -0
- meta_evaluator/meta_evaluator/serialization.py +20 -0
- meta_evaluator/results/__init__.py +41 -0
- meta_evaluator/results/base.py +582 -0
- meta_evaluator/results/enums.py +29 -0
- meta_evaluator/results/exceptions.py +134 -0
- meta_evaluator/results/human_results.py +344 -0
- meta_evaluator/results/judge_results.py +617 -0
- meta_evaluator/results/models.py +188 -0
- meta_evaluator/results/serialization.py +50 -0
- meta_evaluator/scores/__init__.py +24 -0
- meta_evaluator/scores/base_scorer.py +149 -0
- meta_evaluator/scores/base_scoring_result.py +68 -0
- meta_evaluator/scores/enums.py +16 -0
- meta_evaluator/scores/exceptions.py +52 -0
- meta_evaluator/scores/metrics/__init__.py +15 -0
- meta_evaluator/scores/metrics/agreement/alt_test.py +729 -0
- meta_evaluator/scores/metrics/agreement/iaa.py +164 -0
- meta_evaluator/scores/metrics/classification/classification_scorer.py +276 -0
- meta_evaluator/scores/metrics/text_comparison/semantic_similarity.py +336 -0
- meta_evaluator/scores/metrics/text_comparison/text_similarity.py +193 -0
- meta_evaluator/scores/metrics_config.py +136 -0
- meta_evaluator/scores/utils.py +93 -0
- meta_evaluator/scores_reporting/__init__.py +5 -0
- meta_evaluator/scores_reporting/score_report.py +231 -0
- meta_evaluator-0.1.0.dist-info/METADATA +349 -0
- meta_evaluator-0.1.0.dist-info/RECORD +62 -0
- meta_evaluator-0.1.0.dist-info/WHEEL +4 -0
- meta_evaluator-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""This is a placeholder for the MetaEvaluator class.
|
|
2
|
+
|
|
3
|
+
Note: This class is currently under development
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
# Temporarily disable beartype to resolve import issues
|
|
7
|
+
|
|
8
|
+
# Apply beartype after all imports are complete to avoid path resolution conflicts
|
|
9
|
+
from beartype.claw import beartype_this_package
|
|
10
|
+
|
|
11
|
+
from .meta_evaluator import MetaEvaluator
|
|
12
|
+
|
|
13
|
+
beartype_this_package()
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"MetaEvaluator",
|
|
19
|
+
]
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Custom exceptions for the annotator module."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AnnotationError(Exception):
|
|
5
|
+
"""Base exception for all annotation-related errors."""
|
|
6
|
+
|
|
7
|
+
def __init__(self, message: str):
|
|
8
|
+
"""Initialize the exception with a message.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
message: The error message.
|
|
12
|
+
"""
|
|
13
|
+
self.message = message
|
|
14
|
+
super().__init__(self.message)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AnnotatorInitializationError(AnnotationError):
|
|
18
|
+
"""Exception raised when the annotator app faces initialization errors."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, part: str):
|
|
21
|
+
"""Initialize the exception."""
|
|
22
|
+
super().__init__(f"Initialization error: No {part} initialized")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class NameValidationError(AnnotationError):
|
|
26
|
+
"""Exception raised when annotator name validation fails."""
|
|
27
|
+
|
|
28
|
+
def __init__(self):
|
|
29
|
+
"""Initialize the exception."""
|
|
30
|
+
super().__init__("Missing annotator name")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AnnotationValidationError(AnnotationError):
|
|
34
|
+
"""Exception raised when annotation validation fails."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, field: str, error: Exception):
|
|
37
|
+
"""Initialize the exception."""
|
|
38
|
+
super().__init__(f"Error processing annotation for {field}: {error}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SaveError(AnnotationError):
|
|
42
|
+
"""Exception raised when saving annotations fails."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
message: str,
|
|
47
|
+
filepath: str | None = None,
|
|
48
|
+
):
|
|
49
|
+
"""Initialize the exception."""
|
|
50
|
+
super().__init__(f"{message}: {filepath}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class PortOccupiedError(AnnotationError):
|
|
54
|
+
"""Exception raised when the specified port is already in use."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, port: str | int):
|
|
57
|
+
"""Initialize the exception."""
|
|
58
|
+
super().__init__(
|
|
59
|
+
f"Port {port} is already in use. Please specify a different port.",
|
|
60
|
+
)
|