meta-evaluator 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. meta_evaluator/__init__.py +19 -0
  2. meta_evaluator/annotator/exceptions.py +60 -0
  3. meta_evaluator/annotator/interface/__init__.py +7 -0
  4. meta_evaluator/annotator/interface/streamlit_app.py +815 -0
  5. meta_evaluator/annotator/interface/streamlit_session_manager.py +316 -0
  6. meta_evaluator/annotator/launcher/__init__.py +7 -0
  7. meta_evaluator/annotator/launcher/entry_point.py +26 -0
  8. meta_evaluator/annotator/launcher/streamlit_launcher.py +248 -0
  9. meta_evaluator/common/__init__.py +1 -0
  10. meta_evaluator/common/async_utils.py +32 -0
  11. meta_evaluator/common/error_constants.py +12 -0
  12. meta_evaluator/common/models.py +26 -0
  13. meta_evaluator/data/__init__.py +20 -0
  14. meta_evaluator/data/dataloader.py +219 -0
  15. meta_evaluator/data/eval_data.py +913 -0
  16. meta_evaluator/data/exceptions.py +177 -0
  17. meta_evaluator/data/serialization.py +26 -0
  18. meta_evaluator/eval_task/__init__.py +8 -0
  19. meta_evaluator/eval_task/eval_task.py +299 -0
  20. meta_evaluator/eval_task/exceptions.py +28 -0
  21. meta_evaluator/eval_task/serialization.py +20 -0
  22. meta_evaluator/judge/__init__.py +5 -0
  23. meta_evaluator/judge/async_evaluator.py +716 -0
  24. meta_evaluator/judge/enums.py +34 -0
  25. meta_evaluator/judge/exceptions.py +129 -0
  26. meta_evaluator/judge/judge.py +646 -0
  27. meta_evaluator/judge/models.py +304 -0
  28. meta_evaluator/judge/serialization.py +22 -0
  29. meta_evaluator/judge/sync_evaluator.py +667 -0
  30. meta_evaluator/meta_evaluator/__init__.py +5 -0
  31. meta_evaluator/meta_evaluator/base.py +712 -0
  32. meta_evaluator/meta_evaluator/exceptions.py +345 -0
  33. meta_evaluator/meta_evaluator/judge.py +1067 -0
  34. meta_evaluator/meta_evaluator/scoring.py +1134 -0
  35. meta_evaluator/meta_evaluator/serialization.py +20 -0
  36. meta_evaluator/results/__init__.py +41 -0
  37. meta_evaluator/results/base.py +582 -0
  38. meta_evaluator/results/enums.py +29 -0
  39. meta_evaluator/results/exceptions.py +134 -0
  40. meta_evaluator/results/human_results.py +344 -0
  41. meta_evaluator/results/judge_results.py +617 -0
  42. meta_evaluator/results/models.py +188 -0
  43. meta_evaluator/results/serialization.py +50 -0
  44. meta_evaluator/scores/__init__.py +24 -0
  45. meta_evaluator/scores/base_scorer.py +149 -0
  46. meta_evaluator/scores/base_scoring_result.py +68 -0
  47. meta_evaluator/scores/enums.py +16 -0
  48. meta_evaluator/scores/exceptions.py +52 -0
  49. meta_evaluator/scores/metrics/__init__.py +15 -0
  50. meta_evaluator/scores/metrics/agreement/alt_test.py +729 -0
  51. meta_evaluator/scores/metrics/agreement/iaa.py +164 -0
  52. meta_evaluator/scores/metrics/classification/classification_scorer.py +276 -0
  53. meta_evaluator/scores/metrics/text_comparison/semantic_similarity.py +336 -0
  54. meta_evaluator/scores/metrics/text_comparison/text_similarity.py +193 -0
  55. meta_evaluator/scores/metrics_config.py +136 -0
  56. meta_evaluator/scores/utils.py +93 -0
  57. meta_evaluator/scores_reporting/__init__.py +5 -0
  58. meta_evaluator/scores_reporting/score_report.py +231 -0
  59. meta_evaluator-0.1.0.dist-info/METADATA +349 -0
  60. meta_evaluator-0.1.0.dist-info/RECORD +62 -0
  61. meta_evaluator-0.1.0.dist-info/WHEEL +4 -0
  62. meta_evaluator-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,19 @@
1
+ """This is a placeholder for the MetaEvaluator class.
2
+
3
+ Note: This class is currently under development
4
+ """
5
+
6
+ # Temporarily disable beartype to resolve import issues
7
+
8
+ # Apply beartype after all imports are complete to avoid path resolution conflicts
9
+ from beartype.claw import beartype_this_package
10
+
11
+ from .meta_evaluator import MetaEvaluator
12
+
13
+ beartype_this_package()
14
+
15
+ __version__ = "0.1.0"
16
+
17
+ __all__ = [
18
+ "MetaEvaluator",
19
+ ]
@@ -0,0 +1,60 @@
1
+ """Custom exceptions for the annotator module."""
2
+
3
+
4
+ class AnnotationError(Exception):
5
+ """Base exception for all annotation-related errors."""
6
+
7
+ def __init__(self, message: str):
8
+ """Initialize the exception with a message.
9
+
10
+ Args:
11
+ message: The error message.
12
+ """
13
+ self.message = message
14
+ super().__init__(self.message)
15
+
16
+
17
+ class AnnotatorInitializationError(AnnotationError):
18
+ """Exception raised when the annotator app faces initialization errors."""
19
+
20
+ def __init__(self, part: str):
21
+ """Initialize the exception."""
22
+ super().__init__(f"Initialization error: No {part} initialized")
23
+
24
+
25
+ class NameValidationError(AnnotationError):
26
+ """Exception raised when annotator name validation fails."""
27
+
28
+ def __init__(self):
29
+ """Initialize the exception."""
30
+ super().__init__("Missing annotator name")
31
+
32
+
33
+ class AnnotationValidationError(AnnotationError):
34
+ """Exception raised when annotation validation fails."""
35
+
36
+ def __init__(self, field: str, error: Exception):
37
+ """Initialize the exception."""
38
+ super().__init__(f"Error processing annotation for {field}: {error}")
39
+
40
+
41
+ class SaveError(AnnotationError):
42
+ """Exception raised when saving annotations fails."""
43
+
44
+ def __init__(
45
+ self,
46
+ message: str,
47
+ filepath: str | None = None,
48
+ ):
49
+ """Initialize the exception."""
50
+ super().__init__(f"{message}: {filepath}")
51
+
52
+
53
+ class PortOccupiedError(AnnotationError):
54
+ """Exception raised when the specified port is already in use."""
55
+
56
+ def __init__(self, port: str | int):
57
+ """Initialize the exception."""
58
+ super().__init__(
59
+ f"Port {port} is already in use. Please specify a different port.",
60
+ )
@@ -0,0 +1,7 @@
1
+ """Streamlit code for annotation interface."""
2
+
3
+ from .streamlit_app import StreamlitAnnotator
4
+
5
+ __all__ = [
6
+ "StreamlitAnnotator",
7
+ ]