lionagi 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
@@ -147,9 +147,12 @@ class DirectiveMixin:
147
147
  return_branch=False,
148
148
  **kwargs,
149
149
  )
150
- while isinstance(output, tuple | list):
151
- if len(output) == 2 and output[0] == output[1]:
152
- output = output[0]
150
+ if (
151
+ isinstance(output, tuple | list)
152
+ and len(output) == 2
153
+ and output[0] == output[1]
154
+ ):
155
+ output = output[0]
153
156
  return output
154
157
 
155
158
  async def direct(
File without changes
@@ -0,0 +1,8 @@
1
+ judge_model_config = {
2
+ "model": "gpt-4o-mini",
3
+ "temperature": 0.1,
4
+ "api_key_schema": "OPENAI_API_KEY",
5
+ "max_tokens": 1000,
6
+ "costs": (0.15, 0.6),
7
+ "top_p": 0.9,
8
+ }
File without changes
@@ -0,0 +1,526 @@
1
+ code1 = '''
2
+ from collections import deque
3
+ from functools import singledispatchmethod
4
+ from typing import Annotated, Any, ClassVar, TypeVar
5
+
6
+ from lionabc.exceptions import LionValueError
7
+ from lionfuncs import LN_UNDEFINED, copy, time
8
+ from pydantic import Field, field_serializer, field_validator
9
+ from pydantic.fields import FieldInfo
10
+ from pydantic_core import PydanticUndefined
11
+ from typing_extensions import override
12
+
13
+ from lion_core._class_registry import get_class
14
+ from lion_core.converter import Converter
15
+ from lion_core.generic.component_converter import ComponentConverterRegistry
16
+ from lion_core.generic.element import Element
17
+ from lion_core.generic.note import Note
18
+
19
+ T = TypeVar("T", bound=Element)
20
+
21
+ DEFAULT_SERIALIZATION_INCLUDE: set[str] = {
22
+ "ln_id",
23
+ "timestamp",
24
+ "metadata",
25
+ "content",
26
+ "embedding",
27
+ }
28
+
29
+
30
+ NAMED_FIELD = Annotated[str, Field(..., alias="field")]
31
+
32
+
33
+ class Component(Element):
34
+ """Extended base class for components in the Lion framework."""
35
+
36
+ metadata: Note = Field(
37
+ default_factory=Note,
38
+ description="Additional metadata for the component",
39
+ )
40
+
41
+ content: Any = Field(
42
+ default=None,
43
+ description="The main content of the Component",
44
+ )
45
+
46
+ embedding: list[float] = Field(default_factory=list)
47
+
48
+ extra_fields: dict[str, Any] = Field(default_factory=dict)
49
+
50
+ _converter_registry: ClassVar = ComponentConverterRegistry
51
+
52
+ @field_serializer("metadata")
53
+ def _serialize_metadata(self, value):
54
+ return value.to_dict()
55
+
56
+ @field_serializer("extra_fields")
57
+ def _serialize_extra_fields(
58
+ self,
59
+ value: dict[str, FieldInfo],
60
+ ) -> dict[str, Any]:
61
+ """Custom serializer for extra fields."""
62
+ output_dict = {}
63
+ for k in value.keys():
64
+ k_value = self.__dict__.get(k)
65
+ output_dict[k] = k_value
66
+ return output_dict
67
+
68
+ @field_validator("extra_fields")
69
+ def _validate_extra_fields(cls, value: Any) -> dict[str, FieldInfo]:
70
+ """Custom validator for extra fields."""
71
+ if not isinstance(value, dict):
72
+ raise LionValueError("Extra fields must be a dictionary")
73
+
74
+ out_ = {}
75
+ for k, v in value.items():
76
+ out_[k] = Field(**v) if isinstance(v, dict) else v
77
+
78
+ return out_
79
+
80
+ @property
81
+ def all_fields(self) -> dict[str, FieldInfo]:
82
+ """
83
+ Get all fields including model fields and extra fields.
84
+
85
+ Returns:
86
+ dict[str, FieldInfo]: A dictionary containing all fields.
87
+ """
88
+ return {**self.model_fields, **self.extra_fields}
89
+
90
+ def add_field(
91
+ self,
92
+ field_name: NAMED_FIELD,
93
+ /,
94
+ value: Any = LN_UNDEFINED,
95
+ annotation: Any = LN_UNDEFINED,
96
+ field_obj: FieldInfo = LN_UNDEFINED,
97
+ **kwargs,
98
+ ) -> None:
99
+ """
100
+ Add a new field to the component's extra fields.
101
+
102
+ Args:
103
+ field_name: The name of the field to add.
104
+ value: The value of the field.
105
+ annotation: Type annotation for the field.
106
+ field_obj: A pre-configured FieldInfo object.
107
+ **kwargs: Additional keyword arguments for Field configuration.
108
+
109
+ Raises:
110
+ LionValueError: If the field already exists.
111
+ """
112
+ if field_name in self.all_fields:
113
+ raise LionValueError(f"Field '{field_name}' already exists")
114
+
115
+ self.update_field(
116
+ field_name,
117
+ value=value,
118
+ annotation=annotation,
119
+ field_obj=field_obj,
120
+ **kwargs,
121
+ )
122
+
123
+ # when updating field, we do not check the validity of annotation
124
+ # meaning current value will not get validated, and can lead to
125
+ # errors when storing and loading if you change annotation to a type
126
+ # that is not compatible with the current value
127
+ def update_field(
128
+ self,
129
+ field_name: NAMED_FIELD,
130
+ /,
131
+ value: Any = LN_UNDEFINED,
132
+ annotation: Any = LN_UNDEFINED,
133
+ field_obj: FieldInfo | Any = LN_UNDEFINED,
134
+ **kwargs,
135
+ ) -> None:
136
+ """
137
+ Update an existing field or create a new one if it doesn't exist.
138
+
139
+ Args:
140
+ field_name: The name of the field to update or create.
141
+ value: The new value for the field.
142
+ annotation: Type annotation for the field.
143
+ field_obj: A pre-configured FieldInfo object.
144
+ **kwargs: Additional keyword arguments for Field configuration.
145
+
146
+ Raises:
147
+ ValueError: If both 'default' and 'default_factory' are
148
+ provided in kwargs.
149
+ """
150
+
151
+ # pydanitc Field object cannot have both default and default_factory
152
+ if "default" in kwargs and "default_factory" in kwargs:
153
+ raise ValueError(
154
+ "Cannot provide both 'default' and 'default_factory'",
155
+ )
156
+
157
+ # handle field_obj
158
+ if field_obj is not LN_UNDEFINED:
159
+ if not isinstance(field_obj, FieldInfo):
160
+ raise ValueError(
161
+ "Invalid field_obj, should be a pydantic FieldInfo object"
162
+ )
163
+ self.extra_fields[field_name] = field_obj
164
+
165
+ # handle kwargs
166
+ if kwargs:
167
+ if field_name in self.all_fields: # existing field
168
+ for k, v in kwargs.items():
169
+ self.field_setattr(field_name, k, v)
170
+ else:
171
+ self.extra_fields[field_name] = Field(**kwargs)
172
+
173
+ # handle no explicit defined field
174
+ if field_obj is LN_UNDEFINED and not kwargs:
175
+ if field_name not in self.all_fields:
176
+ self.extra_fields[field_name] = Field()
177
+
178
+ field_obj = self.all_fields[field_name]
179
+
180
+ # handle annotation
181
+ if annotation is not LN_UNDEFINED:
182
+ field_obj.annotation = annotation
183
+ if not field_obj.annotation:
184
+ field_obj.annotation = Any
185
+
186
+ # handle value
187
+ if value is LN_UNDEFINED:
188
+ if getattr(self, field_name, LN_UNDEFINED) is not LN_UNDEFINED:
189
+ value = getattr(self, field_name)
190
+
191
+ elif getattr(field_obj, "default") is not PydanticUndefined:
192
+ value = field_obj.default
193
+
194
+ elif getattr(field_obj, "default_factory"):
195
+ value = field_obj.default_factory()
196
+
197
+ setattr(self, field_name, value)
198
+ self._add_last_update(field_name)
199
+
200
+ def _add_last_update(self, field_name: str, /) -> None:
201
+ current_time = time()
202
+ self.metadata.set(["last_updated", field_name], current_time)
203
+
204
+ @override
205
+ def to_dict(self, **kwargs: Any) -> dict:
206
+ """
207
+ Convert the component to a dictionary representation.
208
+
209
+ Args:
210
+ **kwargs: Additional arguments to pass to model_dump.
211
+
212
+ Returns:
213
+ dict[str, Any]: A dictionary representation of the component.
214
+ """
215
+ dict_ = self.model_dump(**kwargs)
216
+ if isinstance(self.content, Note):
217
+ dict_["content"] = dict_["content"]["content"]
218
+ extra_fields = dict_.pop("extra_fields", {})
219
+ dict_ = {**dict_, **extra_fields, "lion_class": self.class_name()}
220
+ for i in list(dict_.keys()):
221
+ if dict_[i] is LN_UNDEFINED:
222
+ dict_.pop(i)
223
+ return dict_
224
+
225
+ def to_note(self, **kwargs: Any) -> Note:
226
+ return Note(**self.to_dict(**kwargs))
227
+
228
+ @override
229
+ @classmethod
230
+ def from_dict(cls, data: dict, /, **kwargs: Any) -> T:
231
+ """
232
+ Create a component instance from a dictionary.
233
+
234
+ Args:
235
+ data: The dictionary containing component data.
236
+ **kwargs: Additional arguments for Pydantic model validation.
237
+
238
+ Returns:
239
+ T: An instance of the Component class or its subclass.
240
+ """
241
+ input_data = copy(data)
242
+ if "lion_class" in input_data:
243
+ cls = get_class(input_data.pop("lion_class"))
244
+ if cls.from_dict.__func__ != Component.from_dict.__func__:
245
+ return cls.from_dict(input_data, **kwargs)
246
+
247
+ extra_fields = {}
248
+ for k, v in list(input_data.items()):
249
+ if k not in cls.model_fields:
250
+ extra_fields[k] = input_data.pop(k)
251
+ obj = cls.model_validate(input_data, **kwargs)
252
+ for k, v in extra_fields.items():
253
+ obj.update_field(k, value=v)
254
+
255
+ metadata = copy(data.get("metadata", {}))
256
+ last_updated = metadata.get("last_updated", None)
257
+ if last_updated is not None:
258
+ obj.metadata.set(["last_updated"], last_updated)
259
+ else:
260
+ obj.metadata.pop(["last_updated"], None)
261
+ return obj
262
+
263
+ @override
264
+ def __setattr__(self, field_name: str, value: Any) -> None:
265
+ if field_name == "metadata":
266
+ raise AttributeError("Cannot directly assign to metadata.")
267
+ elif field_name == "extra_fields":
268
+ raise AttributeError("Cannot directly assign to extra_fields")
269
+ if field_name in self.extra_fields:
270
+ object.__setattr__(self, field_name, value)
271
+ else:
272
+ super().__setattr__(field_name, value)
273
+
274
+ self._add_last_update(field_name)
275
+
276
+ @override
277
+ def __getattr__(self, field_name: str) -> Any:
278
+ if field_name in self.extra_fields:
279
+ default_ = self.extra_fields[field_name].default
280
+ if default_ is not PydanticUndefined:
281
+ return default_
282
+ return LN_UNDEFINED
283
+
284
+ cls_name = self.__class__.__name__
285
+ raise AttributeError(
286
+ f"'{cls_name}' object has no attribute '{field_name}'",
287
+ )
288
+
289
+ @override
290
+ def __str__(self) -> str:
291
+ """Return a concise string representation of the component."""
292
+ content_preview = str(self.content)[:50]
293
+ if len(content_preview) == 50:
294
+ content_preview += "..."
295
+
296
+ output_str = (
297
+ f"{self.__class__.__name__}("
298
+ f"ln_id={self.ln_id[:8]}..., "
299
+ f"timestamp={str(self.created_datetime)[:-6]}, "
300
+ f"content='{content_preview}', "
301
+ f"metadata_keys={list(self.metadata.keys())}, "
302
+ )
303
+
304
+ for i, j in self.model_dump().items():
305
+ if i not in DEFAULT_SERIALIZATION_INCLUDE:
306
+ if isinstance(j, dict):
307
+ output_str += f"{i}={list(j.keys())}, "
308
+ elif isinstance(j, str):
309
+ j_preview = j[:50]
310
+ if len(j_preview) == 50:
311
+ j_preview = j_preview + "..."
312
+ output_str += f"{i}={j_preview}, "
313
+ else:
314
+ output_str += f"{i}={j}, "
315
+
316
+ output_str += f"extra_fields_keys={list(self.extra_fields.keys())})"
317
+
318
+ return output_str
319
+
320
+ @override
321
+ def __repr__(self) -> str:
322
+ """Return a detailed string representation of the component."""
323
+
324
+ def truncate_dict(
325
+ d: dict[str, Any], max_items: int = 5, max_str_len: int = 50
326
+ ) -> dict[str, Any]:
327
+ items = list(d.items())[:max_items]
328
+ truncated = {
329
+ k: (
330
+ v[:max_str_len] + "..."
331
+ if isinstance(v, str) and len(v) > max_str_len
332
+ else v
333
+ )
334
+ for k, v in items
335
+ }
336
+ if len(d) > max_items:
337
+ truncated["..."] = f"({len(d) - max_items} more items)"
338
+ return truncated
339
+
340
+ content_repr = repr(self.content)
341
+ if len(content_repr) > 100:
342
+ content_repr = content_repr[:97] + "..."
343
+
344
+ dict_ = self.model_dump()
345
+ extra_fields = dict_.pop("extra_fields", {})
346
+
347
+ repr_str = (
348
+ f"{self.class_name()}("
349
+ f"ln_id={repr(self.ln_id)}, "
350
+ f"timestamp={str(self.created_datetime)[:-6]}, "
351
+ f"content={content_repr}, "
352
+ f"metadata={truncate_dict(self.metadata.content)}, "
353
+ )
354
+
355
+ for i, j in dict_.items():
356
+ if i not in DEFAULT_SERIALIZATION_INCLUDE:
357
+ if isinstance(j, dict):
358
+ repr_str += f"{i}={truncate_dict(j)}, "
359
+ elif isinstance(j, str):
360
+ j_repr = j
361
+ if len(j) > 100:
362
+ j_repr = j[:97] + "..."
363
+ repr_str += f"{i}={j_repr}, "
364
+ else:
365
+ repr_str += f"{i}={j}, "
366
+
367
+ repr_str += f"extra_fields={truncate_dict(extra_fields)})"
368
+ return repr_str
369
+
370
+ # converter methods
371
+ @classmethod
372
+ def list_converters(cls) -> list[str]:
373
+ """List all registered converters."""
374
+ return cls._get_converter_registry().list_obj_keys()
375
+
376
+ @classmethod
377
+ def _get_converter_registry(cls) -> ComponentConverterRegistry:
378
+ """Get the converter registry for the class."""
379
+ if isinstance(cls._converter_registry, type):
380
+ cls._converter_registry = cls._converter_registry()
381
+ return cls._converter_registry
382
+
383
+ def convert_to(self, obj_key: str, /, **kwargs: Any) -> Any:
384
+ """Convert the component to a specified type"""
385
+ return self._get_converter_registry().convert_to(
386
+ self,
387
+ obj_key,
388
+ **kwargs,
389
+ )
390
+
391
+ @classmethod
392
+ def convert_from(
393
+ cls, obj: Any, obj_key: str = None, /, **kwargs: Any
394
+ ) -> T:
395
+ """Convert data to create a new component instance"""
396
+ data = cls._get_converter_registry().convert_from(
397
+ cls,
398
+ obj,
399
+ obj_key,
400
+ **kwargs,
401
+ )
402
+ return cls.from_dict(data)
403
+
404
+ @classmethod
405
+ def register_converter(cls, converter: type[Converter]) -> None:
406
+ """Register a new converter."""
407
+ cls._get_converter_registry().register(converter)
408
+
409
+ # field management methods
410
+ def field_setattr(
411
+ self,
412
+ field_name: str,
413
+ attr: Any,
414
+ value: Any,
415
+ /,
416
+ ) -> None:
417
+ """Set the value of a field attribute."""
418
+ all_fields = self.all_fields
419
+ if field_name not in all_fields:
420
+ raise KeyError(f"Field {field_name} not found in object fields.")
421
+ field_obj = all_fields[field_name]
422
+ if hasattr(field_obj, attr):
423
+ setattr(field_obj, attr, value)
424
+ else:
425
+ if not isinstance(field_obj.json_schema_extra, dict):
426
+ field_obj.json_schema_extra = {}
427
+ field_obj.json_schema_extra[attr] = value
428
+
429
+ def field_hasattr(
430
+ self,
431
+ field_name: str,
432
+ attr: str,
433
+ /,
434
+ ) -> bool:
435
+ """Check if a field has a specific attribute."""
436
+ all_fields = self.all_fields
437
+ if field_name not in all_fields:
438
+ raise KeyError(f"Field {field_name} not found in object fields.")
439
+ field_obj = all_fields[field_name]
440
+ if hasattr(field_obj, attr):
441
+ return True
442
+ elif isinstance(field_obj.json_schema_extra, dict):
443
+ if field_name in field_obj.json_schema_extra:
444
+ return True
445
+ else:
446
+ return False
447
+
448
+ def field_getattr(
449
+ self,
450
+ field_name: str,
451
+ attr: str,
452
+ default: Any = LN_UNDEFINED,
453
+ /,
454
+ ) -> Any:
455
+ """Get the value of a field attribute."""
456
+ if str(attr).strip("s").lower() == "annotation":
457
+ return self._field_annotation(field_name)
458
+
459
+ all_fields = self.all_fields
460
+ if field_name not in all_fields:
461
+ raise KeyError(f"Field {field_name} not found in object fields.")
462
+ field_obj = all_fields[field_name]
463
+
464
+ # check fieldinfo attr
465
+
466
+ value = getattr(field_obj, attr, LN_UNDEFINED)
467
+ if value is not LN_UNDEFINED:
468
+ return value
469
+ else:
470
+ if isinstance(field_obj.json_schema_extra, dict):
471
+ value = field_obj.json_schema_extra.get(attr, LN_UNDEFINED)
472
+ if value is not LN_UNDEFINED:
473
+ return value
474
+
475
+ # undefined attr
476
+ if default is not LN_UNDEFINED:
477
+ return default
478
+ else:
479
+ raise AttributeError(
480
+ f"field {field_name} has no attribute {attr}",
481
+ )
482
+
483
+ def field_annotation(self, field_name: Any, /) -> dict[str, Any]:
484
+ """Get the annotation of a field."""
485
+ return self._field_annotation(field_name)
486
+
487
+ @singledispatchmethod
488
+ def _field_annotation(self, field_name: Any, /) -> dict[str, Any]:
489
+ """
490
+ Get field annotation for a given field.
491
+
492
+ Args:
493
+ field: The field to get annotation for.
494
+
495
+ Returns:
496
+ A dictionary containing the field annotation.
497
+ """
498
+ return {}
499
+
500
+ @_field_annotation.register(str)
501
+ def _(self, field_name: str, /) -> dict[str, Any]:
502
+ dict_ = {field_name: self.all_fields[field_name].annotation}
503
+ for _f, _anno in dict_.items():
504
+ if "|" in str(_anno):
505
+ _anno = str(_anno)
506
+ _anno = _anno.split("|")
507
+ dict_[_f] = [str(i).lower().strip() for i in _anno]
508
+ else:
509
+ dict_[_f] = [_anno.__name__] if _anno else None
510
+ return dict_
511
+
512
+ @_field_annotation.register(deque)
513
+ @_field_annotation.register(set)
514
+ @_field_annotation.register(list)
515
+ @_field_annotation.register(tuple)
516
+ def _(self, field_name, /) -> dict[str, Any]:
517
+ dict_ = {}
518
+ for f in field_name:
519
+ dict_.update(self._field_annotation(f))
520
+ return dict_
521
+
522
+
523
+ __all__ = ["Component"]
524
+
525
+ # File: lion_core/generic/component.py
526
+ '''
@@ -0,0 +1,48 @@
1
+ from lionagi.lions.judge.rubric import Rubric, RubricItem
2
+
3
+ functionality = RubricItem(
4
+ name="functionality",
5
+ prompt="Assess how well the code meets the functional requirements.",
6
+ weight=2.7,
7
+ )
8
+
9
+ readability = RubricItem(
10
+ name="readability",
11
+ prompt="Evaluate the readability and clarity of the code.",
12
+ weight=1,
13
+ )
14
+
15
+ efficiency = RubricItem(
16
+ name="efficiency",
17
+ prompt="Examine the efficiency of the algorithms used.",
18
+ weight=1.5,
19
+ )
20
+
21
+ style_compliance = RubricItem(
22
+ name="style_compliance",
23
+ prompt="Check adherence to coding style guidelines.",
24
+ weight=0.5,
25
+ )
26
+
27
+ error_handling = RubricItem(
28
+ name="error_handling",
29
+ prompt="Analyze the robustness of error handling mechanisms.",
30
+ weight=1,
31
+ )
32
+
33
+ code_quality_rubric = Rubric(
34
+ name="code_quality_rubric",
35
+ description="Used for assessing code submissions in programming challenges.",
36
+ items={
37
+ i.name: i
38
+ for i in [
39
+ functionality,
40
+ readability,
41
+ efficiency,
42
+ style_compliance,
43
+ error_handling,
44
+ ]
45
+ },
46
+ )
47
+
48
+ __all__ = ["code_quality_rubric"]
File without changes
@@ -0,0 +1,126 @@
1
+ from lionfuncs import to_num
2
+ from pydantic import Field
3
+
4
+ from lionagi import Form
5
+ from lionagi.lions.judge.rubric import Rubric
6
+
7
+
8
+ class CodeAnalysisForm(Form):
9
+ """
10
+ A structured form for analyzing code submissions based on a given rubric.
11
+ """
12
+
13
+ system_prompt: str = Field(
14
+ """
15
+ You are an AI code reviewer proficient in {language}.
16
+ Evaluate the following code submission based on the criteria below,
17
+ each independently scored from 0 to 100.
18
+ Note that you should be objective and fair in your evaluation, with
19
+ focus around sophistication, usability, and correctness.
20
+ """.strip(
21
+ " "
22
+ )
23
+ )
24
+
25
+ code_submission: str | None = Field(
26
+ default=None,
27
+ description="The code_submission to be analyzed.",
28
+ )
29
+
30
+ overall_feedback: str | None = Field(
31
+ None, description="Overall feedback based on the evaluation."
32
+ )
33
+
34
+ rubric: Rubric
35
+ language: str = "Python"
36
+ assignment: str = "task, code_submission -> overall_feedback"
37
+
38
+ def __init__(
39
+ self,
40
+ code_submission,
41
+ rubric: Rubric,
42
+ instruction=None,
43
+ context=None,
44
+ language: str = "Python",
45
+ ):
46
+ super().__init__(
47
+ code_submission=code_submission, language=language, rubric=rubric
48
+ )
49
+ self.task = (
50
+ "Follow the prompt and provide the necessary output.\n"
51
+ f"- Additional instruction: {str(instruction or 'N/A')}\n"
52
+ f"- Additional context: {str(context or 'N/A')}\n"
53
+ )
54
+ self.system_prompt = self.system_prompt.format(language=language)
55
+
56
+ for item_name, item in rubric.items.items():
57
+ description = item.description or ""
58
+ description += item.prompt
59
+ description += (
60
+ "response format: {score: number 0-100, comments: string}"
61
+ )
62
+ self.add_field(
63
+ item_name, value=None, annotation=dict, description=description
64
+ )
65
+ self.append_to_request(item_name)
66
+
67
+ @property
68
+ def display_message(self) -> str:
69
+ """
70
+ Generates a human-readable message summarizing the evaluation.
71
+ """
72
+
73
+ _dict: dict = self.work_fields
74
+ overall_feedback = _dict.pop("overall_feedback")
75
+ ttl_score = 0
76
+ for i in self.rubric.analysis_types:
77
+ score = to_num(
78
+ _dict[i]["score"], upper_bound=100, lower_bound=0, num_type=int
79
+ )
80
+ weighted_score = score * self.rubric.normalized_weight[i]
81
+ ttl_score += weighted_score
82
+
83
+ rubric_msg = ""
84
+
85
+ message_lines = [
86
+ f"Evaluation Results ({self.timestamp[:-7]}):\n",
87
+ f"Language: {self.language}\n",
88
+ f"Total Score: {ttl_score:.1f}/100\n",
89
+ "----------------------------------------",
90
+ "",
91
+ ]
92
+
93
+ for i in message_lines:
94
+ rubric_msg += f"{i}\n"
95
+
96
+ for k, v in _dict.items():
97
+ if k in self.rubric.analysis_types:
98
+ t = k.replace("_", " ").title()
99
+ rubric_msg += f"{t}:\nScore: {v['score']}/100\nCriteria Weight: {self.rubric.normalized_weight[k]*100:.1f}%\nComments: {v['comments']}\n\n"
100
+
101
+ if overall_feedback:
102
+ rubric_msg += f"Overall Feedback:\n{overall_feedback}\n\n"
103
+
104
+ if ttl_score:
105
+ rubric_msg += f"Total Score: {ttl_score:.1f}/100\n\n"
106
+
107
+ if ttl_score:
108
+ if ttl_score < 30:
109
+ rubric_msg += "Baby Steps. Not an AI hacker yet. 😅\n"
110
+ elif ttl_score < 50:
111
+ rubric_msg += "There we go! You are a junior AI hacker! 🥳\n"
112
+ elif ttl_score < 70:
113
+ rubric_msg += "Great Work! You are a graduated AI hacker! 🤩\n"
114
+ elif ttl_score < 88:
115
+ rubric_msg += "A master AI hacker! 🤯\n"
116
+ else:
117
+ rubric_msg += "The Lion King of AI Hacking! 🚀🦁🚀\n"
118
+
119
+ message = f"""
120
+ {rubric_msg}
121
+ ----------------------------------------
122
+ """
123
+ return message
124
+
125
+
126
+ __all__ = ["CodeAnalysisForm"]
@@ -0,0 +1,34 @@
1
+ from pydantic import BaseModel
2
+
3
+
4
+ class RubricItem(BaseModel):
5
+ name: str
6
+ prompt: str
7
+ description: str | None = None
8
+ weight: float | int = 1
9
+ metadata: dict = {}
10
+
11
+
12
+ class Rubric(BaseModel):
13
+ name: str
14
+ description: str
15
+ items: dict[str, RubricItem]
16
+ metadata: dict = {}
17
+
18
+ @property
19
+ def analysis_types(self):
20
+ return list(self.items.keys())
21
+
22
+ @property
23
+ def normalized_weight(self):
24
+ return {
25
+ item.name: item.weight / self.ttl_weights
26
+ for item in self.items.values()
27
+ }
28
+
29
+ @property
30
+ def ttl_weights(self):
31
+ return sum([item.weight for item in self.items.values()])
32
+
33
+
34
+ __all__ = ["Rubric", "RubricItem"]
File without changes
@@ -0,0 +1,49 @@
1
+ import asyncio
2
+
3
+ from aiocache import cached
4
+
5
+ from lionagi import Branch, iModel
6
+ from lionagi.lions.judge.config import judge_model_config
7
+ from lionagi.lions.judge.forms.code_analysis_form import CodeAnalysisForm
8
+ from lionagi.lions.judge.rubric import Rubric
9
+
10
+
11
+ @cached(ttl=3600)
12
+ async def judge_code(
13
+ code_submission: str,
14
+ rubric: Rubric,
15
+ instruction: str = None,
16
+ context: str = None,
17
+ model_config: dict = None,
18
+ display_message: bool = True,
19
+ verbose: bool = True,
20
+ language: str = "Python",
21
+ ) -> CodeAnalysisForm:
22
+ branch = Branch(imodel=iModel(**(model_config or judge_model_config)))
23
+ form = CodeAnalysisForm(
24
+ code_submission=code_submission,
25
+ rubric=rubric,
26
+ instruction=instruction,
27
+ context=context,
28
+ language=language,
29
+ )
30
+ if verbose:
31
+ print("Evaluating code submission...")
32
+ form = await branch.chat(form=form)
33
+ if display_message:
34
+ print(form.display_message)
35
+ return form.to_dict()
36
+
37
+
38
+ async def main():
39
+ from ..data.sample_codes import code1
40
+ from ..data.sample_rurbic import code_quality_rubric
41
+
42
+ return await judge_code(
43
+ code_submission=code1,
44
+ rubric=code_quality_rubric,
45
+ )
46
+
47
+
48
+ if __name__ == "__main__":
49
+ asyncio.run(main())
lionagi/version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.3.1"
1
+ __version__ = "0.3.3"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: lionagi
3
- Version: 0.3.1
3
+ Version: 0.3.3
4
4
  Summary: Towards automated general intelligence.
5
5
  Author: HaiyangLi
6
6
  Author-email: quantocean.li@gmail.com
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3.11
11
11
  Classifier: Programming Language :: Python :: 3.12
12
12
  Requires-Dist: aiocache (>=0.12.0,<0.13.0)
13
13
  Requires-Dist: ipython (>=8.0.0,<9.0.0)
14
- Requires-Dist: lion-core (>=0.3.15,<0.4.0)
14
+ Requires-Dist: lion-core (>=0.3.17,<0.4.0)
15
15
  Requires-Dist: lion-openai (>=0.1.5,<0.2.0)
16
16
  Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
17
17
  Description-Content-Type: text/markdown
@@ -89,7 +89,7 @@ lionagi/core/rule/string.py,sha256=ZoTuqidyR_vS0BNKEZEj_Oc5Ie8hhy2qaBFGX5J4CrU,1
89
89
  lionagi/core/rule/util.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
90
90
  lionagi/core/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
91
91
  lionagi/core/session/branch.py,sha256=cs51rCG4OSPENVrniOlvKAKBXlcyLbzP032HuiCaIX8,14381
92
- lionagi/core/session/directive_mixin.py,sha256=Gp8JUZpsd5_pAhe81hPc20Hzx4hBWQr-2m7i_DnyQpQ,12071
92
+ lionagi/core/session/directive_mixin.py,sha256=bUHuCDH70BZiPKutRXHXgb5vKSJPL9abQLaIkYnC0Kk,12100
93
93
  lionagi/core/session/session.py,sha256=cLieD42-g8s2ticNY2YyzAAwjwxkyk5q7M5dVlHlMBQ,10479
94
94
  lionagi/core/structure/__init__.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
95
95
  lionagi/core/structure/chain.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -213,14 +213,24 @@ lionagi/lions/coder/base_prompts.py,sha256=SLpC442nZm2cEkB8o9j28kpkB-WzKLjH6sOTS
213
213
  lionagi/lions/coder/code_form.py,sha256=xW66cWCsrZu2qGu-wGUGSIPL1uZevGQVCE_vBRH9Kmc,384
214
214
  lionagi/lions/coder/coder.py,sha256=u-n_7PVdKCAz28SAA2bO4oy1qxGIzEl1PX6iqz7oSoI,5829
215
215
  lionagi/lions/coder/util.py,sha256=m9H18JrePpuP1VyjxVXQaLXCGBed04jZIkfNXvF7_gU,2751
216
+ lionagi/lions/judge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
217
+ lionagi/lions/judge/config.py,sha256=hJNMI-07zf5cqU2tr22fzkGvhR7RdtckkYg8UhLTKec,185
218
+ lionagi/lions/judge/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
219
+ lionagi/lions/judge/data/sample_codes.py,sha256=LfXQ6eiwcR_bkFOURPgZZVE7ShllcS2k6nnC82W75Gc,17500
220
+ lionagi/lions/judge/data/sample_rurbic.py,sha256=H-wS8KliqI3e8XdBcT0hZFi_RdFxAyz2aCxCsR5x6YQ,1123
221
+ lionagi/lions/judge/forms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
222
+ lionagi/lions/judge/forms/code_analysis_form.py,sha256=JK2I0TfeZwpeYolkVlOr-cPFhTXjdyqH9-DmwPztTb4,4129
223
+ lionagi/lions/judge/rubric.py,sha256=NCk06pJd86lbSQNavBYEvzV4WORN746A5J_1qsL4DD4,700
224
+ lionagi/lions/judge/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
225
+ lionagi/lions/judge/services/judge_code.py,sha256=CRfecySGpLw8ceKJwgHWgxorMgjOs_aRAjONJZ6PrE0,1267
216
226
  lionagi/lions/researcher/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
217
227
  lionagi/lions/researcher/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
218
228
  lionagi/lions/researcher/data_source/finhub_.py,sha256=W63daXgIwHJQ6TDMR2ALQIDk1mV3msoDQ9RgOYXsGoE,8783
219
229
  lionagi/lions/researcher/data_source/google_.py,sha256=401SKHQaSpxiOUoXl7stadl4qeF7SIX72lUNK7bKesg,6797
220
230
  lionagi/lions/researcher/data_source/wiki_.py,sha256=UPoa2dk_y5sELu7_rkdme2auDpUmc_Dn0Avgjwr2X2g,3145
221
231
  lionagi/lions/researcher/data_source/yfinance_.py,sha256=snAf897J69MyAc6fcFjF0irrMjbAh81EZ3RvaFT3hxE,977
222
- lionagi/version.py,sha256=r4xAFihOf72W9TD-lpMi6ntWSTKTP2SlzKP1ytkjRbI,22
223
- lionagi-0.3.1.dist-info/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
224
- lionagi-0.3.1.dist-info/METADATA,sha256=4_yy8xYYmMJdgwfrECdbCRK7IhJV885tCjI7I9SbnHY,3149
225
- lionagi-0.3.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
226
- lionagi-0.3.1.dist-info/RECORD,,
232
+ lionagi/version.py,sha256=8KcCYTXH99C2-gCLuPILJvtT9YftRWJsartIx6TQ2ZY,22
233
+ lionagi-0.3.3.dist-info/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
234
+ lionagi-0.3.3.dist-info/METADATA,sha256=aPmmyz7FbSYaeDChIRXx9r3C6-2ZVs-S8YzEZtnpyZg,3149
235
+ lionagi-0.3.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
236
+ lionagi-0.3.3.dist-info/RECORD,,