lionagi 0.17.10__py3-none-any.whl → 0.18.0__py3-none-any.whl

lionagi/operations/parse/parse.py

@@ -1,21 +1,35 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
+import contextlib
+import warnings
 from typing import TYPE_CHECKING, Any, Literal
 
 from pydantic import BaseModel
 
+from lionagi.ln import (
+    extract_json,
+    fuzzy_validate_mapping,
+    get_cancelled_exc_class,
+    to_list,
+)
+from lionagi.ln.fuzzy import FuzzyMatchKeysParams
+from lionagi.protocols.types import AssistantResponse
+from lionagi.session.branch import AlcallParams
+
+from ..types import HandleValidation, ParseParam
+
 if TYPE_CHECKING:
     from lionagi.session.branch import Branch
 
 
-async def parse(
+_CALL = None  # type: ignore
+
+
+def prepare_parse_kws(
     branch: "Branch",
     text: str,
-    handle_validation: Literal[
-        "raise", "return_value", "return_none"
-    ] = "return_value",
+    handle_validation: HandleValidation = "return_value",
     max_retries: int = 3,
     request_type: type[BaseModel] = None,
     operative=None,
@@ -30,159 +44,172 @@ async def parse(
     strict: bool = False,
     suppress_conversion_errors: bool = False,
     response_format=None,
+    request_fields=None,
+    return_res_message: bool = False,
+    **kw,
 ):
-    from lionagi.libs.schema.breakdown_pydantic_annotation import (
-        breakdown_pydantic_annotation,
-    )
-    from lionagi.ln.fuzzy._fuzzy_validate import fuzzy_validate_mapping
-
-    if operative is not None:
-        max_retries = operative.max_retries
-        response_format = operative.request_type or response_format
-        request_type = request_type or operative.request_type
 
-    if not request_type and not response_format:
-        raise ValueError(
-            "Either request_type or response_format must be provided"
+    if suppress_conversion_errors:
+        warnings.warn(
+            "Parameter 'suppress_conversion_errors' is deprecated and no longer used. "
+            "It will be removed in a future version.",
+            DeprecationWarning,
+            stacklevel=2,
         )
 
-    request_type = request_type or response_format
+    response_format = (
+        operative.request_type
+        if operative
+        else response_format or request_type
+    )
+    _alcall_params = get_default_call()
+    max_retries = operative.max_retries if operative else max_retries or 3
+
+    fuzzy_params = FuzzyMatchKeysParams(
+        similarity_algo=similarity_algo,
+        similarity_threshold=similarity_threshold,
+        handle_unmatched=handle_unmatched,
+        fill_value=fill_value,
+        fill_mapping=fill_mapping,
+        strict=strict,
+        fuzzy_match=fuzzy_match,
+    )
 
-    # First attempt: try to parse the text directly
-    import logging
+    return {
+        "text": text,
+        "parse_param": ParseParam(
+            response_format=response_format or request_fields,
+            fuzzy_match_params=fuzzy_params,
+            handle_validation=handle_validation,
+            alcall_params=_alcall_params.with_updates(
+                retry_attempts=max_retries
+            ),
+            imodel=branch.parse_model,
+            imodel_kw=kw,
+        ),
+        "return_res_message": return_res_message,
+    }
 
-    initial_error = None
-    parsed_data = None  # Initialize to avoid scoping issues
 
-    try:
-        # Try fuzzy validation first
-        parsed_data = fuzzy_validate_mapping(
-            text,
-            breakdown_pydantic_annotation(request_type),
-            similarity_algo=similarity_algo,
-            similarity_threshold=similarity_threshold,
-            fuzzy_match=fuzzy_match,
-            handle_unmatched=handle_unmatched,
-            fill_value=fill_value,
-            fill_mapping=fill_mapping,
-            strict=strict,
-            suppress_conversion_errors=False,  # Don't suppress on first attempt
+async def parse(
+    branch: "Branch",
+    text: str,
+    parse_param: ParseParam,
+    return_res_message: bool = False,
+) -> Any | tuple[Any, AssistantResponse | None]:
+
+    # Try direct validation first
+    with contextlib.suppress(Exception):
+        result = _validate_dict_or_model(
+            text, parse_param.response_format, parse_param.fuzzy_match_params
+        )
+        return result if not return_res_message else (result, None)
+
+    async def _inner_parse(i):
+        _, res = await branch.chat(
+            instruction="reformat text into specified model or structure",
+            guidance="follow the required response format, using the model schema as a guide",
+            context=[{"text_to_format": text}],
+            request_fields=(
+                parse_param.response_format
+                if isinstance(parse_param.response_format, dict)
+                else None
+            ),
+            response_format=(
+                parse_param.response_format
+                if isinstance(parse_param.response_format, BaseModel)
+                else None
+            ),
+            imodel=parse_param.imodel or branch.parse_model,
+            sender=branch.user,
+            recipient=branch.id,
+            return_ins_res_message=True,
         )
 
-        logging.debug(f"Parsed data from fuzzy validation: {parsed_data}")
+        res.metadata["is_parsed"] = True
+        res.metadata["original_text"] = text
 
-        # Validate with pydantic
-        if operative is not None:
-            response_model = operative.update_response_model(parsed_data)
-        else:
-            response_model = request_type.model_validate(parsed_data)
+        return (
+            _validate_dict_or_model(
+                res.response,
+                parse_param.response_format,
+                parse_param.fuzzy_match_params,
+            ),
+            res,
+        )
 
-        # If successful, return immediately
-        if isinstance(response_model, BaseModel):
-            return response_model
+    _call = parse_param.alcall_params or get_default_call()
+    if isinstance(parse_param.alcall_params, dict):
+        _call = AlcallParams(**parse_param.alcall_params)
 
+    try:
+        result = await _call([0], _inner_parse)
+    except get_cancelled_exc_class():
+        raise
     except Exception as e:
-        initial_error = e
-        # Log the initial parsing error for debugging
-        logging.debug(
-            f"Initial parsing failed for text '{text[:100]}...': {e}"
-        )
-        logging.debug(
-            f"Parsed data was: {locals().get('parsed_data', 'not set')}"
-        )
-
-        # Only continue if we have retries left
-        if max_retries <= 0:
-            if handle_validation == "raise":
+        match parse_param.handle_validation:
+            case "raise":
                 raise ValueError(f"Failed to parse response: {e}") from e
-            elif handle_validation == "return_none":
-                return None
-            else:  # return_value
-                return text
-
-    # If direct parsing failed, try using the parse model
-    num_try = 0
-    last_error = initial_error
-
-    # Check if the parsed_data exists but just failed validation
-    # This might mean we have the right structure but wrong values
-    if parsed_data is not None and isinstance(parsed_data, dict):
-        logging.debug(
-            f"Have parsed_data dict, checking if it's close to valid..."
-        )
-        # If we got a dict with the right keys, maybe we just need to clean it up
-        expected_fields = set(request_type.model_fields.keys())
-        parsed_fields = set(parsed_data.keys())
-        if expected_fields == parsed_fields and all(
-            parsed_data.get(k) is not None for k in expected_fields
-        ):
-            # We have the right structure with non-None values, don't retry with parse model
-            logging.debug(
-                "Structure matches with valid values, returning original error"
+            case "return_none":
+                return (None, None) if return_res_message else None
+            case "return_value":
+                return (text, None) if return_res_message else text
+    return (*result[0],) if return_res_message else result[0][0]
+
+
+def _validate_dict_or_model(
+    text: str,
+    response_format: type[BaseModel] | dict,
+    fuzzy_match_params: FuzzyMatchKeysParams | dict = None,
+):
+    try:
+        if isinstance(fuzzy_match_params, dict):
+            fuzzy_match_params = FuzzyMatchKeysParams(**fuzzy_match_params)
+
+        d_ = extract_json(text, fuzzy_parse=True, return_one_if_single=False)
+        dict_, keys_ = None, None
+        if d_:
+            dict_ = to_list(d_, flatten=True)[0]
+        if isinstance(fuzzy_match_params, FuzzyMatchKeysParams):
+            keys_ = (
+                response_format.model_fields
+                if isinstance(response_format, type)
+                else response_format
             )
-            if handle_validation == "raise":
-                raise ValueError(
-                    f"Failed to parse response: {initial_error}"
-                ) from initial_error
-            elif handle_validation == "return_none":
-                return None
-            else:
-                return text
-
-    while num_try < max_retries:
-        num_try += 1
-
-        try:
-            logging.debug(f"Retry {num_try}: Using parse model to reformat")
-            _, res = await branch.chat(
-                instruction="reformat text into specified model",
-                guidance="follow the required response format, using the model schema as a guide",
-                context=[{"text_to_format": text}],
-                response_format=request_type,
-                sender=branch.user,
-                recipient=branch.id,
-                imodel=branch.parse_model,
-                return_ins_res_message=True,
+            dict_ = fuzzy_validate_mapping(
+                dict_, keys_, **fuzzy_match_params.to_dict()
             )
-
-            # Try to parse the reformatted response
-            parsed_data = fuzzy_validate_mapping(
-                res.response,
-                breakdown_pydantic_annotation(request_type),
-                similarity_algo=similarity_algo,
-                similarity_threshold=similarity_threshold,
-                fuzzy_match=fuzzy_match,
-                handle_unmatched=handle_unmatched,
-                fill_value=fill_value,
-                fill_mapping=fill_mapping,
-                strict=strict,
-                suppress_conversion_errors=suppress_conversion_errors,
+        elif fuzzy_match_params:
+            keys_ = (
+                response_format.model_fields
+                if isinstance(response_format, type)
+                else response_format
             )
+            dict_ = fuzzy_validate_mapping(
+                dict_,
+                keys_,
+                handle_unmatched="force",
+                fill_value=None,
+                strict=False,
+            )
+        if isinstance(response_format, type) and issubclass(
+            response_format, BaseModel
+        ):
+            return response_format.model_validate(dict_)
+        return dict_
 
-            if operative is not None:
-                response_model = operative.update_response_model(parsed_data)
-            else:
-                response_model = request_type.model_validate(parsed_data)
-
-            # If successful, return
-            if isinstance(response_model, BaseModel):
-                return response_model
-
-        except InterruptedError as e:
-            raise e
-        except Exception as e:
-            last_error = e
-            # Continue to next retry
-            continue
-
-    # All retries exhausted
-    match handle_validation:
-        case "return_value":
-            return text
-        case "return_none":
-            return None
-        case "raise":
-            error_msg = "Failed to parse response into request format"
-            if last_error:
-                error_msg += f": {last_error}"
-            raise ValueError(error_msg) from last_error
+    except Exception as e:
+        raise ValueError(f"Failed to parse text: {e}") from e
+
+
+def get_default_call() -> AlcallParams:
+    global _CALL
+    if _CALL is None:
+        _CALL = AlcallParams(
+            retry_initial_delay=1,
+            retry_backoff=1.85,
+            retry_attempts=3,
+            max_concurrent=1,
+            throttle_period=1,
+        )
+    return _CALL

lionagi/operations/plan/__init__.py

@@ -1,5 +1,4 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
 from .plan import plan

lionagi/operations/plan/plan.py

@@ -1,5 +1,4 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
 from typing import Any, Literal

lionagi/operations/plan/prompt.py

@@ -1,5 +1,4 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
 PLAN_PROMPT = """

lionagi/operations/select/__init__.py

@@ -1,3 +1,2 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0

lionagi/operations/select/select.py

@@ -1,5 +1,4 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
 from enum import Enum
@@ -9,7 +8,7 @@ from pydantic import BaseModel
 
 from lionagi.fields.instruct import Instruct
 
-from .utils import SelectionModel
+from .utils import SelectionModel, parse_selection, parse_to_representation
 
 if TYPE_CHECKING:
     from lionagi.session.branch import Branch
@@ -25,42 +24,104 @@ async def select(
     verbose: bool = False,
     **kwargs: Any,
 ) -> SelectionModel | tuple[SelectionModel, "Branch"]:
+    """
+    Select from choices using LLM - legacy wrapper with backwards compatibility.
+
+    Args:
+        branch: Branch instance to use
+        instruct: Instruction for selection
+        choices: Available choices (list, dict, or Enum)
+        max_num_selections: Max number of selections
+        branch_kwargs: Kwargs for branch creation (deprecated)
+        return_branch: Return (result, branch) tuple
+        verbose: Print progress
+        **kwargs: Additional operate kwargs
+
+    Returns:
+        SelectionModel or (SelectionModel, Branch) tuple
+    """
     if verbose:
         print(f"Starting selection with up to {max_num_selections} choices.")
 
-    from .utils import SelectionModel, parse_selection, parse_to_representation
+    # Handle branch creation for backwards compatibility
+    if branch is None and branch_kwargs:
+        from lionagi.session.branch import Branch
 
-    branch = branch or Branch(**(branch_kwargs or {}))
+        branch = Branch(**branch_kwargs)
+
+    result = await select_v1(
+        branch=branch,
+        instruct=instruct,
+        choices=choices,
+        max_num_selections=max_num_selections,
+        verbose=verbose,
+        **kwargs,
+    )
+
+    if return_branch:
+        return result, branch
+    return result
+
+
+async def select_v1(
+    branch: "Branch",
+    instruct: Instruct | dict[str, Any],
+    choices: list[str] | type[Enum] | dict[str, Any],
+    max_num_selections: int = 1,
+    verbose: bool = False,
+    **operate_kwargs: Any,
+) -> SelectionModel:
+    """
+    Context-based selection implementation.
+
+    Args:
+        branch: Branch instance
+        instruct: Selection instruction
+        choices: Available choices
+        max_num_selections: Maximum selections allowed
+        verbose: Print progress
+        **operate_kwargs: Additional operate parameters
+
+    Returns:
+        SelectionModel with corrected selections
+    """
+    # Parse choices into keys and representations
     selections, contents = parse_to_representation(choices)
     prompt = SelectionModel.PROMPT.format(
         max_num_selections=max_num_selections, choices=selections
     )
 
+    # Build instruction dictionary
     if isinstance(instruct, Instruct):
-        instruct = instruct.to_dict()
-
-    instruct = instruct or {}
+        instruct_dict = instruct.to_dict()
+    else:
+        instruct_dict = instruct or {}
 
-    if instruct.get("instruction", None) is not None:
-        instruct["instruction"] = (
-            f"{instruct['instruction']}\n\n{prompt} \n\n "
+    # Append selection prompt to instruction
+    if instruct_dict.get("instruction", None) is not None:
+        instruct_dict["instruction"] = (
+            f"{instruct_dict['instruction']}\n\n{prompt} \n\n "
         )
     else:
-        instruct["instruction"] = prompt
+        instruct_dict["instruction"] = prompt
 
-    context = instruct.get("context", None) or []
+    # Add choice representations to context
+    context = instruct_dict.get("context", None) or []
     context = [context] if not isinstance(context, list) else context
     context.extend([{k: v} for k, v in zip(selections, contents)])
-    instruct["context"] = context
+    instruct_dict["context"] = context
 
+    # Call branch.operate with SelectionModel as response format
     response_model: SelectionModel = await branch.operate(
         response_format=SelectionModel,
-        **kwargs,
-        **instruct,
+        **operate_kwargs,
+        **instruct_dict,
     )
+
     if verbose:
         print(f"Received selection: {response_model.selected}")
 
+    # Extract and normalize selected values
     selected = response_model
     if isinstance(response_model, BaseModel) and hasattr(
         response_model, "selected"
@@ -68,14 +129,13 @@ async def select(
         selected = response_model.selected
     selected = [selected] if not isinstance(selected, list) else selected
 
+    # Parse selections back to original choice values
     corrected_selections = [parse_selection(i, choices) for i in selected]
 
+    # Update response model with corrected selections
     if isinstance(response_model, BaseModel):
         response_model.selected = corrected_selections
-
     elif isinstance(response_model, dict):
         response_model["selected"] = corrected_selections
 
-    if return_branch:
-        return response_model, branch
     return response_model

lionagi/operations/select/utils.py

@@ -1,5 +1,4 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
+# Copyright (c) 2023-2025, HaiyangLi <quantocean.li at gmail dot com>
 # SPDX-License-Identifier: Apache-2.0
 
 import inspect
@@ -8,7 +7,7 @@ from typing import Any, ClassVar
 
 from pydantic import BaseModel, Field, JsonValue
 
-from lionagi.libs.validate.string_similarity import string_similarity
+from lionagi.ln.fuzzy import string_similarity
 from lionagi.utils import is_same_dtype