fastworkflow 2.15.5__py3-none-any.whl → 2.17.13__py3-none-any.whl

fastworkflow/utils/signatures.py

@@ -1,4 +1,5 @@
 import sys
+import ast
 import dspy
 import os
 from contextlib import suppress
@@ -156,7 +157,7 @@ Today's date is {today}.
     def create_signature_from_pydantic_model(
         pydantic_model: Type[BaseModel]
     ) -> Type[dspy.Signature]:
-            """
+        """
         Create a DSPy Signature class from a Pydantic model with type annotations.
         
         Args:
@@ -165,79 +166,76 @@ Today's date is {today}.
         Returns:
             A DSPy Signature class
         """
-            signature_components = {
-                "command": (str, dspy.InputField(desc="User's request"))
-            }
+        signature_components = {
+            "statement": (str, dspy.InputField(desc="Statement according to Dhar"))
+        }
 
-            steps = ["query: The original user query (Always include this)."]
-            field_num = 1
+        steps = []
+        field_num = 1
 
-            for attribute_name, attribute_metadata in pydantic_model.model_fields.items():
-                is_optional = False
-                attribute_type = attribute_metadata.annotation
+        for attribute_name, attribute_metadata in pydantic_model.model_fields.items():
+            is_optional = False
+            attribute_type = attribute_metadata.annotation
 
-                if hasattr(attribute_type, "__origin__") and attribute_type.__origin__ is Union:
-                    union_elements = get_args(attribute_type)
-                    if type(None) in union_elements:
-                        is_optional = True
-                        attribute_type = next((elem for elem in union_elements if elem is not type(None)), str)
-
-                NOT_FOUND = fastworkflow.get_env_var("NOT_FOUND")
-                if attribute_type is str:            
-                    default_value = NOT_FOUND
-                elif attribute_type is int:
-                    default_value = INVALID_INT_VALUE
-                elif attribute_type is float:
-                    default_value = -sys.float_info.max
-                else:
-                    default_value = None
+            if hasattr(attribute_type, "__origin__") and attribute_type.__origin__ is Union:
+                union_elements = get_args(attribute_type)
+                if type(None) in union_elements:
+                    is_optional = True
+                    attribute_type = next((elem for elem in union_elements if elem is not type(None)), str)
 
-                if (
-                    attribute_metadata.default is not PydanticUndefined and 
-                    attribute_metadata.default is not None and 
-                    attribute_metadata.default != Ellipsis
-                ):
-                    default_value = attribute_metadata.default
+            NOT_FOUND = fastworkflow.get_env_var("NOT_FOUND")
+            if attribute_type is str:            
+                default_value = NOT_FOUND
+            elif attribute_type is int:
+                default_value = INVALID_INT_VALUE
+            elif attribute_type is float:
+                default_value = -sys.float_info.max
+            else:
+                default_value = None
+
+            if (
+                attribute_metadata.default is not PydanticUndefined and 
+                attribute_metadata.default is not None and 
+                attribute_metadata.default != Ellipsis
+            ):
+                default_value = attribute_metadata.default
 
-                info_text = attribute_metadata.description or f"The {attribute_name}"
+            info_text = attribute_metadata.description or f"The {attribute_name}"
 
-                if attribute_name != "query": 
-                    steps.append(f"Step {field_num}: Identify the {attribute_name} ({info_text}).")
-                    field_num += 1
+            if attribute_name != "query": 
+                steps.append(f"Step {field_num}: Identify the {attribute_name} ({info_text}).")
+                field_num += 1
 
-                if isinstance(attribute_type, type) and issubclass(attribute_type, Enum):
-                    possible_values = [f"'{option.value}'" for option in attribute_type]
-                    info_text += f". Valid values: {', '.join(possible_values)}"
+            if isinstance(attribute_type, type) and issubclass(attribute_type, Enum):
+                possible_values = [f"'{option.value}'" for option in attribute_type]
+                info_text += f". Valid values: {', '.join(possible_values)}"
 
-                if attribute_metadata.examples:
-                    sample_values = ", ".join([f"'{sample}'" for sample in attribute_metadata.examples])
-                    info_text += f". Examples: {sample_values}"
+            if attribute_metadata.examples:
+                sample_values = ", ".join([f"'{sample}'" for sample in attribute_metadata.examples])
+                info_text += f". Examples: {sample_values}"
 
-                requirement_status = "Optional" if is_optional else "Required"
-                info_text += f". This field is {requirement_status}."
+            requirement_status = "Optional" if is_optional else "Required"
+            info_text += f". This field is {requirement_status}."
 
-                if is_optional:
-                    info_text += f" If not mentioned in the query, use: '{default_value or 'None'}'."
-                elif default_value is not None:
-                    info_text += f" Default value: '{default_value}'."
+            if is_optional:
+                info_text += f" If not mentioned in the query, use: '{default_value or 'None'}'."
+            elif default_value is not None:
+                info_text += f" Default value: '{default_value}'."
 
-                field_definition = dspy.OutputField(desc=info_text, default=default_value)
-                signature_components[attribute_name] = (attribute_metadata.annotation, field_definition)
+            field_definition = dspy.OutputField(desc=info_text, default=default_value)
+            signature_components[attribute_name] = (attribute_metadata.annotation, field_definition)
 
             steps.extend((
-                f"Step {field_num}: Check for any missing details.",
-                "Return the default value for the parameters for which default value is specified.",
-                "For parameters specified as enums, return the default value if the parameter value is not explicitly specified in the query",
-                "Return None for the parameter value which is missing in the query",
-                "Always return the query in the output.",
+                f"Step {field_num}: ",
+                "For missing parameter values - return the default if it is specified otherwise return None",
             ))
-            generated_docstring = f"""Extract structured parameters from a user query using step-by-step reasoning. Today's date is {date.today()}.
-
-        {chr(10).join(steps)}
-        """
-            instructions = generated_docstring
+            
+        generated_docstring = (
+            f"Extract parameter values from the statement according to Dhar. Today's date is {date.today()}.\n"
+            f"{chr(10).join(steps)}"
+        )
 
-            return dspy.Signature(signature_components, instructions)
+        return dspy.Signature(signature_components, generated_docstring)
     
     def extract_parameters(self, CommandParameters: Type[BaseModel] = None, subject_command_name: str = None, workflow_folderpath: str = None) -> BaseModel:
         """
@@ -269,7 +267,7 @@ Today's date is {today}.
                 self.predictor = dspy.ChainOfThought(signature)
                 
             def forward(self, command=None):
-                return self.predictor(command=command)
+                return self.predictor(statement=command)
         
         param_extractor = ParamExtractor(params_signature)
         
@@ -285,8 +283,24 @@ Today's date is {today}.
                 trainset=trainset
             )
 
+            def basic_checks(args, pred):
+                for field_name in field_names:
+                    # return 0 if it extracts an example value instead of correct value | None
+                    extracted_param_value = getattr(pred, field_name)
+                    examples = model_class.model_fields[field_name].examples
+                    if examples and extracted_param_value in examples:
+                        return 0.0
+                return 1.0
+
+            # Create a refined module that tries up to 3 times
+            best_of_3 = dspy.BestOfN(
+                module=compiled_model, 
+                N=3, 
+                reward_fn=basic_checks, 
+                threshold=1.0)
+
             try:
-                dspy_result = compiled_model(command=self.command)
+                dspy_result = best_of_3(command=self.command)
                 for field_name in field_names:
                     default = model_class.model_fields[field_name].default
                     param_dict[field_name] = getattr(dspy_result, field_name, default)
@@ -302,7 +316,7 @@ Today's date is {today}.
     def validate_parameters(self,
                             app_workflow: fastworkflow.Workflow, 
                             subject_command_name: str,
-                            cmd_parameters: BaseModel) -> Tuple[bool, str, Dict[str, List[str]]]:
+                            cmd_parameters: BaseModel) -> Tuple[bool, str, Dict[str, List[str]], List[str]]:
         """
         Check if the parameters are valid in the current context, including database lookups.
         """
@@ -319,62 +333,259 @@ Today's date is {today}.
         invalid_fields = []
         all_suggestions = {}
 
-        # Check required fields
         for field_name, field_info in type(cmd_parameters).model_fields.items():
-            field_value = getattr(cmd_parameters, field_name, None)
+                field_value = getattr(cmd_parameters, field_name, None)
+
+                if field_value not in [NOT_FOUND, None, INVALID_INT_VALUE, INVALID_FLOAT_VALUE]:
+                        annotation = field_info.annotation
+
+                        # Build list of candidate concrete types (exclude NoneType from Union)
+                        candidate_types: List[Type[Any]] = []
+                        if hasattr(annotation, "__origin__") and annotation.__origin__ is Union:
+                            for t in get_args(annotation):
+                                if t is not type(None):  # noqa: E721
+                                    candidate_types.append(t)  # type: ignore[arg-type]
+                        else:
+                            candidate_types = [annotation]  # type: ignore[list-item]
+
+                        def build_type_suggestion() -> List[str]:
+                            examples = getattr(field_info, "examples", []) or []
+                            example = examples[0] if examples else None
+                            # Enum suggestions list valid values
+                            enum_types = [t for t in candidate_types if isinstance(t, type) and issubclass(t, Enum)]
+                            if enum_types:
+                                opts = [f"'{opt.value}'" for t in enum_types for opt in t]
+                                return [f"Please provide a value matching the expected type/format. Valid values: {', '.join(opts)}"]
+                            # List suggestions
+                            def _is_list_type(tt):
+                                try:
+                                    return hasattr(tt, "__origin__") and tt.__origin__ in (list, List)
+                                except Exception:
+                                    return False
+                            list_types = [t for t in candidate_types if _is_list_type(t)]
+                            if list_types:
+                                inner_args = get_args(list_types[0])
+                                inner = inner_args[0] if inner_args else str
+                                inner_name = inner.__name__ if isinstance(inner, type) else str(inner)
+                                hint = (
+                                    f"Please provide a list of {inner_name} values. Accepted formats: "
+                                    f"JSON list (e.g., [\"a\", \"b\"]), Python list (e.g., ['a', 'b']), "
+                                    f"or comma-separated (e.g., a,b)."
+                                )
+                                return [hint]
+                            # Fallback: show expected type names (handles unions)
+                            name_list: List[str] = []
+                            for t in candidate_types:
+                                if isinstance(t, type):
+                                    name_list.append(t.__name__)
+                                else:
+                                    name_list.append(str(t))
+                            base = f"Please provide a value matching the expected type/format: {' or '.join(name_list)}"
+                            if example is not None:
+                                base = f"{base} (e.g., {example})"
+                            return [base]
+
+                        valid_by_type = False
+                        corrected_value: Optional[Any] = None
+                        def _is_list_type(tt):
+                            try:
+                                return hasattr(tt, "__origin__") and tt.__origin__ in (list, List)
+                            except Exception:
+                                return False
+
+                        def _parse_list_like_string(s: str) -> Optional[list]:
+                                if not isinstance(s, str):
+                                    return None
+                                text = s.strip()
+                                if text.startswith("[") and text.endswith("]"):
+                                        with suppress(Exception):
+                                                parsed = json.loads(text)
+                                                if isinstance(parsed, list):
+                                                    return parsed
+                                # Try Python literal list
+                                with suppress(Exception):
+                                        parsed = ast.literal_eval(text)
+                                        if isinstance(parsed, list):
+                                            return parsed
+                                # Comma-separated values
+                                if "," in text:
+                                    parts = [p.strip() for p in text.split(",")]
+                                    cleaned = [
+                                        (p[1:-1] if len(p) >= 2 and ((p[0] == p[-1] == '"') or (p[0] == p[-1] == "'")) else p)
+                                        for p in parts
+                                    ]
+                                    return cleaned
+                                # Single value - treat as a list with one element
+                                if text:
+                                    # Remove quotes if present
+                                    if len(text) >= 2 and ((text[0] == text[-1] == '"') or (text[0] == text[-1] == "'")):
+                                        return [text[1:-1]]
+                                    return [text]
+                                return None
+
+                        def _coerce_scalar(expected_type: Type[Any], val: Any) -> Tuple[bool, Optional[Any]]:
+                                # str
+                                if expected_type is str:
+                                    return True, str(val)
+                                # bool
+                                if expected_type is bool:
+                                    if isinstance(val, bool):
+                                        return True, val
+                                    elif isinstance(val, str):
+                                        lower_val = val.lower().strip()
+                                        if lower_val in ('true', 'false'):
+                                            return True, lower_val == 'true'
+                                        # Also handle string representations of integers
+                                        elif lower_val in ('0', '1'):
+                                            return True, lower_val == '1'
+                                    elif isinstance(val, int):
+                                        return True, bool(val)
+                                    return False, None
+                                # int
+                                if expected_type is int:
+                                    if isinstance(val, bool):
+                                        return False, None
+                                    if isinstance(val, int):
+                                        return True, val
+                                    if isinstance(val, str):
+                                        with suppress(Exception):
+                                            return True, int(val.strip())
+                                    return False, None
+                                # float
+                                if expected_type is float:
+                                    if isinstance(val, (int, float)) and not isinstance(val, bool):
+                                        return True, float(val)
+                                    if isinstance(val, str):
+                                        with suppress(Exception):
+                                            return True, float(val.strip())
+                                    return False, None
+                                # Enum
+                                if isinstance(expected_type, type) and issubclass(expected_type, Enum):
+                                    ok, enum_val = _try_coerce_enum(expected_type, val)
+                                    return (ok, enum_val if ok else None)
+                                                            # Unknown: accept if already instance
+                                return (True, val) if isinstance(val, expected_type) else (False, None)
+
+                        def _try_coerce_list(list_type: Any, value: Any) -> Tuple[bool, Optional[list]]:
+                            inner_args = get_args(list_type)
+                            inner_type = inner_args[0] if inner_args else str
+                            raw_list: Optional[list] = None
+                            if isinstance(value, list):
+                                raw_list = value
+                            elif isinstance(value, str):
+                                raw_list = _parse_list_like_string(value)
+                            if raw_list is None:
+                                return False, None
+                            coerced_list = []
+                            for item in raw_list:
+                                ok, coerced = _coerce_scalar(inner_type, item)
+                                if not ok:
+                                    return False, None
+                                coerced_list.append(coerced)
+                            return True, coerced_list
+
+                        def _try_coerce_enum(enum_cls: Type[Enum], val: Any) -> Tuple[bool, Optional[Enum]]:
+                            if isinstance(val, enum_cls):
+                                return True, val
+                            if isinstance(val, str):
+                                for member in enum_cls:
+                                    if val == member.value or val.lower() == str(member.name).lower():
+                                        return True, member
+                            return False, None
+
+                        for t in candidate_types:
+                            # list[...] and typing.List[...] support
+                            if _is_list_type(t):
+                                ok, coerced_list = _try_coerce_list(t, field_value)
+                                if ok:
+                                    corrected_value = coerced_list
+                                    valid_by_type = True
+                                    break
+                            # str, bool, int, float - use _coerce_scalar for consistency
+                            if t in (str, bool, int, float):
+                                ok, coerced = _coerce_scalar(t, field_value)
+                                if ok:
+                                    corrected_value = coerced
+                                    valid_by_type = True
+                                    break                            # Enum
+                            if isinstance(t, type) and issubclass(t, Enum):
+                                ok, enum_val = _try_coerce_enum(t, field_value)
+                                if ok:
+                                    corrected_value = enum_val
+                                    valid_by_type = True
+                                    break
+
+                        if valid_by_type:
+                            if corrected_value is not None:
+                                pattern = next(
+                                    (meta.pattern
+                                        for meta in getattr(field_info, "metadata", [])
+                                        if hasattr(meta, "pattern")),
+                                    None,
+                                )
+                                if pattern and field_value is not None and field_value != NOT_FOUND:
+                                    invalid_value = None
+                                    if hasattr(field_info, "json_schema_extra") and field_info.json_schema_extra:
+                                        invalid_value = field_info.json_schema_extra.get("invalid_value")
+
+                                    # if invalid_value and field_value == invalid_value:
+                                    #     invalid_fields.append(f"{field_name} '{field_value}'")
+                                    #     pattern_str = str(pattern)
+                                    #     examples = getattr(field_info, "examples", [])
+                                    #     example = examples[0] if examples else ""
+                                    #     all_suggestions[field_name] = [f"Please use the format matching pattern {pattern_str} (e.g., {example})"]
+                                    #     is_valid = False
+
+                                    # else:
+                                    pattern_regex = re.compile(pattern)
+                                    if not pattern_regex.fullmatch(str(field_value)):
+                                        invalid_fields.append(f"{field_name} '{field_value}'")
+                                        pattern_str = str(pattern)
+                                        examples = getattr(field_info, "examples", [])
+                                        example = examples[0] if examples else ""
+
+                                        invalid_fields.append(f"{field_name} '{field_value}'")
+                                        all_suggestions[field_name] = [f"Please use the format matching pattern {pattern_str} (e.g., {example})"]
+                                        is_valid = False
+                                    else:
+                                        try:
+                                            setattr(cmd_parameters, field_name, corrected_value)
+                                        except Exception as e:
+                                            logger.critical(f"Failed to set attribute {field_name} with value {corrected_value}")
+                                            raise e
+                                else:
+                                    try:
+                                        setattr(cmd_parameters, field_name, corrected_value)
+                                    except Exception as e:
+                                        logger.critical(f"Failed to set attribute {field_name} with value {corrected_value}")
+                                        raise e
+                        else:
+                            invalid_fields.append(f"{field_name} '{field_value}'")
+                            all_suggestions[field_name] = build_type_suggestion()
+                            is_valid = False
 
-            is_optional = False
-            attribute_type = field_info.annotation
-            if hasattr(attribute_type, "__origin__") and attribute_type.__origin__ is Union:
-                union_elements = get_args(attribute_type)
-                if type(None) in union_elements:
-                    is_optional = True
-
-            is_required=True
-            if is_optional:
-                    is_required=False
-
-            # Only add to missing fields if it's required AND has no value
-            if is_required and \
-                field_value in [
-                    NOT_FOUND, 
-                    None,
-                    INVALID_INT_VALUE,
-                    INVALID_FLOAT_VALUE
-                ]:
-                missing_fields.append(field_name)
-                is_valid = False
-
-            pattern = next(
-                (meta.pattern
-                    for meta in getattr(field_info, "metadata", [])
-                    if hasattr(meta, "pattern")),
-                None,
-            )
-            if pattern and field_value is not None and field_value != NOT_FOUND:
-                invalid_value = None
-                if hasattr(field_info, "json_schema_extra") and field_info.json_schema_extra:
-                    invalid_value = field_info.json_schema_extra.get("invalid_value")
+                is_optional = False
+                attribute_type = field_info.annotation
+                if hasattr(attribute_type, "__origin__") and attribute_type.__origin__ is Union:
+                    union_elements = get_args(attribute_type)
+                    if type(None) in union_elements:
+                        is_optional = True
 
-                if invalid_value and field_value == invalid_value:
-                    invalid_fields.append(f"{field_name} '{field_value}'")
-                    pattern_str = str(pattern)
-                    examples = getattr(field_info, "examples", [])
-                    example = examples[0] if examples else ""
-                    all_suggestions[field_name] = [f"Please use the format matching pattern {pattern_str} (e.g., {example})"]
+                is_required=True
+                if is_optional:
+                        is_required=False
+
+                # Only add to missing fields if it's required AND has no value
+                if is_required and \
+                                field_value in [
+                        NOT_FOUND, 
+                        None,
+                        INVALID_INT_VALUE,
+                        INVALID_FLOAT_VALUE
+                    ]:
+                    missing_fields.append(field_name)
                     is_valid = False
 
-                else:
-                    pattern_regex = re.compile(pattern)
-                    if not pattern_regex.fullmatch(str(field_value)):
-                        invalid_fields.append(f"{field_name} '{field_value}'")
-                        pattern_str = str(pattern)
-                        examples = getattr(field_info, "examples", [])
-                        example = examples[0] if examples else ""
-                        all_suggestions[field_name] = [f"Please use the format matching pattern {pattern_str} (e.g., {example})"]
-                        is_valid = False
-
-
         for field_name, field_info in type(cmd_parameters).model_fields.items():
             field_value = getattr(cmd_parameters, field_name, None)
 
@@ -404,20 +615,20 @@ Today's date is {today}.
         if is_valid:
             if not (
                 self.input_for_param_extraction_class and \
-                hasattr(self.input_for_param_extraction_class, 'validate_extracted_parameters')
+                            hasattr(self.input_for_param_extraction_class, 'validate_extracted_parameters')
             ):
-                return (True, "All required parameters are valid.", {})
-            
+                return (True, "All required parameters are valid.", {}, [])
+
             try:
                 is_valid, message = self.input_for_param_extraction_class.validate_extracted_parameters(app_workflow, subject_command_name, cmd_parameters)
             except Exception as e:
                 message = f"Exception in {subject_command_name}'s validate_extracted_parameters function: {str(e)}"
                 logger.critical(message)                    
-                return (False, message, {})
-            
+                return (False, message, {}, [])
+
             if is_valid:
-                return (True, "All required parameters are valid.", {})
-            return (False, message, {})
+                return (True, "All required parameters are valid.", {}, [])
+            return (False, message, {}, [])
 
         message = ''
         if missing_fields:
@@ -428,27 +639,14 @@ Today's date is {today}.
                 if hasattr(type(cmd_parameters).model_fields.get(missing_field), "json_schema_extra") and type(cmd_parameters).model_fields.get(missing_field).json_schema_extra:
                     is_available_from = type(cmd_parameters).model_fields.get(missing_field).json_schema_extra.get("available_from")
                 if is_available_from:
-                    message += f"abort and use the {' or '.join(is_available_from)} command(s) to get {missing_field} information. OR...\n"
+                    msg_prefix = "abort and "
+                    if "run_as_agent" in app_workflow.context:
+                        msg_prefix = ""
+                    message += f"{msg_prefix}use the {' or '.join(is_available_from)} command(s) to get {missing_field} information. OR...\n"
 
         if invalid_fields:
             message += f"{INVALID_INFORMATION_ERRMSG}" + ", ".join(invalid_fields) + "\n"
 
-        with suppress(Exception):
-            graph_path = os.path.join(app_workflow.folderpath, "command_dependency_graph.json")
-            suggestions_texts: list[str] = []
-            for field in missing_fields:
-                plans = get_dependency_suggestions(graph_path, subject_command_name, field, min_weight=0.7, max_depth=3)
-                if plans:
-                    # Format a concise plan: main command and any immediate sub-steps
-                    top = plans[0]
-                    def format_plan(p):
-                        if not p.get('sub_plans'):
-                            return p['command']
-                        return p['command'] + " -> " + " -> ".join(sp['command'] for sp in p['sub_plans'])
-                    suggestions_texts.append(f"To get '{field}', try: {format_plan(top)}")
-            if suggestions_texts:
-                message += "\n" + "\n".join(suggestions_texts) + "\n"
-
         for field, suggestions in all_suggestions.items():
             if suggestions:
                 is_format_instruction = any(("format" in str(s).lower() or "pattern" in str(s).lower()) for s in suggestions)
@@ -468,6 +666,7 @@ Today's date is {today}.
         if combined_fields:
             combined_fields_str = ", ".join(combined_fields)
             message += f"\nProvide corrected parameter values in the exact order specified below, separated by commas:\n{combined_fields_str}"
-            message += "\nFor parameter values that include a comma, provide separately from other values, and one at a time."
+            if "run_as_agent" not in app_workflow.context:
+                message += "\nFor parameter values that include a comma, provide separately from other values, and one at a time."
 
-        return (False, message, all_suggestions)
+        return (False, message, all_suggestions, combined_fields)

fastworkflow/workflow.py

@@ -288,21 +288,17 @@ class Workflow:
     
     def end_command_processing(self) -> None:
         """Process the end of a command"""
-        mark_dirty = False
         # important to clear the current command from the workflow context
         if "command" in self._context:
             del self._context["command"]
-            mark_dirty = True
 
         # important to clear parameter extraction error state (if any)
         if "stored_parameters" in self._context:
             del self._context["stored_parameters"]
-            mark_dirty = True
 
         self._context["NLU_Pipeline_Stage"] = fastworkflow.NLUPipelineStage.INTENT_DETECTION
 
-        if mark_dirty:
-            self._mark_dirty()
+        self._mark_dirty()
 
     def close(self) -> bool:
         """close the session"""