rasa-pro 3.13.11__py3-none-any.whl → 3.14.0__py3-none-any.whl

rasa/utils/log_utils.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import os
 import sys
-from typing import Any, Optional
+from typing import Any, List, Optional
 
 import structlog
 from structlog.dev import ConsoleRenderer
@@ -19,24 +19,135 @@ from rasa.shared.constants import (
 )
 
 FORCE_JSON_LOGGING = os.environ.get("FORCE_JSON_LOGGING")
+ANSI_CYAN_BOLD = "\033[1;36m"
+ANSI_RESET = "\033[0m"
+
+
+def conditional_set_exc_info(
+    logger: WrappedLogger, name: str, event_dict: EventDict
+) -> EventDict:
+    """Set exception info only if exception does not have suppress_stack_trace flag."""
+    exc_info = event_dict.get("exc_info")
+    if exc_info is not None:
+        is_debug_mode = logger.isEnabledFor(logging.DEBUG)
+
+        if (
+            hasattr(exc_info, "suppress_stack_trace")
+            and exc_info.suppress_stack_trace
+            and not is_debug_mode
+        ):
+            event_dict.pop("exc_info", None)
+        else:
+            return structlog.dev.set_exc_info(logger, name, event_dict)
+    return event_dict
 
 
 class HumanConsoleRenderer(ConsoleRenderer):
     """Console renderer that outputs human-readable logs."""
 
     def __call__(self, logger: WrappedLogger, name: str, event_dict: EventDict) -> str:
+        should_highlight = event_dict.get("highlight", False)
+        terminal_width = self._get_terminal_width()
+
+        # Use event_info as title for the log entry
         if "event_info" in event_dict:
             event_key = event_dict["event"]
             event_dict["event"] = event_dict["event_info"]
             event_dict["event_key"] = event_key
-            del event_dict["event_info"]
-
-        return super().__call__(logger, name, event_dict)
+            event_dict.pop("event_info", None)
+
+        # In case the log entry should be highlighted
+        # make sure to surround the log entry with ===
+        event_dict = self._highlight_log_entry(
+            event_dict, terminal_width, should_highlight
+        )
+
+        # Format JSON data for better readability
+        event_dict = self._format_json_data(event_dict)
+
+        # Render the log entry first
+        result = super().__call__(logger, name, event_dict)
+
+        # ensure that newlines are properly rendered
+        result = "\n".join(result.split("\\n"))
+
+        # Add closing === if we highlighted this entry
+        if should_highlight:
+            result += f"\n{'=' * terminal_width}"
+
+        return result
+
+    def _highlight_log_entry(
+        self, event_dict: EventDict, terminal_width: int, should_highlight: bool
+    ) -> EventDict:
+        if should_highlight:
+            # Only highlight if log level is DEBUG
+            # structlog passes log level as 'level' or 'levelname'
+            level = event_dict.get("level", event_dict.get("levelname", "")).upper()
+            if level == "DEBUG":
+                # Add opening === before the event info
+                if "event" in event_dict and isinstance(event_dict["event"], str):
+                    event_info = event_dict["event"]
+                    event_dict["event"] = (
+                        f"\n{'=' * terminal_width}\n"
+                        f"{ANSI_CYAN_BOLD}{event_info}{ANSI_RESET}\n"
+                    )
+
+            event_dict.pop("highlight", None)
+
+        return event_dict
+
+    def _format_json_data(self, event_dict: EventDict) -> EventDict:
+        """Format JSON data in the event dict for better readability."""
+        # Get the list of fields to format from the event dict
+        fields_to_format = event_dict.get("json_formatting", [])
+
+        if not fields_to_format:
+            return event_dict
+
+        import json
+
+        # Format only the specified fields
+        for key in fields_to_format:
+            if key in event_dict:
+                value = event_dict[key]
+
+                try:
+                    # Try to parse as JSON if it's a string
+                    if isinstance(value, str):
+                        parsed = json.loads(value)
+                        # If it's a dict or list, format it nicely
+                        if isinstance(parsed, (dict, list)):
+                            formatted = json.dumps(parsed, indent=2, ensure_ascii=False)
+                            event_dict[key] = formatted
+                    elif isinstance(value, (dict, list)):
+                        # Format JSON with indentation for better readability
+                        formatted = json.dumps(value, indent=2, ensure_ascii=False)
+                        event_dict[key] = formatted
+                except (TypeError, ValueError, json.JSONDecodeError):
+                    # If it's not JSON serializable or if it's not valid JSON,
+                    # leave it as is
+                    pass
+
+        # Remove the json_formatting key from the output
+        event_dict.pop("json_formatting", None)
+
+        return event_dict
+
+    def _get_terminal_width(self) -> int:
+        """Get the width of the terminal."""
+        import shutil
+
+        try:
+            return shutil.get_terminal_size((80, 20)).columns
+        except Exception:
+            return 80
 
 
 def configure_structlog(
     log_level: Optional[int] = None,
     include_time: bool = False,
+    additional_processors: Optional[List[structlog.typing.Processor]] = None,
 ) -> None:
     """Configure logging of the server."""
     if log_level is None:  # Log level NOTSET is 0 so we use `is None` here
@@ -51,7 +162,7 @@ def configure_structlog(
         level=log_level,
     )
 
-    shared_processors = [
+    shared_processors: List[structlog.typing.Processor] = [
         # Processors that have nothing to do with output,
         # e.g., add timestamps or log level names.
         # If log level is too low, abort pipeline and throw away log entry.
@@ -66,7 +177,7 @@ def configure_structlog(
         structlog.processors.StackInfoRenderer(),
         # If some value is in bytes, decode it to a unicode str.
         structlog.processors.UnicodeDecoder(),
-        structlog.dev.set_exc_info,
+        conditional_set_exc_info,
         # add structlog sentry integration. only log fatal log entries
         # as events as we are tracking exceptions anyways
         SentryProcessor(event_level=logging.FATAL),
@@ -75,6 +186,9 @@ def configure_structlog(
     if include_time:
         shared_processors.append(structlog.processors.TimeStamper(fmt="iso"))
 
+    if additional_processors:
+        shared_processors.extend(additional_processors)
+
     if not FORCE_JSON_LOGGING and sys.stderr.isatty():
         # Pretty printing when we run in a terminal session.
         # Automatically prints pretty tracebacks when "rich" is installed
@@ -90,7 +204,7 @@ def configure_structlog(
         ]
 
     structlog.configure(
-        processors=processors,  # type: ignore
+        processors=processors,
         context_class=dict,
         # `logger_factory` is used to create wrapped loggers that are used for
         # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON

rasa/utils/ml_utils.py

@@ -119,7 +119,7 @@ def response_for_template(
     rsps = responses.get(template_name, [])
     usuable_responses = [r for r in rsps if r.get(KEY_RESPONSES_TEXT)]
     if usuable_responses:
-        selected_response = np.random.choice(usuable_responses)  # type: ignore
+        selected_response = np.random.choice(usuable_responses)
         return selected_response.get(KEY_RESPONSES_TEXT)
     else:
         structlogger.warning(

rasa/utils/openapi.py

@@ -0,0 +1,144 @@
+from typing import Any, Dict, List, Type
+
+from pydantic.main import BaseModel
+from sanic_openapi import openapi
+from sanic_openapi.openapi3.types import Schema
+
+_SUPPORTED_ATTRIBUTES = frozenset(["format", "enum", "required", "example"])
+
+
+def _to_schema(
+    definition_stack: List[str], schema_def: Dict[str, Any], definitions: Dict[str, Any]
+) -> Schema:
+    type = schema_def.get("type")
+
+    if type == "object":
+        properties_spec = schema_def.get("properties", {})
+        properties = {}
+        for key in properties_spec:
+            properties[key] = _to_schema(
+                definition_stack=definition_stack,
+                schema_def=properties_spec[key],
+                definitions=definitions,
+            )
+        schema = openapi.Object(
+            title=schema_def.get("title"),
+            description=schema_def.get("description"),
+            required=schema_def.get("required"),
+            properties=properties,
+        )
+    elif type == "array":
+        schema = openapi.Array(
+            description=schema_def.get("description"),
+            required=schema_def.get("required"),
+            items=_to_schema(
+                definition_stack=definition_stack,
+                schema_def=schema_def.get("items"),
+                definitions=definitions,
+            ),
+        )
+    elif type is None:
+        if allof_spec := schema_def.get("allOf"):  # Model, Enum
+            definition = allof_spec[0]["$ref"].split("/")[-1]
+            definition_data = definitions.get(definition)
+            if definition_data is None:
+                schema = openapi.Object(
+                    title=definition, description=schema_def.get("description")
+                )
+            else:
+                schema = (
+                    _to_schema(
+                        definition_stack=definition_stack + [definition],
+                        schema_def={**definition_data},
+                        definitions=definitions,
+                    )
+                    if definition not in definition_stack
+                    else openapi.Object(
+                        title=definition, description=schema_def.get("description")
+                    )
+                )
+
+        elif anyof_spec := schema_def.get("anyOf"):  # Union
+            anyof = []
+            for any in anyof_spec:
+                if any.get("type"):
+                    schema_type_obj = Schema(
+                        **{
+                            "type": any.get("type"),
+                            "description": any.get("description"),
+                        }
+                    )
+                    anyof.append(schema_type_obj)
+                else:
+                    definition = any["$ref"].split("/")[-1]
+                    if definition not in definition_stack:
+                        definition_data = definitions.get(definition)
+                        if definition_data is not None:
+                            anyof.append(
+                                _to_schema(
+                                    definition_stack=definition_stack + [definition],
+                                    schema_def=definition_data,
+                                    definitions=definitions,
+                                )
+                            )
+                        else:
+                            anyof.append(
+                                openapi.Object(
+                                    title=definition,
+                                    description=schema_def.get(
+                                        "description", definition
+                                    ),
+                                    properties={},
+                                )
+                            )
+                    else:
+                        anyof.append(
+                            openapi.Object(
+                                title=definition,
+                                description=schema_def.get("description", definition),
+                                properties={},
+                            )
+                        )
+            schema = Schema(anyOf=anyof)
+        elif ref := schema_def.get("$ref"):  # $ref
+            definition = ref.split("/")[-1]
+            definition_data = definitions.get(definition)
+            if definition_data is not None:
+                schema = _to_schema(
+                    definition_stack=definition_stack,
+                    schema_def=definition_data,
+                    definitions=definitions,
+                )
+            else:
+                schema = openapi.Object(
+                    title=definition, description=schema_def.get("description")
+                )
+        else:  # Any type
+            schema = Schema(
+                **{"type": "object", "description": schema_def.get("description")}
+            )
+
+    else:
+        schema_spec = {
+            "type": schema_def.get("type"),
+            "description": schema_def.get("description"),
+        }
+        for spec in _SUPPORTED_ATTRIBUTES:
+            if schema_def.get(spec):
+                schema_spec[spec] = schema_def.get(spec)
+        schema = Schema(**schema_spec)
+
+    return schema
+
+
+def model_to_schema(model: Type[BaseModel]) -> Schema:
+    schema = model.model_json_schema()
+    # Handle both $defs (newer JSON Schema) and definitions (older JSON Schema)
+    definitions = schema.get("$defs") or schema.get("definitions") or {}
+    return _to_schema(
+        definition_stack=[],
+        schema_def=dict(
+            filter(lambda key: key[0] not in ("definitions", "$defs"), schema.items())
+        ),
+        definitions=definitions,
+    )

rasa/utils/plotting.py

@@ -99,7 +99,7 @@ def plot_confusion_matrix(
     zmax = confusion_matrix.max() if len(confusion_matrix) > 0 else 1
     plt.clf()
     if not color_map:
-        color_map = plt.cm.Blues
+        color_map = plt.cm.get_cmap("Blues")
     plt.imshow(
         confusion_matrix,
         interpolation="nearest",

rasa/utils/pypred.py

@@ -0,0 +1,45 @@
+"""Wrapper module for pypred that provides a fixed Predicate class.
+
+This module should be used instead of importing directly from pypred.
+
+This patch fixes an issue where pypred creates excessive logs of being unable
+to write a file when run in an environment with no write access is given.
+
+https://rasahq.atlassian.net/browse/ATO-1925
+
+The solution is based on https://github.com/FreeCAD/FreeCAD/issues/6315
+"""
+
+import logging
+from typing import Any
+
+import ply.yacc
+import pypred.parser
+from pypred import Predicate as OriginalPredicate  # noqa: TID251
+
+# Store the original yacc function
+_original_yacc = ply.yacc.yacc
+
+# Create a logger that suppresses warnings to avoid yacc table file version warnings
+_yacc_logger = logging.getLogger("ply.yacc")
+_yacc_logger.setLevel(logging.ERROR)
+
+
+def patched_yacc(*args: Any, **kwargs: Any) -> Any:
+    # Disable generation of debug ('parser.out') and table
+    # cache ('parsetab.py'), as it requires a writable location.
+    kwargs["write_tables"] = False
+    kwargs["module"] = pypred.parser
+    # Suppress yacc warnings by using a logger that only shows errors
+    kwargs["errorlog"] = _yacc_logger
+    return _original_yacc(*args, **kwargs)
+
+
+# Apply the patch
+ply.yacc.yacc = patched_yacc
+
+
+class Predicate(OriginalPredicate):
+    """Fixed version of pypred.Predicate that uses the patched yacc parser."""
+
+    pass

rasa/utils/tensorflow/__init__.py

@@ -0,0 +1,7 @@
+import importlib.util
+import logging
+
+logger = logging.getLogger(__name__)
+
+# check if TensorFlow is available
+TENSORFLOW_AVAILABLE = importlib.util.find_spec("tensorflow") is not None

rasa/utils/tensorflow/callback.py

@@ -2,111 +2,146 @@ import logging
 from pathlib import Path
 from typing import Any, Dict, Optional, Text
 
-import tensorflow as tf
-from tqdm import tqdm
+from rasa.utils.tensorflow import TENSORFLOW_AVAILABLE
+
+if TENSORFLOW_AVAILABLE:
+    import tensorflow as tf
+    from tqdm import tqdm
+else:
+    # Placeholder values when TensorFlow is not available
+    tf = None
+    tqdm = None
 
 import rasa.shared.utils.io
 
 logger = logging.getLogger(__name__)
 
 
-class RasaTrainingLogger(tf.keras.callbacks.Callback):
-    """Callback for logging the status of training."""
-
-    def __init__(self, epochs: int, silent: bool) -> None:
-        """Initializes the callback.
-
-        Args:
-            epochs: Total number of epochs.
-            silent: If 'True' the entire progressbar wrapper is disabled.
-        """
-        super().__init__()
-
-        disable = silent or rasa.shared.utils.io.is_logging_disabled()
-        self.progress_bar = tqdm(range(epochs), desc="Epochs", disable=disable)
-
-    def on_epoch_end(self, epoch: int, logs: Optional[Dict[Text, Any]] = None) -> None:
-        """Updates the logging output on every epoch end.
-
-        Args:
-            epoch: The current epoch.
-            logs: The training metrics.
-        """
-        self.progress_bar.update(1)
-        self.progress_bar.set_postfix(logs)
-
-    def on_train_end(self, logs: Optional[Dict[Text, Any]] = None) -> None:
-        """Closes the progress bar after training.
-
-        Args:
-            logs: The training metrics.
-        """
-        self.progress_bar.close()
-
-
-class RasaModelCheckpoint(tf.keras.callbacks.Callback):
-    """Callback for saving intermediate model checkpoints."""
-
-    def __init__(self, checkpoint_dir: Path) -> None:
-        """Initializes the callback.
-
-        Args:
-            checkpoint_dir: Directory to store checkpoints to.
-        """
-        super().__init__()
-
-        self.checkpoint_file = checkpoint_dir / "checkpoint.tf_model"
-        self.best_metrics_so_far: Dict[Text, Any] = {}
-
-    def on_epoch_end(self, epoch: int, logs: Optional[Dict[Text, Any]] = None) -> None:
-        """Save the model on epoch end if the model has improved.
-
-        Args:
-            epoch: The current epoch.
-            logs: The training metrics.
-        """
-        if self._does_model_improve(logs):
-            logger.debug(f"Creating model checkpoint at epoch={epoch + 1} ...")
-            self.model.save_weights(
-                self.checkpoint_file, overwrite=True, save_format="tf"
-            )
-
-    def _does_model_improve(self, curr_results: Dict[Text, Any]) -> bool:
-        """Checks whether the current results are better than the best so far.
-
-        Results are considered better if each metric is equal or better than the best so
-        far, and at least one is better.
-
-        Args:
-            curr_results: The training metrics for this epoch.
-        """
-        curr_metric_names = [
-            k
-            for k in curr_results.keys()
-            if k.startswith("val") and (k.endswith("_acc") or k.endswith("_f1"))
-        ]
-        # the "val" prefix is prepended to metrics in fit if _should_eval returns true
-        # for this particular epoch
-        if len(curr_metric_names) == 0:
-            # the metrics are not validation metrics
-            return False
-        # initialize best_metrics_so_far with the first results
-        if not self.best_metrics_so_far:
-            for metric_name in curr_metric_names:
-                self.best_metrics_so_far[metric_name] = float(curr_results[metric_name])
-            return True
-
-        at_least_one_improved = False
-        improved_metrics = {}
-        for metric_name in self.best_metrics_so_far.keys():
-            if float(curr_results[metric_name]) < self.best_metrics_so_far[metric_name]:
-                # at least one of the values is worse
+if TENSORFLOW_AVAILABLE:
+
+    class RasaTrainingLogger(tf.keras.callbacks.Callback):
+        """Callback for logging the status of training."""
+
+        def __init__(self, epochs: int, silent: bool) -> None:
+            """Initializes the callback.
+
+            Args:
+                epochs: Total number of epochs.
+                silent: If 'True' the entire progressbar wrapper is disabled.
+            """
+            super().__init__()
+
+            disable = silent or rasa.shared.utils.io.is_logging_disabled()
+            self.progress_bar = tqdm(range(epochs), desc="Epochs", disable=disable)
+
+        def on_epoch_end(
+            self, epoch: int, logs: Optional[Dict[Text, Any]] = None
+        ) -> None:
+            """Updates the logging output on every epoch end.
+
+            Args:
+                epoch: The current epoch.
+                logs: The training metrics.
+            """
+            self.progress_bar.update(1)
+            self.progress_bar.set_postfix(logs)
+
+        def on_train_end(self, logs: Optional[Dict[Text, Any]] = None) -> None:
+            """Closes the progress bar after training.
+
+            Args:
+                logs: The training metrics.
+            """
+            self.progress_bar.close()
+
+    class RasaModelCheckpoint(tf.keras.callbacks.Callback):
+        """Callback for saving intermediate model checkpoints."""
+
+        def __init__(self, checkpoint_dir: Path) -> None:
+            """Initializes the callback.
+
+            Args:
+                checkpoint_dir: Directory to store checkpoints to.
+            """
+            super().__init__()
+
+            self.checkpoint_file = checkpoint_dir / "checkpoint.weights.h5"
+            self.best_metrics_so_far: Dict[Text, Any] = {}
+
+        def on_epoch_end(
+            self, epoch: int, logs: Optional[Dict[Text, Any]] = None
+        ) -> None:
+            """Save the model on epoch end if the model has improved.
+
+            Args:
+                epoch: The current epoch.
+                logs: The training metrics.
+            """
+            if self._does_model_improve(logs):
+                logger.debug(f"Creating model checkpoint at epoch={epoch + 1} ...")
+                # Ensure model is built before saving weights
+                if not self.model.built:
+                    # Build the model with dummy data to ensure it's built
+                    import tensorflow as tf
+
+                    dummy_input = tf.zeros((1, 1))
+                    _ = self.model(dummy_input)
+
+                # Ensure the directory exists before saving
+                import os
+
+                os.makedirs(os.path.dirname(self.checkpoint_file), exist_ok=True)
+                self.model.save_weights(self.checkpoint_file, overwrite=True)
+
+        def _does_model_improve(self, curr_results: Dict[Text, Any]) -> bool:
+            """Checks whether the current results are better than the best so far.
+
+            Results are considered better if each metric is
+            equal or better than the best so far, and at least one is better.
+
+            Args:
+                curr_results: The training metrics for this epoch.
+            """
+            curr_metric_names = [
+                k
+                for k in curr_results.keys()
+                if k.startswith("val") and (k.endswith("_acc") or k.endswith("_f1"))
+            ]
+            # the "val" prefix is prepended to metrics in fit
+            # if _should_eval returns true
+            # for this particular epoch
+            if len(curr_metric_names) == 0:
+                # the metrics are not validation metrics
                 return False
-            if float(curr_results[metric_name]) > self.best_metrics_so_far[metric_name]:
-                at_least_one_improved = True
-                improved_metrics[metric_name] = float(curr_results[metric_name])
-
-        # all current values >= previous best and at least one is better
-        if at_least_one_improved:
-            self.best_metrics_so_far.update(improved_metrics)
-        return at_least_one_improved
+            # initialize best_metrics_so_far with the first results
+            if not self.best_metrics_so_far:
+                for metric_name in curr_metric_names:
+                    self.best_metrics_so_far[metric_name] = float(
+                        curr_results[metric_name]
+                    )
+                    return True
+
+            at_least_one_improved = False
+            improved_metrics = {}
+            for metric_name in self.best_metrics_so_far.keys():
+                if (
+                    float(curr_results[metric_name])
+                    < self.best_metrics_so_far[metric_name]
+                ):
+                    # at least one of the values is worse
+                    return False
+                if (
+                    float(curr_results[metric_name])
+                    > self.best_metrics_so_far[metric_name]
+                ):
+                    at_least_one_improved = True
+                    improved_metrics[metric_name] = float(curr_results[metric_name])
+
+            # all current values >= previous best and at least one is better
+            if at_least_one_improved:
+                self.best_metrics_so_far.update(improved_metrics)
+            return at_least_one_improved
+else:
+    # Placeholder classes when TensorFlow is not available
+    RasaTrainingLogger = None  # type: ignore
+    RasaModelCheckpoint = None  # type: ignore

rasa/utils/tensorflow/crf.py

@@ -9,7 +9,7 @@ from tensorflow.types.experimental import TensorLike
 # (modified to our neeeds)
 
 
-class CrfDecodeForwardRnnCell(tf.keras.layers.AbstractRNNCell):
+class CrfDecodeForwardRnnCell(tf.keras.layers.Layer):
     """Computes the forward decoding in a linear-chain CRF."""
 
     def __init__(self, transition_params: TensorLike, **kwargs: Any) -> None: