rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/shared/utils/yaml.py

@@ -1,4 +1,5 @@
 import datetime
+import io
 import logging
 import os
 import re
@@ -8,19 +9,13 @@ from dataclasses import field
 from functools import lru_cache
 from io import StringIO
 from pathlib import Path
-from typing import Dict, List, Optional, Any, Callable, Tuple, Union
+from typing import Any, List, Optional, Tuple, Dict, Callable, Union
 
 import jsonschema
 from importlib_resources import files
 from packaging import version
 from pykwalify.core import Core
 from pykwalify.errors import SchemaError
-from ruamel import yaml as yaml
-from ruamel.yaml import RoundTripRepresenter, YAMLError
-from ruamel.yaml.constructor import DuplicateKeyError, BaseConstructor, ScalarNode
-from ruamel.yaml.comments import CommentedSeq, CommentedMap
-from ruamel.yaml.loader import SafeLoader
-
 from rasa.shared.constants import (
     ASSERTIONS_SCHEMA_EXTENSIONS_FILE,
     ASSERTIONS_SCHEMA_FILE,
@@ -31,6 +26,7 @@ from rasa.shared.constants import (
     LATEST_TRAINING_DATA_FORMAT_VERSION,
     SCHEMA_EXTENSIONS_FILE,
     RESPONSES_SCHEMA_FILE,
+    SENSITIVE_DATA,
 )
 from rasa.shared.exceptions import (
     YamlException,
@@ -50,6 +46,11 @@ from rasa.shared.utils.io import (
     raise_warning,
     read_json_file,
 )
+from ruamel import yaml as yaml
+from ruamel.yaml import YAML, RoundTripRepresenter, YAMLError
+from ruamel.yaml.comments import CommentedSeq, CommentedMap
+from ruamel.yaml.constructor import DuplicateKeyError, BaseConstructor, ScalarNode
+from ruamel.yaml.loader import SafeLoader
 
 logger = logging.getLogger(__name__)
 
@@ -62,8 +63,17 @@ READ_YAML_FILE_CACHE_MAXSIZE = os.environ.get(
 
 @dataclass
 class PathWithError:
+    """Represents a validation error at a specific location in the YAML content.
+
+    Attributes:
+        message (str): A description of the validation error.
+        path (List[str]): Path to the node where the error occurred.
+        key (Optional[str]): The specific key associated with the error, if any.
+    """
+
     message: str
     path: List[str] = field(default_factory=list)
+    key: Optional[str] = None
 
 
 def fix_yaml_loader() -> None:
@@ -87,6 +97,12 @@ def replace_environment_variables() -> None:
     def env_var_constructor(loader: BaseConstructor, node: ScalarNode) -> str:
         """Process environment variables found in the YAML."""
         value = loader.construct_scalar(node)
+
+        # get key of current node
+        key_node = list(loader.constructed_objects)[-1]
+        if isinstance(key_node, ScalarNode) and key_node.value in SENSITIVE_DATA:
+            return value
+
         expanded_vars = os.path.expandvars(value)
         not_expanded = [
             w for w in expanded_vars.split() if w.startswith("$") and w in value
@@ -104,7 +120,6 @@ def replace_environment_variables() -> None:
 
 
 fix_yaml_loader()
-replace_environment_variables()
 
 
 class YamlValidationException(YamlException, ValueError):
@@ -138,21 +153,72 @@ class YamlValidationException(YamlException, ValueError):
         if self.validation_errors:
             unique_errors = {}
             for error in self.validation_errors:
-                line_number = self._line_number_for_path(self.content, error.path)
+                line_number = self._line_number_for_path(
+                    self.content, error.path, error.key
+                )
 
                 if line_number and self.filename:
-                    error_representation = f"  in {self.filename}:{line_number}:\n"
+                    error_location = f"  in {self.filename}:{line_number}:\n"
                 elif line_number:
-                    error_representation = f"  in Line {line_number}:\n"
+                    error_location = f"  in Line {line_number}:\n"
                 else:
-                    error_representation = ""
+                    error_location = ""
 
-                error_representation += f"      {error.message}"
-                unique_errors[error.message] = error_representation
+                code_snippet = self._get_code_snippet(line_number)
+                error_message = f"{error_location}\n{code_snippet}{error.message}\n"
+                unique_errors[error.message] = error_message
             error_msg = "\n".join(unique_errors.values())
             msg += f":\n{error_msg}"
         return msg
 
+    def _get_code_snippet(
+        self,
+        error_line: Optional[int],
+        context_lines: int = 2,
+    ) -> str:
+        """Extract code snippet from the YAML lines around the error.
+
+        Args:
+            error_line: Line number where the error occurred (1-based).
+            context_lines: Number of context lines before and after the error line.
+                Default is 2, balancing context and readability. Adjust as needed.
+
+        Returns:
+            A string containing the code snippet with the error highlighted.
+        """
+        yaml_lines = self._get_serialized_yaml_lines()
+        if not yaml_lines or error_line is None:
+            return ""
+
+        start = max(error_line - context_lines - 1, 0)
+        end = min(error_line + context_lines, len(yaml_lines))
+        snippet_lines = yaml_lines[start:end]
+        snippet = ""
+        for idx, line_content in enumerate(snippet_lines, start=start + 1):
+            prefix = ">>> " if idx == error_line else "    "
+            line_number_str = str(idx)
+            snippet += f"{prefix}{line_number_str} | {line_content}\n"
+        return snippet
+
+    def _get_serialized_yaml_lines(self) -> List[str]:
+        """Serialize the content back to YAML and return the lines."""
+        yaml_lines = []
+        try:
+            yaml = YAML()
+            yaml.default_flow_style = False
+            # Set width to 1000, so we don't break the lines of the original YAML file
+            yaml.width = 1000  # type: ignore[assignment]
+            yaml.indent(mapping=2, sequence=4, offset=2)
+            stream = io.StringIO()
+            yaml.dump(self.content, stream)
+            serialized_yaml = stream.getvalue()
+            yaml_lines = serialized_yaml.splitlines()
+            return yaml_lines
+        except Exception as exc:
+            logger.debug(f"Error serializing YAML content: {exc}")
+
+        return yaml_lines
+
     def _calculate_number_of_lines(
         self,
         current: Union[CommentedSeq, CommentedMap],
@@ -220,7 +286,9 @@ class YamlValidationException(YamlException, ValueError):
         # Return the calculated child offset and True indicating a line number was found
         return child_offset, True
 
-    def _line_number_for_path(self, current: Any, path: List[str]) -> Optional[int]:
+    def _line_number_for_path(
+        self, current: Any, path: List[str], key: Optional[str] = None
+    ) -> Optional[int]:
         """Get line number for a yaml path in the current content.
 
         Implemented using recursion: algorithm goes down the path navigating to the
@@ -229,6 +297,7 @@ class YamlValidationException(YamlException, ValueError):
         Args:
             current: current content
             path: path to traverse within the content
+            key: the key associated with the error, if any
 
         Returns:
             the line number of the path in the content.
@@ -239,6 +308,10 @@ class YamlValidationException(YamlException, ValueError):
         this_line = current.lc.line + 1 if hasattr(current, "lc") else None
 
         if not path:
+            if key and hasattr(current, "lc"):
+                if hasattr(current.lc, "data") and key in current.lc.data:
+                    key_line_no = current.lc.data[key][0] + 1
+                    return key_line_no
             return this_line
 
         head, tail = path[0], path[1:]
@@ -248,7 +321,7 @@ class YamlValidationException(YamlException, ValueError):
 
         if head:
             if isinstance(current, dict) and head in current:
-                line = self._line_number_for_path(current[head], tail)
+                line = self._line_number_for_path(current[head], tail, key)
                 if line is None:
                     line_offset, found_lc = self._calculate_number_of_lines(
                         current, head
@@ -258,14 +331,17 @@ class YamlValidationException(YamlException, ValueError):
                     return this_line + line_offset
                 return line
             elif isinstance(current, list) and head.isdigit():
-                return self._line_number_for_path(current[int(head)], tail) or this_line
+                return (
+                    self._line_number_for_path(current[int(head)], tail, key)
+                    or this_line
+                )
             else:
                 return this_line
-        return self._line_number_for_path(current, tail) or this_line
+        return self._line_number_for_path(current, tail, key) or this_line
 
 
 def read_schema_file(
-    schema_file: str, package_name: str = PACKAGE_NAME
+    schema_file: str, package_name: str = PACKAGE_NAME, expand_env_vars: bool = True
 ) -> Union[List[Any], Dict[str, Any]]:
     """Read a schema file from the package.
 
@@ -273,12 +349,13 @@ def read_schema_file(
         schema_file: The schema file to read.
         package_name: the name of the package the schema is located in. defaults
             to `rasa`.
+        expand_env_vars: Whether to expand environment variables in the file.
 
     Returns:
         The schema as a dictionary.
     """
     schema_path = str(files(package_name).joinpath(schema_file))
-    return read_yaml_file(schema_path)
+    return read_yaml_file(schema_path, expand_env_vars=expand_env_vars)
 
 
 def parse_raw_yaml(raw_yaml_content: str) -> Dict[str, Any]:
@@ -323,13 +400,26 @@ def validate_yaml_content_using_schema(
     try:
         core.validate(raise_exception=True)
     except SchemaError:
+        # PyKwalify propagates each validation error up the data hierarchy, resulting
+        # in multiple redundant errors for a single issue. To present a clear message
+        # about the root cause, we use only the first error.
+        error = core.errors[0]
+
+        # Increment numeric indices by 1 to convert from 0-based to 1-based indexing
+        error_message = re.sub(
+            r"(/)(\d+)", lambda m: f"/{int(m.group(2)) + 1}", str(error)
+        )
+
         raise YamlValidationException(
             "Please make sure the file is correct and all "
             "mandatory parameters are specified. Here are the errors "
             "found during validation",
             [
-                PathWithError(message=str(e), path=e.path.split("/"))
-                for e in core.errors
+                PathWithError(
+                    message=error_message,
+                    path=error.path.removeprefix("/").split("/"),
+                    key=getattr(error, "key", None),
+                )
             ],
             content=yaml_content,
         )
@@ -339,6 +429,7 @@ def validate_raw_yaml_using_schema(
     raw_yaml_content: str,
     schema_content: Dict[str, Any],
     schema_extensions: Optional[List[str]] = None,
+    expand_env_vars: bool = True,
 ) -> None:
     """Validate raw yaml content using a schema.
 
@@ -348,6 +439,7 @@ def validate_raw_yaml_using_schema(
         raw_yaml_content: the raw YAML content to be validated (usually a string)
         schema_content: the schema for the yaml_file_content
         schema_extensions: pykwalify schema extension files
+        expand_env_vars: Whether to expand environment variables.
     """
     try:
         # we need "rt" since
@@ -355,7 +447,11 @@ def validate_raw_yaml_using_schema(
         # will include e.g. at which line an object was parsed. this is very
         # helpful when we validate files later on and want to point the user to the
         # right line
-        yaml_data = read_yaml(raw_yaml_content, reader_type=["safe", "rt"])
+        yaml_data = read_yaml(
+            raw_yaml_content,
+            reader_type=["safe", "rt"],
+            expand_env_vars=expand_env_vars,
+        )
     except (YAMLError, DuplicateKeyError) as e:
         raise YamlSyntaxException(underlying_yaml_exception=e)
 
@@ -363,7 +459,10 @@ def validate_raw_yaml_using_schema(
 
 
 def validate_raw_yaml_using_schema_file(
-    raw_yaml_content: str, schema_path: str, package_name: str = PACKAGE_NAME
+    raw_yaml_content: str,
+    schema_path: str,
+    package_name: str = PACKAGE_NAME,
+    expand_env_vars: bool = True,
 ) -> None:
     """Validate raw yaml content using a schema from file.
 
@@ -372,15 +471,21 @@ def validate_raw_yaml_using_schema_file(
         schema_path: the schema used for validation
         package_name: the name of the package the schema is located in. defaults
             to `rasa`.
+        expand_env_vars: Whether to expand environment variables in the file.
     """
-    schema_content = read_schema_file(schema_path, package_name)
-    validate_raw_yaml_using_schema(raw_yaml_content, schema_content)
+    schema_content = read_schema_file(
+        schema_path, package_name, expand_env_vars=expand_env_vars
+    )
+    validate_raw_yaml_using_schema(
+        raw_yaml_content, schema_content, expand_env_vars=expand_env_vars
+    )
 
 
 def validate_raw_yaml_content_using_schema_with_responses(
     raw_yaml_content: str,
     schema_content: Union[List[Any], Dict[str, Any]],
     package_name: str = PACKAGE_NAME,
+    expand_env_vars: bool = True,
 ) -> None:
     """Validate raw yaml content using a schema with responses sub-schema.
 
@@ -389,18 +494,26 @@ def validate_raw_yaml_content_using_schema_with_responses(
         schema_content: the content of the YAML schema
         package_name: the name of the package the schema is located in. defaults
         to `rasa`.
+        expand_env_vars: Whether to expand environment variables in the file.
     """
     # bot responses are part of the schema extension
     # it will be included if the schema explicitly references it with include: responses
-    bot_responses_schema_content = read_schema_file(RESPONSES_SCHEMA_FILE, package_name)
+    bot_responses_schema_content = read_schema_file(
+        RESPONSES_SCHEMA_FILE, package_name, expand_env_vars=expand_env_vars
+    )
     schema_content = dict(schema_content, **bot_responses_schema_content)
     schema_extensions = [str(files(package_name).joinpath(SCHEMA_EXTENSIONS_FILE))]
 
-    validate_raw_yaml_using_schema(raw_yaml_content, schema_content, schema_extensions)
+    validate_raw_yaml_using_schema(
+        raw_yaml_content, schema_content, schema_extensions, expand_env_vars
+    )
 
 
 def validate_raw_yaml_using_schema_file_with_responses(
-    raw_yaml_content: str, schema_path: str, package_name: str = PACKAGE_NAME
+    raw_yaml_content: str,
+    schema_path: str,
+    package_name: str = PACKAGE_NAME,
+    expand_env_vars: bool = True,
 ) -> None:
     """Validate domain yaml content using a schema from file with responses sub-schema.
 
@@ -409,10 +522,11 @@ def validate_raw_yaml_using_schema_file_with_responses(
         schema_path: the schema of the yaml file
         package_name: the name of the package the schema is located in. defaults
             to `rasa`.
+        expand_env_vars: Whether to expand environment variables in the file.
     """
-    schema_content = read_schema_file(schema_path, package_name)
+    schema_content = read_schema_file(schema_path, package_name, expand_env_vars)
     validate_raw_yaml_content_using_schema_with_responses(
-        raw_yaml_content, schema_content, package_name
+        raw_yaml_content, schema_content, package_name, expand_env_vars
     )
 
 
@@ -432,11 +546,14 @@ def read_yaml(
         ruamel.yaml.parser.ParserError: If there was an error when parsing the YAML.
     """
     custom_constructor = kwargs.get("custom_constructor", None)
+    expand_env_vars = kwargs.get("expand_env_vars", True)
 
     # Create YAML parser with custom constructor
     yaml_parser, reset_constructors = create_yaml_parser(
         reader_type, custom_constructor
     )
+    if expand_env_vars:
+        replace_environment_variables()
     yaml_content = yaml_parser.load(content) or {}
 
     # Reset to default constructors
@@ -492,6 +609,10 @@ def create_yaml_parser(
             yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG,
             original_sequence_constructor,
         )
+        # replace env var constructor with one that does not expand env vars
+        yaml_parser.constructor.add_constructor(
+            "!env_var", lambda loader, node: loader.construct_scalar(node)
+        )
 
     def custom_date_constructor(loader: SafeLoader, node: ScalarNode) -> str:
         """Custom constructor for parsing dates in the format '%Y-%m-%d'.
@@ -526,7 +647,9 @@ def _is_ascii(text: str) -> bool:
 
 @lru_cache(maxsize=READ_YAML_FILE_CACHE_MAXSIZE)
 def read_yaml_file(
-    filename: Union[str, Path], reader_type: Union[str, Tuple[str]] = "safe"
+    filename: Union[str, Path],
+    reader_type: Union[str, Tuple[str]] = "safe",
+    expand_env_vars: bool = True,
 ) -> Union[List[Any], Dict[str, Any]]:
     """Parses a yaml file.
 
@@ -535,6 +658,7 @@ def read_yaml_file(
     Args:
         filename: The path to the file which should be read.
         reader_type: Reader type to use. By default "safe" will be used.
+        expand_env_vars: Whether to expand environment variables in the file.
 
     Returns:
         Parsed content of the file.
@@ -543,7 +667,11 @@ def read_yaml_file(
         fixed_reader_type = (
             list(reader_type) if isinstance(reader_type, tuple) else reader_type
         )
-        return read_yaml(read_file(filename, DEFAULT_ENCODING), fixed_reader_type)
+        return read_yaml(
+            read_file(filename, DEFAULT_ENCODING),
+            fixed_reader_type,
+            expand_env_vars=expand_env_vars,
+        )
     except (YAMLError, DuplicateKeyError) as e:
         raise YamlSyntaxException(filename, e)
 
@@ -566,11 +694,14 @@ def read_config_file(
     return read_validated_yaml(filename, CONFIG_SCHEMA_FILE, reader_type)
 
 
-def read_model_configuration(filename: Union[Path, str]) -> Dict[str, Any]:
+def read_model_configuration(
+    filename: Union[Path, str], expand_env_vars: bool = True
+) -> Dict[str, Any]:
     """Parses a model configuration file.
 
     Args:
         filename: The path to the file which should be read.
+        expand_env_vars: Whether to expand environment variables in the file.
 
     Raises:
         YamlValidationException: In case the model configuration doesn't match the
@@ -579,7 +710,9 @@ def read_model_configuration(filename: Union[Path, str]) -> Dict[str, Any]:
     Returns:
         Parsed config file.
     """
-    return read_validated_yaml(filename, MODEL_CONFIG_SCHEMA_FILE)
+    return read_validated_yaml(
+        filename, MODEL_CONFIG_SCHEMA_FILE, expand_env_vars=expand_env_vars
+    )
 
 
 def dump_obj_as_yaml_to_string(
@@ -677,6 +810,7 @@ def read_validated_yaml(
     filename: Union[str, Path],
     schema: str,
     reader_type: Union[str, List[str]] = "safe",
+    expand_env_vars: bool = True,
 ) -> Any:
     """Validates YAML file content and returns parsed content.
 
@@ -685,6 +819,7 @@ def read_validated_yaml(
         schema: The path to the schema file which should be used for validating the
             file content.
         reader_type: Reader type to use. By default, "safe" will be used.
+        expand_env_vars: Whether to expand environment variables in the file.
 
     Returns:
         The parsed file content.
@@ -695,8 +830,10 @@ def read_validated_yaml(
     """
     content = read_file(filename)
 
-    validate_raw_yaml_using_schema_file(content, schema)
-    return read_yaml(content, reader_type)
+    validate_raw_yaml_using_schema_file(
+        content, schema, expand_env_vars=expand_env_vars
+    )
+    return read_yaml(content, reader_type, expand_env_vars=expand_env_vars)
 
 
 def validate_training_data(json_data: Dict[str, Any], schema: Dict[str, Any]) -> None:
@@ -815,6 +952,7 @@ def validate_yaml_with_jsonschema(
     humanize_error: Callable[
         [jsonschema.ValidationError], str
     ] = default_error_humanizer,
+    expand_env_vars: bool = True,
 ) -> None:
     """Validate data format.
 
@@ -825,6 +963,7 @@ def validate_yaml_with_jsonschema(
             to `rasa`.
         humanize_error: a function to convert a jsonschema.ValidationError into a
             human-readable error message. Defaults to `default_error_humanizer`.
+        expand_env_vars: Whether to expand environment variables in the file.
 
     Raises:
         YamlSyntaxException: if the yaml file is not valid.
@@ -842,7 +981,11 @@ def validate_yaml_with_jsonschema(
         # will include e.g. at which line an object was parsed. this is very
         # helpful when we validate files later on and want to point the user to the
         # right line
-        source_data = read_yaml(yaml_file_content, reader_type=["safe", "rt"])
+        source_data = read_yaml(
+            yaml_file_content,
+            reader_type=["safe", "rt"],
+            expand_env_vars=expand_env_vars,
+        )
     except (YAMLError, DuplicateKeyError) as e:
         raise YamlSyntaxException(underlying_yaml_exception=e)
 

rasa/studio/data_handler.py

@@ -360,7 +360,9 @@ def import_data_from_studio(
     studio_domain.persist(domain_file)
 
     data_from_studio = TrainingDataImporter.load_from_dict(
-        domain_path=str(domain_file), training_data_paths=data_paths
+        domain_path=str(domain_file),
+        training_data_paths=data_paths,
+        expand_env_vars=False,
     )
 
     return data_from_studio, data_original