rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/shared/utils/yaml.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 import os
 import re
@@ -12,15 +13,17 @@ from typing import Dict, List, Optional, Any, Callable, Tuple, Union
 import jsonschema
 from importlib_resources import files
 from packaging import version
-from packaging.version import LegacyVersion
 from pykwalify.core import Core
 from pykwalify.errors import SchemaError
 from ruamel import yaml as yaml
 from ruamel.yaml import RoundTripRepresenter, YAMLError
 from ruamel.yaml.constructor import DuplicateKeyError, BaseConstructor, ScalarNode
 from ruamel.yaml.comments import CommentedSeq, CommentedMap
+from ruamel.yaml.loader import SafeLoader
 
 from rasa.shared.constants import (
+    ASSERTIONS_SCHEMA_EXTENSIONS_FILE,
+    ASSERTIONS_SCHEMA_FILE,
     MODEL_CONFIG_SCHEMA_FILE,
     CONFIG_SCHEMA_FILE,
     DOCS_URL_TRAINING_DATA,
@@ -413,30 +416,108 @@ def validate_raw_yaml_using_schema_file_with_responses(
     )
 
 
-def read_yaml(content: str, reader_type: Union[str, List[str]] = "safe") -> Any:
+def read_yaml(
+    content: str,
+    reader_type: Union[str, List[str]] = "safe",
+    **kwargs: Any,
+) -> Any:
     """Parses yaml from a text.
 
     Args:
         content: A text containing yaml content.
         reader_type: Reader type to use. By default, "safe" will be used.
+        **kwargs: Any
 
     Raises:
         ruamel.yaml.parser.ParserError: If there was an error when parsing the YAML.
     """
-    if _is_ascii(content):
-        # Required to make sure emojis are correctly parsed
-        content = (
-            content.encode("utf-8")
-            .decode("raw_unicode_escape")
-            .encode("utf-16", "surrogatepass")
-            .decode("utf-16")
-        )
+    custom_constructor = kwargs.get("custom_constructor", None)
+
+    # Create YAML parser with custom constructor
+    yaml_parser, reset_constructors = create_yaml_parser(
+        reader_type, custom_constructor
+    )
+    yaml_content = yaml_parser.load(content) or {}
+
+    # Reset to default constructors
+    reset_constructors()
+
+    return yaml_content
+
+
+def create_yaml_parser(
+    reader_type: str,
+    custom_constructor: Optional[Callable] = None,
+) -> Tuple[yaml.YAML, Callable[[], None]]:
+    """Create a YAML parser with an optional custom constructor.
+
+    Args:
+        reader_type (str): The type of the reader
+        (e.g., 'safe', 'rt', 'unsafe').
+        custom_constructor (Optional[Callable]):
+        A custom constructor function for YAML parsing.
 
+    Returns:
+        Tuple[yaml.YAML, Callable[[], None]]: A tuple containing
+        the YAML parser and a function to reset constructors to
+        their original state.
+    """
     yaml_parser = yaml.YAML(typ=reader_type)
     yaml_parser.version = YAML_VERSION  # type: ignore[assignment]
     yaml_parser.preserve_quotes = True  # type: ignore[assignment]
 
-    return yaml_parser.load(content) or {}
+    # Save the original constructors
+    original_mapping_constructor = yaml_parser.constructor.yaml_constructors.get(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
+    )
+    original_sequence_constructor = yaml_parser.constructor.yaml_constructors.get(
+        yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG
+    )
+
+    if custom_constructor is not None:
+        # Attach the custom constructor to the loader
+        yaml_parser.constructor.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, custom_constructor
+        )
+        yaml_parser.constructor.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, custom_constructor
+        )
+
+    def reset_constructors() -> None:
+        """Reset the constructors back to their original state."""
+        yaml_parser.constructor.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, original_mapping_constructor
+        )
+        yaml_parser.constructor.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG,
+            original_sequence_constructor,
+        )
+
+    def custom_date_constructor(loader: SafeLoader, node: ScalarNode) -> str:
+        """Custom constructor for parsing dates in the format '%Y-%m-%d'.
+
+        This constructor parses dates in the '%Y-%m-%d' format and returns them as
+        strings instead of datetime objects. This change was introduced because the
+        default timestamp constructor in ruamel.yaml returns datetime objects, which
+        caused issues in our use case where the `api_version` in the LLM config must
+        be a string, but was being interpreted as a datetime object.
+        """
+        value = loader.construct_scalar(node)
+        try:
+            # Attempt to parse the date
+            date_obj = datetime.datetime.strptime(value, "%Y-%m-%d").date()
+            # Return the date as a string instead of a datetime object
+            return date_obj.strftime("%Y-%m-%d")
+        except ValueError:
+            # If the date is not in the correct format, return the original value
+            return value
+
+    # Add the custom date constructor
+    yaml_parser.constructor.add_constructor(
+        "tag:yaml.org,2002:timestamp", custom_date_constructor
+    )
+
+    return yaml_parser, reset_constructors
 
 
 def _is_ascii(text: str) -> bool:
@@ -684,9 +765,6 @@ def validate_training_data_format_version(
         parsed_version = version.parse(version_value)
         latest_version = version.parse(LATEST_TRAINING_DATA_FORMAT_VERSION)
 
-        if isinstance(parsed_version, LegacyVersion):
-            raise TypeError
-
         if parsed_version < latest_version:
             raise_warning(
                 f"Training data file {filename} has a lower "
@@ -702,7 +780,7 @@ def validate_training_data_format_version(
         if latest_version >= parsed_version:
             return True
 
-    except TypeError:
+    except (TypeError, version.InvalidVersion):
         raise_warning(
             f"Training data file {filename} must specify "
             f"'{KEY_TRAINING_DATA_FORMAT_VERSION}' as string, for example:\n"
@@ -784,3 +862,31 @@ def validate_yaml_with_jsonschema(
             errors,
             content=source_data,
         )
+
+
+def validate_yaml_data_using_schema_with_assertions(
+    yaml_data: Any,
+    schema_content: Union[List[Any], Dict[str, Any]],
+    package_name: str = PACKAGE_NAME,
+) -> None:
+    """Validate raw yaml content using a schema with assertions sub-schema.
+
+    Args:
+        yaml_data: the parsed yaml data to be validated
+        schema_content: the content of the YAML schema
+        package_name: the name of the package the schema is located in. defaults
+        to `rasa`.
+    """
+    # test case assertions are part of the schema extension
+    # it will be included if the schema explicitly references it with
+    # include: assertions
+    e2e_test_cases_schema_content = read_schema_file(
+        ASSERTIONS_SCHEMA_FILE, package_name
+    )
+
+    schema_content = dict(schema_content, **e2e_test_cases_schema_content)
+    schema_extensions = [
+        str(files(package_name).joinpath(ASSERTIONS_SCHEMA_EXTENSIONS_FILE))
+    ]
+
+    validate_yaml_content_using_schema(yaml_data, schema_content, schema_extensions)

rasa/studio/auth.py

@@ -21,15 +21,17 @@ from rasa.studio.results_logger import with_studio_error_handler, StudioResult
 
 
 class StudioAuth:
-    def __init__(
-        self,
-        studio_config: StudioConfig,
-    ) -> None:
+    """Handles the authentication with the Rasa Studio authentication server."""
+
+    def __init__(self, studio_config: StudioConfig) -> None:
         self.config = studio_config
+        verify = not studio_config.disable_verify
+
         self.keycloak_openid = KeycloakOpenID(
             server_url=studio_config.authentication_server_url,
             client_id=studio_config.client_id,
             realm_name=studio_config.realm_name,
+            verify=verify,
         )
 
     def health_check(self) -> bool:

rasa/studio/config.py

@@ -2,13 +2,14 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass
-from typing import Dict, Optional, Text
+from typing import Any, Dict, Optional, Text
 
 from rasa.utils.common import read_global_config_value, write_global_config_value
 
 from rasa.studio.constants import (
     RASA_STUDIO_AUTH_SERVER_URL_ENV,
     RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV,
+    RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV,
     RASA_STUDIO_CLI_REALM_NAME_KEY_ENV,
     RASA_STUDIO_CLI_STUDIO_URL_ENV,
     STUDIO_CONFIG_KEY,
@@ -19,6 +20,7 @@ STUDIO_URL_KEY = "studio_url"
 CLIENT_ID_KEY = "client_id"
 REALM_NAME_KEY = "realm_name"
 CLIENT_SECRET_KEY = "client_secret"
+DISABLE_VERIFY = "disable_verify"
 
 
 @dataclass
@@ -27,13 +29,15 @@ class StudioConfig:
     studio_url: Optional[Text]
     client_id: Optional[Text]
     realm_name: Optional[Text]
+    disable_verify: bool = False
 
-    def to_dict(self) -> Dict[Text, Optional[Text]]:
+    def to_dict(self) -> Dict[Text, Optional[Any]]:
         return {
             AUTH_SERVER_URL_KEY: self.authentication_server_url,
             STUDIO_URL_KEY: self.studio_url,
             CLIENT_ID_KEY: self.client_id,
             REALM_NAME_KEY: self.realm_name,
+            DISABLE_VERIFY: self.disable_verify,
         }
 
     @classmethod
@@ -43,6 +47,7 @@ class StudioConfig:
             studio_url=data[STUDIO_URL_KEY],
             client_id=data[CLIENT_ID_KEY],
             realm_name=data[REALM_NAME_KEY],
+            disable_verify=data.get(DISABLE_VERIFY, False),
         )
 
     def write_config(self) -> None:
@@ -73,7 +78,7 @@ class StudioConfig:
         config = read_global_config_value(STUDIO_CONFIG_KEY, unavailable_ok=True)
 
         if config is None:
-            return StudioConfig(None, None, None, None)
+            return StudioConfig(None, None, None, None, False)
 
         if not isinstance(config, dict):
             raise ValueError(
@@ -83,7 +88,7 @@ class StudioConfig:
             )
 
         for key in config:
-            if not isinstance(config[key], str):
+            if not isinstance(config[key], str) and key != DISABLE_VERIFY:
                 raise ValueError(
                     "Invalid config file format. "
                     f"Key '{key}' is not a text value."
@@ -102,6 +107,9 @@ class StudioConfig:
             studio_url=StudioConfig._read_env_value(RASA_STUDIO_CLI_STUDIO_URL_ENV),
             client_id=StudioConfig._read_env_value(RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV),
             realm_name=StudioConfig._read_env_value(RASA_STUDIO_CLI_REALM_NAME_KEY_ENV),
+            disable_verify=bool(
+                os.getenv(RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV, False)
+            ),
         )
 
     @staticmethod
@@ -124,4 +132,5 @@ class StudioConfig:
             studio_url=self.studio_url or other.studio_url,
             client_id=self.client_id or other.client_id,
             realm_name=self.realm_name or other.realm_name,
+            disable_verify=self.disable_verify or other.disable_verify,
         )

rasa/studio/constants.py

@@ -10,6 +10,7 @@ RASA_STUDIO_AUTH_SERVER_URL_ENV = "RASA_STUDIO_AUTH_SERVER_URL"
 RASA_STUDIO_CLI_STUDIO_URL_ENV = "RASA_STUDIO_CLI_STUDIO_URL"
 RASA_STUDIO_CLI_REALM_NAME_KEY_ENV = "RASA_STUDIO_CLI_REALM_NAME_KEY"
 RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV = "RASA_STUDIO_CLI_CLIENT_ID_KEY"
+RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV = "RASA_STUDIO_CLI_DISABLE_VERIFY_KEY"
 
 STUDIO_NLU_FILENAME = "studio_nlu.yml"
 STUDIO_DOMAIN_FILENAME = "studio_domain.yml"

rasa/studio/data_handler.py

@@ -76,7 +76,9 @@ class StudioDataHandler:
 
         return request
 
-    def _make_request(self, GQL_req: Dict[Any, Any]) -> Dict[Any, Any]:
+    def _make_request(
+        self, GQL_req: Dict[Any, Any], verify: bool = True
+    ) -> Dict[Any, Any]:
         token = KeycloakTokenReader().get_token()
         if token.is_expired():
             token = self.refresh_token(token)
@@ -93,6 +95,7 @@ class StudioDataHandler:
                 "Authorization": f"{token.token_type} {token.access_token}",
                 "Content-Type": "application/json",
             },
+            verify=verify,
         )
 
         if res.status_code != 200:
@@ -128,7 +131,9 @@ class StudioDataHandler:
             The data from Rasa Studio.
         """
         GQL_req = self._build_request()
-        response = self._make_request(GQL_req)
+        verify = not self.studio_config.disable_verify
+
+        response = self._make_request(GQL_req, verify=verify)
         self._extract_data(response)
 
     def request_data(
@@ -145,7 +150,9 @@ class StudioDataHandler:
             The data from Rasa Studio.
         """
         GQL_req = self._build_request(intent_names, entity_names)
-        response = self._make_request(GQL_req)
+        verify = not self.studio_config.disable_verify
+
+        response = self._make_request(GQL_req, verify=verify)
         self._extract_data(response)
 
     def get_config(self) -> Optional[str]:

rasa/studio/download.py

@@ -210,8 +210,8 @@ def _handle_download_no_overwrite(
     if domain_path.is_dir():
         studio_domain_path = domain_path / STUDIO_DOMAIN_FILENAME
         new_domain_data = data_handler.combine_domains(
-            data_from_studio.get_domain().as_dict(),
-            data_original.get_domain().as_dict(),
+            data_from_studio.get_user_domain().as_dict(),
+            data_original.get_user_domain().as_dict(),
         )
         studio_domain = Domain.from_dict(new_domain_data)
         if not studio_domain.is_empty():
@@ -219,7 +219,9 @@ def _handle_download_no_overwrite(
         else:
             logger.warning("No additional domain data found.")
     elif domain_path.is_file():
-        domain_merged = data_original.get_domain().merge(data_from_studio.get_domain())
+        domain_merged = data_original.get_user_domain().merge(
+            data_from_studio.get_user_domain()
+        )
         domain_merged.persist(domain_path)
 
     if len(data_paths) == 1 and data_paths[0].is_file():
@@ -230,7 +232,9 @@ def _handle_download_no_overwrite(
             )
             data_nlu.persist_nlu(data_path)
         if handler.has_flows():
-            data_flows = data_original.get_flows().merge(data_from_studio.get_flows())
+            data_flows = data_original.get_user_flows().merge(
+                data_from_studio.get_user_flows()
+            )
             YamlFlowsWriter.dump(data_flows.underlying_flows, data_path)
 
     elif len(data_paths) == 1 and data_paths[0].is_dir():
@@ -276,8 +280,8 @@ def _persist_flows_diff(
 ) -> None:
     """Creates a new flows file from the diff of original and studio data."""
     new_flows_data = data_handler.create_new_flows_from_diff(
-        data_from_studio.get_flows().underlying_flows,
-        data_original.get_flows().underlying_flows,
+        data_from_studio.get_user_flows().underlying_flows,
+        data_original.get_user_flows().underlying_flows,
     )
     if new_flows_data:
         YamlFlowsWriter.dump(new_flows_data, data_path)
@@ -317,11 +321,13 @@ def _handle_download_with_overwrite(
     )
 
     if domain_path.is_file():
-        domain_merged = data_from_studio.get_domain().merge(data_original.get_domain())
+        domain_merged = data_from_studio.get_user_domain().merge(
+            data_original.get_user_domain()
+        )
         domain_merged.persist(domain_path)
     elif domain_path.is_dir():
         default = domain_path / Path(STUDIO_DOMAIN_FILENAME)
-        studio_domain = data_from_studio.get_domain()
+        studio_domain = data_from_studio.get_user_domain()
 
         paths = get_domain_path(domain_path, data_from_studio, mapper)
 
@@ -358,8 +364,8 @@ def _handle_download_with_overwrite(
             )
             nlu_data_merged.persist_nlu(data_paths[0])
         if handler.has_flows():
-            flows_data_merged = data_from_studio.get_flows().merge(
-                data_original.get_flows()
+            flows_data_merged = data_from_studio.get_user_flows().merge(
+                data_original.get_user_flows()
             )
             YamlFlowsWriter.dump(flows_data_merged.underlying_flows, data_paths[0])
     elif len(data_paths) == 1 and data_paths[0].is_dir():
@@ -374,12 +380,12 @@ def _handle_download_with_overwrite(
                 nlu_data = nlu_data.merge(nlu_file.get_nlu_data())
             pretty_write_nlu_yaml(read_yaml(nlu_data.nlu_as_yaml()), paths["nlu_path"])
         if handler.has_flows():
-            flows_data = data_from_studio.get_flows()
+            flows_data = data_from_studio.get_user_flows()
             if paths["flow_path"].exists():
                 flows_file = TrainingDataImporter.load_from_dict(
                     training_data_paths=[str(paths["flow_path"])]
                 )
-                flows_data = flows_data.merge(flows_file.get_flows())
+                flows_data = flows_data.merge(flows_file.get_user_flows())
             YamlFlowsWriter.dump(flows_data.underlying_flows, paths["flow_path"])
     else:
         #  TODO: we are not handling the case of multiple data paths?
@@ -407,7 +413,7 @@ def get_training_path(
     for intent in data_original.get_nlu_data().intents:
         for path in mapper.get_file(intent, "intents").get("training", []):
             nlu_paths.add(path)
-    flows = [flow.id for flow in data_original.get_flows().underlying_flows]
+    flows = [flow.id for flow in data_original.get_user_flows().underlying_flows]
     for flow in flows:
         for path in mapper.get_file(flow, "flows").get("training", []):
             flow_paths.add(path)

rasa/studio/train.py

@@ -20,19 +20,18 @@ from rasa.shared.core.flows.yaml_flows_io import YamlFlowsWriter
 from rasa.shared.importers.importer import TrainingDataImporter
 from rasa.shared.utils.yaml import read_yaml, write_yaml
 from rasa.studio import data_handler
-from rasa.utils.common import get_temp_dir_name
-
 from rasa.studio.config import StudioConfig
 from rasa.studio.data_handler import (
     StudioDataHandler,
     import_data_from_studio,
 )
+from rasa.utils.common import get_temp_dir_name
 
 logger = logging.getLogger(__name__)
 
 
 def handle_train(args: argparse.Namespace) -> Optional[str]:
-    from rasa import train as train_all
+    from rasa.api import train as train_all
 
     handler = StudioDataHandler(
         studio_config=StudioConfig.read_config(), assistant_name=args.assistant_name[0]

rasa/studio/upload.py

@@ -56,7 +56,10 @@ def _get_selected_entities_and_intents(
 
 def handle_upload(args: argparse.Namespace) -> None:
     """Uploads primitives to rasa studio."""
-    endpoint = StudioConfig.read_config().studio_url
+    studio_config = StudioConfig.read_config()
+    endpoint = studio_config.studio_url
+    verify = not studio_config.disable_verify
+
     if not endpoint:
         rasa.shared.utils.cli.print_error_and_exit(
             "No GraphQL endpoint found in config. Please run `rasa studio config`."
@@ -76,9 +79,9 @@ def handle_upload(args: argparse.Namespace) -> None:
 
         # check safely if args.calm is set and not fail if not
         if hasattr(args, "calm") and args.calm:
-            upload_calm_assistant(args, endpoint)
+            upload_calm_assistant(args, endpoint, verify=verify)
         else:
-            upload_nlu_assistant(args, endpoint)
+            upload_nlu_assistant(args, endpoint, verify=verify)
 
 
 config_keys = [
@@ -126,7 +129,9 @@ def _get_assistant_name(config: Dict[Text, Any]) -> str:
 
 
 @with_studio_error_handler
-def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResult:
+def upload_calm_assistant(
+    args: argparse.Namespace, endpoint: str, verify: bool = True
+) -> StudioResult:
     """Uploads the CALM assistant data to Rasa Studio.
 
     Args:
@@ -151,7 +156,7 @@ def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResu
 
     # Prepare config and domain
     config = importer.get_config()
-    domain_from_files = importer.get_domain().as_dict()
+    domain_from_files = importer.get_user_domain().as_dict()
     endpoints_from_files = read_yaml_file(args.endpoints)
     config_from_files = read_yaml_file(args.config)
 
@@ -188,8 +193,7 @@ def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResu
         training_data_paths=training_data_paths
     )
 
-    user_flows = flow_importer.get_flows().user_flows
-    flows = list(user_flows)
+    flows = list(flow_importer.get_user_flows())
 
     # We instantiate the TrainingDataImporter again on purpose to avoid
     # adding patterns to domain's actions. More info https://t.ly/W8uuc
@@ -217,11 +221,13 @@ def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResu
     )
 
     structlogger.info("Uploading to Rasa Studio...")
-    return make_request(endpoint, graphql_req)
+    return make_request(endpoint, graphql_req, verify)
 
 
 @with_studio_error_handler
-def upload_nlu_assistant(args: argparse.Namespace, endpoint: str) -> StudioResult:
+def upload_nlu_assistant(
+    args: argparse.Namespace, endpoint: str, verify: bool = True
+) -> StudioResult:
     """Uploads the classic (dm1) assistant data to Rasa Studio.
 
     Args:
@@ -269,15 +275,16 @@ def upload_nlu_assistant(args: argparse.Namespace, endpoint: str) -> StudioResul
     graphql_req = build_request(assistant_name, nlu_examples_yaml, domain_yaml)
 
     structlogger.info("Uploading to Rasa Studio...")
-    return make_request(endpoint, graphql_req)
+    return make_request(endpoint, graphql_req, verify)
 
 
-def make_request(endpoint: str, graphql_req: Dict) -> StudioResult:
+def make_request(endpoint: str, graphql_req: Dict, verify: bool = True) -> StudioResult:
     """Makes a request to the studio endpoint to upload data.
 
     Args:
         endpoint: The studio endpoint
         graphql_req: The graphql request
+        verify: Whether to verify SSL
     """
     token = KeycloakTokenReader().get_token()
     res = requests.post(
@@ -287,6 +294,7 @@ def make_request(endpoint: str, graphql_req: Dict) -> StudioResult:
             "Authorization": f"{token.token_type} {token.access_token}",
             "Content-Type": "application/json",
         },
+        verify=verify,
     )
 
     if results_logger.response_has_errors(res.json()):