rasa-pro 3.10.11__py3-none-any.whl → 3.10.13__py3-none-any.whl

rasa/shared/nlu/training_data/features.py

@@ -1,15 +1,133 @@
 from __future__ import annotations
-from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set
+
 import itertools
+from dataclasses import dataclass
+from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set
 
 import numpy as np
 import scipy.sparse
+from safetensors.numpy import save_file, load_file
 
-import rasa.shared.utils.io
 import rasa.shared.nlu.training_data.util
+import rasa.shared.utils.io
 from rasa.shared.nlu.constants import FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE
 
 
+@dataclass
+class FeatureMetadata:
+    data_type: str
+    attribute: str
+    origin: Union[str, List[str]]
+    is_sparse: bool
+    shape: tuple
+    safetensors_key: str
+
+
+def save_features(
+    features_dict: Dict[Text, List[Features]], file_name: str
+) -> Dict[str, Any]:
+    """Save a dictionary of Features lists to disk using safetensors.
+
+    Args:
+        features_dict: Dictionary mapping strings to lists of Features objects
+        file_name: File to save the features to
+
+    Returns:
+        The metadata to reconstruct the features.
+    """
+    # All tensors are stored in a single safetensors file
+    tensors_to_save = {}
+    # Metadata will be stored separately
+    metadata = {}
+
+    for key, features_list in features_dict.items():
+        feature_metadata_list = []
+
+        for idx, feature in enumerate(features_list):
+            # Create a unique key for this tensor in the safetensors file
+            safetensors_key = f"{key}_{idx}"
+
+            # Convert sparse matrices to dense if needed
+            if feature.is_sparse():
+                # For sparse matrices, use the COO format
+                coo = feature.features.tocoo()  # type:ignore[union-attr]
+                # Save data, row indices and col indices separately
+                tensors_to_save[f"{safetensors_key}_data"] = coo.data
+                tensors_to_save[f"{safetensors_key}_row"] = coo.row
+                tensors_to_save[f"{safetensors_key}_col"] = coo.col
+            else:
+                tensors_to_save[safetensors_key] = feature.features
+
+            # Store metadata
+            metadata_item = FeatureMetadata(
+                data_type=feature.type,
+                attribute=feature.attribute,
+                origin=feature.origin,
+                is_sparse=feature.is_sparse(),
+                shape=feature.features.shape,
+                safetensors_key=safetensors_key,
+            )
+            feature_metadata_list.append(vars(metadata_item))
+
+        metadata[key] = feature_metadata_list
+
+    # Save tensors
+    save_file(tensors_to_save, file_name)
+
+    return metadata
+
+
+def load_features(
+    filename: str, metadata: Dict[str, Any]
+) -> Dict[Text, List[Features]]:
+    """Load Features dictionary from disk.
+
+    Args:
+        filename: File name of the safetensors file.
+        metadata: Metadata to reconstruct the features.
+
+    Returns:
+        Dictionary mapping strings to lists of Features objects
+    """
+    # Load tensors
+    tensors = load_file(filename)
+
+    # Reconstruct the features dictionary
+    features_dict: Dict[Text, List[Features]] = {}
+
+    for key, feature_metadata_list in metadata.items():
+        features_list = []
+
+        for meta in feature_metadata_list:
+            safetensors_key = meta["safetensors_key"]
+
+            if meta["is_sparse"]:
+                # Reconstruct sparse matrix from COO format
+                data = tensors[f"{safetensors_key}_data"]
+                row = tensors[f"{safetensors_key}_row"]
+                col = tensors[f"{safetensors_key}_col"]
+
+                features_matrix = scipy.sparse.coo_matrix(
+                    (data, (row, col)), shape=tuple(meta["shape"])
+                ).tocsr()  # Convert back to CSR format
+            else:
+                features_matrix = tensors[safetensors_key]
+
+            # Reconstruct Features object
+            features = Features(
+                features=features_matrix,
+                feature_type=meta["data_type"],
+                attribute=meta["attribute"],
+                origin=meta["origin"],
+            )
+
+            features_list.append(features)
+
+        features_dict[key] = features_list
+
+    return features_dict
+
+
 class Features:
     """Stores the features produced by any featurizer."""
 

rasa/shared/utils/io.py

@@ -13,6 +13,7 @@ from typing import Any, cast, Callable, Dict, List, Optional, Text, Type, TypeVa
 import warnings
 import random
 import string
+
 import portalocker
 
 from rasa.shared.constants import (

rasa/shared/utils/yaml.py

@@ -416,47 +416,6 @@ def validate_raw_yaml_using_schema_file_with_responses(
     )
 
 
-def process_content(content: str) -> str:
-    """
-    Process the content to handle both Windows paths and emojis.
-    Windows paths are processed by escaping backslashes but emojis are left untouched.
-
-    Args:
-        content: yaml content to be processed
-    """
-    # Detect common Windows path patterns: e.g., C:\ or \\
-    UNESCAPED_WINDOWS_PATH_PATTERN = re.compile(
-        r"(?<!\w)[a-zA-Z]:(\\[a-zA-Z0-9_ -]+)*(\\)?(?!\\n)"
-    )
-    ESCAPED_WINDOWS_PATH_PATTERN = re.compile(
-        r"(?<!\w)[a-zA-Z]:(\\\\[a-zA-Z0-9_ -]+)+\\\\?(?!\\n)"
-    )
-
-    # Function to escape backslashes in Windows paths but leave other content as is
-    def escape_windows_paths(match: re.Match) -> str:
-        path = str(match.group(0))
-        return path.replace("\\", "\\\\")  # Escape backslashes only in Windows paths
-
-    def unescape_windows_paths(match: re.Match) -> str:
-        path = str(match.group(0))
-        return path.replace("\\\\", "\\")
-
-    # First, process Windows paths by escaping backslashes
-    content = re.sub(UNESCAPED_WINDOWS_PATH_PATTERN, escape_windows_paths, content)
-
-    # Ensure proper handling of emojis by decoding Unicode sequences
-    content = (
-        content.encode("utf-8")
-        .decode("raw_unicode_escape")
-        .encode("utf-16", "surrogatepass")
-        .decode("utf-16")
-    )
-
-    content = re.sub(ESCAPED_WINDOWS_PATH_PATTERN, unescape_windows_paths, content)
-
-    return content
-
-
 def read_yaml(
     content: str,
     reader_type: Union[str, List[str]] = "safe",
@@ -472,9 +431,6 @@ def read_yaml(
     Raises:
         ruamel.yaml.parser.ParserError: If there was an error when parsing the YAML.
     """
-    if _is_ascii(content):
-        content = process_content(content)
-
     custom_constructor = kwargs.get("custom_constructor", None)
 
     # Create YAML parser with custom constructor

rasa/studio/auth.py

@@ -23,12 +23,10 @@ from rasa.studio.results_logger import with_studio_error_handler, StudioResult
 class StudioAuth:
     """Handles the authentication with the Rasa Studio authentication server."""
 
-    def __init__(
-        self,
-        studio_config: StudioConfig,
-        verify: bool = True,
-    ) -> None:
+    def __init__(self, studio_config: StudioConfig) -> None:
         self.config = studio_config
+        verify = not studio_config.disable_verify
+
         self.keycloak_openid = KeycloakOpenID(
             server_url=studio_config.authentication_server_url,
             client_id=studio_config.client_id,

rasa/studio/config.py

@@ -2,13 +2,14 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass
-from typing import Dict, Optional, Text
+from typing import Any, Dict, Optional, Text
 
 from rasa.utils.common import read_global_config_value, write_global_config_value
 
 from rasa.studio.constants import (
     RASA_STUDIO_AUTH_SERVER_URL_ENV,
     RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV,
+    RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV,
     RASA_STUDIO_CLI_REALM_NAME_KEY_ENV,
     RASA_STUDIO_CLI_STUDIO_URL_ENV,
     STUDIO_CONFIG_KEY,
@@ -19,6 +20,7 @@ STUDIO_URL_KEY = "studio_url"
 CLIENT_ID_KEY = "client_id"
 REALM_NAME_KEY = "realm_name"
 CLIENT_SECRET_KEY = "client_secret"
+DISABLE_VERIFY = "disable_verify"
 
 
 @dataclass
@@ -27,13 +29,15 @@ class StudioConfig:
     studio_url: Optional[Text]
     client_id: Optional[Text]
     realm_name: Optional[Text]
+    disable_verify: bool = False
 
-    def to_dict(self) -> Dict[Text, Optional[Text]]:
+    def to_dict(self) -> Dict[Text, Optional[Any]]:
         return {
             AUTH_SERVER_URL_KEY: self.authentication_server_url,
             STUDIO_URL_KEY: self.studio_url,
             CLIENT_ID_KEY: self.client_id,
             REALM_NAME_KEY: self.realm_name,
+            DISABLE_VERIFY: self.disable_verify,
         }
 
     @classmethod
@@ -43,6 +47,7 @@ class StudioConfig:
             studio_url=data[STUDIO_URL_KEY],
             client_id=data[CLIENT_ID_KEY],
             realm_name=data[REALM_NAME_KEY],
+            disable_verify=data.get(DISABLE_VERIFY, False),
         )
 
     def write_config(self) -> None:
@@ -73,7 +78,7 @@ class StudioConfig:
         config = read_global_config_value(STUDIO_CONFIG_KEY, unavailable_ok=True)
 
         if config is None:
-            return StudioConfig(None, None, None, None)
+            return StudioConfig(None, None, None, None, False)
 
         if not isinstance(config, dict):
             raise ValueError(
@@ -83,7 +88,7 @@ class StudioConfig:
             )
 
         for key in config:
-            if not isinstance(config[key], str):
+            if not isinstance(config[key], str) and key != DISABLE_VERIFY:
                 raise ValueError(
                     "Invalid config file format. "
                     f"Key '{key}' is not a text value."
@@ -102,6 +107,9 @@ class StudioConfig:
             studio_url=StudioConfig._read_env_value(RASA_STUDIO_CLI_STUDIO_URL_ENV),
             client_id=StudioConfig._read_env_value(RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV),
             realm_name=StudioConfig._read_env_value(RASA_STUDIO_CLI_REALM_NAME_KEY_ENV),
+            disable_verify=bool(
+                os.getenv(RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV, False)
+            ),
         )
 
     @staticmethod
@@ -124,4 +132,5 @@ class StudioConfig:
             studio_url=self.studio_url or other.studio_url,
             client_id=self.client_id or other.client_id,
             realm_name=self.realm_name or other.realm_name,
+            disable_verify=self.disable_verify or other.disable_verify,
         )

rasa/studio/constants.py

@@ -10,6 +10,7 @@ RASA_STUDIO_AUTH_SERVER_URL_ENV = "RASA_STUDIO_AUTH_SERVER_URL"
 RASA_STUDIO_CLI_STUDIO_URL_ENV = "RASA_STUDIO_CLI_STUDIO_URL"
 RASA_STUDIO_CLI_REALM_NAME_KEY_ENV = "RASA_STUDIO_CLI_REALM_NAME_KEY"
 RASA_STUDIO_CLI_CLIENT_ID_KEY_ENV = "RASA_STUDIO_CLI_CLIENT_ID_KEY"
+RASA_STUDIO_CLI_DISABLE_VERIFY_KEY_ENV = "RASA_STUDIO_CLI_DISABLE_VERIFY_KEY"
 
 STUDIO_NLU_FILENAME = "studio_nlu.yml"
 STUDIO_DOMAIN_FILENAME = "studio_domain.yml"

rasa/studio/data_handler.py

@@ -76,7 +76,9 @@ class StudioDataHandler:
 
         return request
 
-    def _make_request(self, GQL_req: Dict[Any, Any]) -> Dict[Any, Any]:
+    def _make_request(
+        self, GQL_req: Dict[Any, Any], verify: bool = True
+    ) -> Dict[Any, Any]:
         token = KeycloakTokenReader().get_token()
         if token.is_expired():
             token = self.refresh_token(token)
@@ -93,6 +95,7 @@ class StudioDataHandler:
                 "Authorization": f"{token.token_type} {token.access_token}",
                 "Content-Type": "application/json",
             },
+            verify=verify,
         )
 
         if res.status_code != 200:
@@ -128,7 +131,9 @@ class StudioDataHandler:
             The data from Rasa Studio.
         """
         GQL_req = self._build_request()
-        response = self._make_request(GQL_req)
+        verify = not self.studio_config.disable_verify
+
+        response = self._make_request(GQL_req, verify=verify)
         self._extract_data(response)
 
     def request_data(
@@ -145,7 +150,9 @@ class StudioDataHandler:
             The data from Rasa Studio.
         """
         GQL_req = self._build_request(intent_names, entity_names)
-        response = self._make_request(GQL_req)
+        verify = not self.studio_config.disable_verify
+
+        response = self._make_request(GQL_req, verify=verify)
         self._extract_data(response)
 
     def get_config(self) -> Optional[str]:

rasa/studio/upload.py

@@ -56,7 +56,10 @@ def _get_selected_entities_and_intents(
 
 def handle_upload(args: argparse.Namespace) -> None:
     """Uploads primitives to rasa studio."""
-    endpoint = StudioConfig.read_config().studio_url
+    studio_config = StudioConfig.read_config()
+    endpoint = studio_config.studio_url
+    verify = not studio_config.disable_verify
+
     if not endpoint:
         rasa.shared.utils.cli.print_error_and_exit(
             "No GraphQL endpoint found in config. Please run `rasa studio config`."
@@ -76,9 +79,9 @@ def handle_upload(args: argparse.Namespace) -> None:
 
         # check safely if args.calm is set and not fail if not
         if hasattr(args, "calm") and args.calm:
-            upload_calm_assistant(args, endpoint)
+            upload_calm_assistant(args, endpoint, verify=verify)
         else:
-            upload_nlu_assistant(args, endpoint)
+            upload_nlu_assistant(args, endpoint, verify=verify)
 
 
 config_keys = [
@@ -126,7 +129,9 @@ def _get_assistant_name(config: Dict[Text, Any]) -> str:
 
 
 @with_studio_error_handler
-def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResult:
+def upload_calm_assistant(
+    args: argparse.Namespace, endpoint: str, verify: bool = True
+) -> StudioResult:
     """Uploads the CALM assistant data to Rasa Studio.
 
     Args:
@@ -216,11 +221,13 @@ def upload_calm_assistant(args: argparse.Namespace, endpoint: str) -> StudioResu
     )
 
     structlogger.info("Uploading to Rasa Studio...")
-    return make_request(endpoint, graphql_req)
+    return make_request(endpoint, graphql_req, verify)
 
 
 @with_studio_error_handler
-def upload_nlu_assistant(args: argparse.Namespace, endpoint: str) -> StudioResult:
+def upload_nlu_assistant(
+    args: argparse.Namespace, endpoint: str, verify: bool = True
+) -> StudioResult:
     """Uploads the classic (dm1) assistant data to Rasa Studio.
 
     Args:
@@ -268,15 +275,16 @@ def upload_nlu_assistant(args: argparse.Namespace, endpoint: str) -> StudioResul
     graphql_req = build_request(assistant_name, nlu_examples_yaml, domain_yaml)
 
     structlogger.info("Uploading to Rasa Studio...")
-    return make_request(endpoint, graphql_req)
+    return make_request(endpoint, graphql_req, verify)
 
 
-def make_request(endpoint: str, graphql_req: Dict) -> StudioResult:
+def make_request(endpoint: str, graphql_req: Dict, verify: bool = True) -> StudioResult:
     """Makes a request to the studio endpoint to upload data.
 
     Args:
         endpoint: The studio endpoint
         graphql_req: The graphql request
+        verify: Whether to verify SSL
     """
     token = KeycloakTokenReader().get_token()
     res = requests.post(
@@ -286,6 +294,7 @@ def make_request(endpoint: str, graphql_req: Dict) -> StudioResult:
             "Authorization": f"{token.token_type} {token.access_token}",
             "Content-Type": "application/json",
         },
+        verify=verify,
     )
 
     if results_logger.response_has_errors(res.json()):

rasa/utils/io.py

@@ -2,7 +2,6 @@ import asyncio
 import filecmp
 import logging
 import os
-import pickle
 import tempfile
 import warnings
 import re
@@ -98,29 +97,6 @@ def enable_async_loop_debugging(
     return event_loop
 
 
-def pickle_dump(filename: Union[Text, Path], obj: Any) -> None:
-    """Saves object to file.
-
-    Args:
-        filename: the filename to save the object to
-        obj: the object to store
-    """
-    with open(filename, "wb") as f:
-        pickle.dump(obj, f)
-
-
-def pickle_load(filename: Union[Text, Path]) -> Any:
-    """Loads an object from a file.
-
-    Args:
-        filename: the filename to load the object from
-
-    Returns: the loaded object
-    """
-    with open(filename, "rb") as f:
-        return pickle.load(f)
-
-
 def create_temporary_file(data: Any, suffix: Text = "", mode: Text = "w+") -> Text:
     """Creates a tempfile.NamedTemporaryFile object for data."""
     encoding = None if "b" in mode else rasa.shared.utils.io.DEFAULT_ENCODING
@@ -191,48 +167,6 @@ def create_validator(
     return FunctionValidator
 
 
-def json_unpickle(
-    file_name: Union[Text, Path], encode_non_string_keys: bool = False
-) -> Any:
-    """Unpickle an object from file using json.
-
-    Args:
-        file_name: the file to load the object from
-        encode_non_string_keys: If set to `True` then jsonpickle will encode non-string
-          dictionary keys instead of coercing them into strings via `repr()`.
-
-    Returns: the object
-    """
-    import jsonpickle.ext.numpy as jsonpickle_numpy
-    import jsonpickle
-
-    jsonpickle_numpy.register_handlers()
-
-    file_content = rasa.shared.utils.io.read_file(file_name)
-    return jsonpickle.loads(file_content, keys=encode_non_string_keys)
-
-
-def json_pickle(
-    file_name: Union[Text, Path], obj: Any, encode_non_string_keys: bool = False
-) -> None:
-    """Pickle an object to a file using json.
-
-    Args:
-        file_name: the file to store the object to
-        obj: the object to store
-        encode_non_string_keys: If set to `True` then jsonpickle will encode non-string
-          dictionary keys instead of coercing them into strings via `repr()`.
-    """
-    import jsonpickle.ext.numpy as jsonpickle_numpy
-    import jsonpickle
-
-    jsonpickle_numpy.register_handlers()
-
-    rasa.shared.utils.io.write_text_file(
-        jsonpickle.dumps(obj, keys=encode_non_string_keys), file_name
-    )
-
-
 def get_emoji_regex() -> Pattern:
     """Returns regex to identify emojis."""
     return re.compile(