rasa-pro 3.9.18__py3-none-any.whl

rasa/nlu/extractors/duckling_entity_extractor.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+import time
+import json
+import logging
+import os
+import requests
+from typing import Any, List, Optional, Text, Dict
+
+import rasa.utils.endpoints as endpoints_utils
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.constants import DOCS_URL_COMPONENTS
+from rasa.shared.nlu.constants import ENTITIES, TEXT
+from rasa.nlu.extractors.extractor import EntityExtractorMixin
+from rasa.shared.nlu.training_data.message import Message
+import rasa.shared.utils.io
+
+
+logger = logging.getLogger(__name__)
+
+
+def extract_value(match: Dict[Text, Any]) -> Dict[Text, Any]:
+    if match["value"].get("type") == "interval":
+        value = {
+            "to": match["value"].get("to", {}).get("value"),
+            "from": match["value"].get("from", {}).get("value"),
+        }
+    else:
+        value = match["value"].get("value")
+
+    return value
+
+
+def convert_duckling_format_to_rasa(
+    matches: List[Dict[Text, Any]],
+) -> List[Dict[Text, Any]]:
+    extracted = []
+
+    for match in matches:
+        value = extract_value(match)
+        entity = {
+            "start": match["start"],
+            "end": match["end"],
+            "text": match.get("body", match.get("text", None)),
+            "value": value,
+            "confidence": 1.0,
+            "additional_info": match["value"],
+            "entity": match["dim"],
+        }
+
+        extracted.append(entity)
+
+    return extracted
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.ENTITY_EXTRACTOR, is_trainable=False
+)
+class DucklingEntityExtractor(GraphComponent, EntityExtractorMixin):
+    """Searches for structured entities, e.g. dates, using a duckling server."""
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            # by default all dimensions recognized by duckling are returned
+            # dimensions can be configured to contain an array of strings
+            # with the names of the dimensions to filter for
+            "dimensions": None,
+            # http url of the running duckling server
+            "url": None,
+            # locale - if not set, we will use the language of the model
+            "locale": None,
+            # timezone like Europe/Berlin
+            # if not set the default timezone of Duckling is going to be used
+            "timezone": None,
+            # Timeout for receiving response from HTTP URL of the running
+            # duckling server. If not set the default timeout of duckling HTTP URL
+            # is set to 3 seconds.
+            "timeout": 3,
+        }
+
+    def __init__(self, config: Dict[Text, Any]) -> None:
+        """Creates the extractor.
+
+        Args:
+            config: The extractor's config.
+        """
+        self.component_config = config
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> DucklingEntityExtractor:
+        """Creates component (see parent class for full docstring)."""
+        return cls(config)
+
+    def _url(self) -> Optional[Text]:
+        """Return url of the duckling service. Environment var will override."""
+        if os.environ.get("RASA_DUCKLING_HTTP_URL"):
+            return os.environ["RASA_DUCKLING_HTTP_URL"]
+
+        return self.component_config.get("url")
+
+    def _payload(self, text: Text, reference_time: int) -> Dict[Text, Any]:
+        dimensions = self.component_config["dimensions"]
+        return {
+            "text": text,
+            "locale": self.component_config["locale"],
+            "tz": self.component_config.get("timezone"),
+            "dims": json.dumps(dimensions),
+            "reftime": reference_time,
+        }
+
+    def _duckling_parse(self, text: Text, reference_time: int) -> List[Dict[Text, Any]]:
+        """Sends the request to the duckling server and parses the result.
+
+        Args:
+            text: Text for duckling server to parse.
+            reference_time: Reference time in milliseconds.
+
+        Returns:
+            JSON response from duckling server with parse data.
+        """
+        parse_url = endpoints_utils.concat_url(self._url(), "/parse")
+        try:
+            payload = self._payload(text, reference_time)
+            headers = {
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"
+            }
+            response = requests.post(
+                parse_url,
+                data=payload,
+                headers=headers,
+                timeout=self.component_config.get("timeout"),
+            )
+            if response.status_code == 200:
+                return response.json()
+            else:
+                logger.error(
+                    f"Failed to get a proper response from remote "
+                    f"duckling at '{parse_url}. "
+                    f"Status Code: {response.status_code}. "
+                    f"Response: {response.text}"
+                )
+                return []
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.ReadTimeout,
+        ) as e:
+            logger.error(
+                "Failed to connect to duckling http server. Make sure "
+                "the duckling server is running/healthy/not stale and the proper host "
+                "and port are set in the configuration. More "
+                "information on how to run the server can be found on "
+                "github: "
+                "https://github.com/facebook/duckling#quickstart "
+                "Error: {}".format(e)
+            )
+            return []
+
+    @staticmethod
+    def _reference_time_from_message(message: Message) -> int:
+        if message.time is not None:
+            try:
+                return message.time * 1000
+            except ValueError as e:
+                logging.warning(
+                    "Could not parse timestamp {}. Instead "
+                    "current UTC time will be passed to "
+                    "duckling. Error: {}".format(message.time, e)
+                )
+        # fallbacks to current time, multiplied by 1000 because duckling
+        # requires the reftime in milliseconds
+        return int(time.time()) * 1000
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Augments the message with potentially extracted entities."""
+        if self._url() is None:
+            rasa.shared.utils.io.raise_warning(
+                "Duckling HTTP component in pipeline, but no "
+                "`url` configuration in the config "
+                "file nor is `RASA_DUCKLING_HTTP_URL` "
+                "set as an environment variable. No entities will be extracted!",
+                docs=DOCS_URL_COMPONENTS + "#DucklingEntityExtractor",
+            )
+            return messages
+
+        for message in messages:
+            reference_time = self._reference_time_from_message(message)
+            matches = self._duckling_parse(message.get(TEXT), reference_time)
+            all_extracted = convert_duckling_format_to_rasa(matches)
+            dimensions = self.component_config["dimensions"]
+            extracted = self.filter_irrelevant_entities(all_extracted, dimensions)
+            extracted = self.add_extractor_name(extracted)
+            message.set(
+                ENTITIES, message.get(ENTITIES, []) + extracted, add_to_output=True
+            )
+
+        return messages

rasa/nlu/extractors/entity_synonyms.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+import os
+from typing import Any, Dict, List, Optional, Text
+import logging
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.shared.constants import DOCS_URL_TRAINING_DATA
+from rasa.shared.nlu.constants import ENTITIES, TEXT
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.utils import write_json_to_file
+from rasa.nlu.extractors.extractor import EntityExtractorMixin
+import rasa.utils.io
+import rasa.shared.utils.io
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.ENTITY_EXTRACTOR, is_trainable=True
+)
+class EntitySynonymMapper(GraphComponent, EntityExtractorMixin):
+    """Maps entities to their synonyms if they appear in the training data."""
+
+    SYNONYM_FILENAME = "synonyms.json"
+
+    def __init__(
+        self,
+        config: Optional[Dict[Text, Any]],
+        model_storage: ModelStorage,
+        resource: Resource,
+        synonyms: Optional[Dict[Text, Any]] = None,
+    ) -> None:
+        """Creates the mapper.
+
+        Args:
+            config: The mapper's config.
+            model_storage: Storage which the component can use to persist and load
+                itself.
+            resource: Resource locator for this component which can be used to persist
+                and load itself from the `model_storage`.
+            synonyms: A dictionary of previously known synonyms.
+        """
+        self._config = config
+        self._model_storage = model_storage
+        self._resource = resource
+
+        self.synonyms = synonyms if synonyms else {}
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        synonyms: Optional[Dict[Text, Any]] = None,
+    ) -> EntitySynonymMapper:
+        """Creates component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource, synonyms)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Trains the synonym lookup table."""
+        for key, value in list(training_data.entity_synonyms.items()):
+            self._add_entities_if_synonyms(key, value)
+
+        for example in training_data.entity_examples:
+            for entity in example.get(ENTITIES, []):
+                entity_val = example.get(TEXT)[entity["start"] : entity["end"]]
+                self._add_entities_if_synonyms(entity_val, str(entity.get("value")))
+
+        self._persist()
+        return self._resource
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Modifies entities attached to message to resolve synonyms.
+
+        Args:
+            messages: List containing the latest user message
+
+        Returns:
+            List containing the latest user message with entities resolved to
+            synonyms if there is a match.
+        """
+        for message in messages:
+            updated_entities = message.get(ENTITIES, [])[:]
+            self.replace_synonyms(updated_entities)
+            message.set(ENTITIES, updated_entities, add_to_output=True)
+
+        return messages
+
+    def _persist(self) -> None:
+        if self.synonyms:
+            with self._model_storage.write_to(self._resource) as storage:
+                entity_synonyms_file = storage / EntitySynonymMapper.SYNONYM_FILENAME
+
+                write_json_to_file(
+                    entity_synonyms_file, self.synonyms, separators=(",", ": ")
+                )
+
+    # Adapt to get path from model storage and resource
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> EntitySynonymMapper:
+        """Loads trained component (see parent class for full docstring)."""
+        synonyms = None
+        try:
+            with model_storage.read_from(resource) as storage:
+                entity_synonyms_file = storage / EntitySynonymMapper.SYNONYM_FILENAME
+
+                if os.path.isfile(entity_synonyms_file):
+                    synonyms = rasa.shared.utils.io.read_json_file(entity_synonyms_file)
+                else:
+                    synonyms = None
+                    rasa.shared.utils.io.raise_warning(
+                        f"Failed to load synonyms file from '{entity_synonyms_file}'.",
+                        docs=DOCS_URL_TRAINING_DATA + "#synonyms",
+                    )
+        except ValueError:
+            logger.debug(
+                f"Failed to load {cls.__class__.__name__} from model storage. Resource "
+                f"'{resource.name}' doesn't exist."
+            )
+
+        return cls(config, model_storage, resource, synonyms)
+
+    def replace_synonyms(self, entities: List[Dict[Text, Any]]) -> None:
+        """Replace any entities which match a synonym with the synonymous entity."""
+        for entity in entities:
+            # need to wrap in `str` to handle e.g. entity values of type int
+            entity_value = str(entity["value"])
+            if entity_value.lower() in self.synonyms:
+                entity["value"] = self.synonyms[entity_value.lower()]
+                self.add_processor_name(entity)
+
+    def _add_entities_if_synonyms(self, entity: Text, synonym: Optional[Text]) -> None:
+        """Adds entities to the synonym lookup table.
+
+        Lowercase is used as keys to make the lookup case-insensitive.
+        """
+        if synonym is not None:
+            entity = str(entity)
+            synonym = str(synonym)
+            if entity != synonym:
+                entity_lowercase = entity.lower()
+                if (
+                    entity_lowercase in self.synonyms
+                    and self.synonyms[entity_lowercase] != synonym
+                ):
+                    rasa.shared.utils.io.raise_warning(
+                        f"Found conflicting synonym definitions "
+                        f"for {entity_lowercase!r}. Overwriting target "
+                        f"{self.synonyms[entity_lowercase]!r} with "
+                        f"{synonym!r}. "
+                        f"Check your training data and remove "
+                        f"conflicting synonym definitions to "
+                        f"prevent this from happening.",
+                        docs=DOCS_URL_TRAINING_DATA + "#synonyms",
+                    )
+
+                self.synonyms[entity_lowercase] = synonym
+
+            # add a self-reference to handle entities extracted in alternate cases
+            # i.e. for a synonym Austria,
+            # entities extracted as AUSTRIA, austria, ausTRIA, etc
+            # should also have the value of `Austria`
+            synonym_lowercase = synonym.lower()
+            if synonym_lowercase not in self.synonyms:
+                self.synonyms[synonym_lowercase] = synonym