rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/cli/evaluate.py

@@ -0,0 +1,222 @@
+import argparse
+from pathlib import Path
+from typing import List, Text, Optional
+
+import rasa.cli.arguments.evaluate as arguments
+import rasa.shared.utils.cli
+from rasa import telemetry
+from rasa.cli import SubParsersAction
+from rasa.core.evaluation.marker_base import Marker, OperatorMarker
+from rasa.core.evaluation.marker_tracker_loader import MarkerTrackerLoader
+from rasa.core.tracker_store import TrackerStore
+from rasa.core.utils import AvailableEndpoints
+from rasa.shared.core.domain import Domain
+
+STATS_OVERALL_SUFFIX = "-overall.csv"
+STATS_SESSION_SUFFIX = "-per-session.csv"
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all evaluate parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    evaluate_parser = subparsers.add_parser(
+        "evaluate",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tools for evaluating models.",
+    )
+
+    evaluate_subparsers = evaluate_parser.add_subparsers()
+
+    marker_parser = evaluate_subparsers.add_parser(
+        "markers",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Applies marker conditions to existing trackers.",
+    )
+
+    markers_subparser = marker_parser.add_subparsers(dest="strategy")
+
+    markers_first_n_subparser = markers_subparser.add_parser(
+        "first_n",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Select trackers sequentially until N are taken.",
+    )
+    arguments.set_markers_first_n_arguments(markers_first_n_subparser)
+
+    arguments.set_markers_arguments(markers_first_n_subparser)
+
+    markers_sample_subparser = markers_subparser.add_parser(
+        "sample_n",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Select trackers by sampling N.",
+    )
+    arguments.set_markers_sample_arguments(markers_sample_subparser)
+
+    arguments.set_markers_arguments(markers_sample_subparser)
+
+    markers_all_subparser = markers_subparser.add_parser(
+        "all",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Select all trackers.",
+    )
+
+    arguments.set_markers_arguments(markers_all_subparser)
+
+    marker_parser.set_defaults(func=_run_markers_cli)
+
+
+def _run_markers_cli(args: argparse.Namespace) -> None:
+    """Run markers algorithm using parameters from CLI.
+
+    Args:
+        args: The arguments passed in from the CLI.
+    """
+    seed = args.seed if "seed" in args else None
+    count = args.count if "count" in args else None
+
+    stats_file_prefix = args.stats_file_prefix if args.stats else None
+
+    _run_markers(
+        seed,
+        count,
+        args.endpoints,
+        args.domain,
+        args.strategy,
+        args.config,
+        args.output_filename,
+        stats_file_prefix,
+    )
+
+
+def _run_markers(
+    seed: Optional[int],
+    count: Optional[int],
+    endpoint_config: Path,
+    domain_path: Optional[Text],
+    strategy: Text,
+    config: Path,
+    output_filename: Path,
+    stats_file_prefix: Optional[Path] = None,
+) -> None:
+    """Run markers algorithm over specified config and tracker store.
+
+    Args:
+        seed: (Optional) The seed to initialise the random number generator for
+              use with the 'sample' strategy.
+        count: (Optional) Number of trackers to extract from (for any strategy
+               except 'all').
+        endpoint_config: Path to the endpoint configuration defining the tracker
+                         store to use.
+        domain_path: Path to the domain specification to use when validating the
+                     marker definitions.
+        strategy: Strategy to use when selecting trackers to extract from.
+        config: Path to the markers definition file to use.
+        output_filename: Path to write out the extracted markers.
+        stats_file_prefix: (Optional) A prefix used to create paths where files with
+            statistics on the marker extraction results will be written.
+            It must consists of the path to the where those files should be stored
+            and the common file prefix, e.g. '<path-to-stats-folder>/statistics'.
+            Statistics derived from all marker extractions will be stored in
+            '<path-to-stats-folder>/statistics-overall.csv', while the statistics
+            computed per session will be stored in
+            '<path-to-stats-folder>/statistics-per-session.csv'.
+    """
+    telemetry.track_markers_extraction_initiated(
+        strategy=strategy,
+        only_extract=stats_file_prefix is not None,
+        seed=seed is not None,
+        count=count,
+    )
+
+    domain = Domain.load(domain_path) if domain_path else None
+    markers = Marker.from_path(config)
+    if domain and not markers.validate_against_domain(domain):
+        rasa.shared.utils.cli.print_error_and_exit(
+            "Validation errors were found in the markers definition. "
+            "Please see errors listed above and fix before running again."
+        )
+
+    # Calculate telemetry
+    # All loaded markers are combined with one virtual OR over all markers
+    num_markers = len(markers.sub_markers)
+    max_depth = markers.max_depth() - 1
+    # Find maximum branching of marker
+    branching_factor = max(
+        (
+            len(sub_marker.sub_markers)
+            for marker in markers.sub_markers
+            for sub_marker in marker.flatten()
+            if isinstance(sub_marker, OperatorMarker)
+        ),
+        default=0,
+    )
+
+    telemetry.track_markers_parsed_count(num_markers, max_depth, branching_factor)
+
+    tracker_loader = _create_tracker_loader(
+        endpoint_config, strategy, domain, count, seed
+    )
+
+    def _append_suffix(path: Optional[Path], suffix: Text) -> Optional[Path]:
+        return path.parent / (path.name + suffix) if path else None
+
+    try:
+        import asyncio
+
+        asyncio.run(
+            markers.evaluate_trackers(
+                trackers=tracker_loader.load(),
+                output_file=output_filename,
+                session_stats_file=_append_suffix(
+                    stats_file_prefix, STATS_SESSION_SUFFIX
+                ),
+                overall_stats_file=_append_suffix(
+                    stats_file_prefix, STATS_OVERALL_SUFFIX
+                ),
+            )
+        )
+    except (FileExistsError, NotADirectoryError) as e:
+        rasa.shared.utils.cli.print_error_and_exit(message=str(e))
+
+
+def _create_tracker_loader(
+    endpoint_config: Text,
+    strategy: Text,
+    domain: Domain,
+    count: Optional[int],
+    seed: Optional[int],
+) -> MarkerTrackerLoader:
+    """Create a tracker loader against the configured tracker store.
+
+    Args:
+        endpoint_config: Path to the endpoint configuration defining the tracker
+                         store to use.
+        strategy: Strategy to use when selecting trackers to extract from.
+        domain: The domain to use when connecting to the tracker store.
+        count: (Optional) Number of trackers to extract from (for any strategy
+               except 'all').
+        seed: (Optional) The seed to initialise the random number generator for
+              use with the 'sample_n' strategy.
+
+    Returns:
+        A MarkerTrackerLoader object configured with the specified strategy against
+        the configured tracker store.
+    """
+    endpoints = AvailableEndpoints.get_instance(endpoint_config)
+    tracker_store = TrackerStore.create(endpoints.tracker_store, domain=domain)
+    return MarkerTrackerLoader(tracker_store, strategy, count, seed)

rasa/cli/export.py

@@ -0,0 +1,250 @@
+import argparse
+import asyncio
+import logging
+import typing
+from typing import List, Text, Optional
+
+from rasa import telemetry
+from rasa.cli import SubParsersAction
+import rasa.core.utils
+import rasa.shared.utils.cli
+import rasa.utils.common
+from rasa.cli.arguments import export as arguments
+from rasa.shared.constants import DOCS_URL_EVENT_BROKERS, DOCS_URL_TRACKER_STORES
+from rasa.exceptions import PublishingError
+from rasa.shared.exceptions import RasaException
+from rasa.core.brokers.pika import PikaEventBroker
+
+if typing.TYPE_CHECKING:
+    from rasa.core.brokers.broker import EventBroker
+    from rasa.core.tracker_store import TrackerStore
+    from rasa.core.exporter import Exporter
+    from rasa.core.utils import AvailableEndpoints
+
+logger = logging.getLogger(__name__)
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add subparser for `rasa export`.
+
+    Args:
+        subparsers: Subparsers action object to which `argparse.ArgumentParser`
+            objects can be added.
+        parents: `argparse.ArgumentParser` objects whose arguments should also be
+            included.
+    """
+    export_parser = subparsers.add_parser(
+        "export",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Export conversations using an event broker.",
+    )
+    export_parser.set_defaults(func=export_trackers)
+
+    arguments.set_export_arguments(export_parser)
+
+
+def _get_tracker_store(endpoints: "AvailableEndpoints") -> "TrackerStore":
+    """Get `TrackerStore` from `endpoints`.
+
+    Prints an error and exits if no tracker store could be loaded.
+
+    Args:
+        endpoints: `AvailableEndpoints` to initialize the tracker store from.
+
+    Returns:
+        Initialized tracker store.
+
+    """
+    if not endpoints.tracker_store:
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Could not find a `tracker_store` section in the supplied "
+            f"endpoints file. Instructions on how to configure a tracker store "
+            f"can be found here: {DOCS_URL_TRACKER_STORES}. "
+            f"Exiting. "
+        )
+
+    from rasa.core.tracker_store import TrackerStore
+
+    return TrackerStore.create(endpoints.tracker_store)
+
+
+async def _get_event_broker(endpoints: "AvailableEndpoints") -> "EventBroker":
+    """Get `EventBroker` from `endpoints`.
+
+    Prints an error and exits if no event broker could be loaded.
+
+    Args:
+        endpoints: `AvailableEndpoints` to initialize the event broker from.
+
+    Returns:
+        Initialized event broker.
+
+    """
+    from rasa.core.brokers.broker import EventBroker
+
+    broker = await EventBroker.create(endpoints.event_broker)
+
+    if not broker:
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Could not find an `event_broker` section in the supplied "
+            f"endpoints file. Instructions on how to configure an event broker "
+            f"can be found here: {DOCS_URL_EVENT_BROKERS}. Exiting."
+        )
+    return broker
+
+
+def _get_requested_conversation_ids(
+    conversation_ids_arg: Optional[Text] = None,
+) -> Optional[List[Text]]:
+    """Get list of conversation IDs requested as a command-line argument.
+
+    Args:
+        conversation_ids_arg: Value of `--conversation-ids` command-line argument.
+            If provided, this is a string of comma-separated conversation IDs.
+
+    Return:
+        List of conversation IDs requested as a command-line argument.
+        `None` if that argument was left unspecified.
+
+    """
+    if not conversation_ids_arg:
+        return None
+
+    return conversation_ids_arg.split(",")
+
+
+def _assert_max_timestamp_is_greater_than_min_timestamp(
+    args: argparse.Namespace,
+) -> None:
+    """Inspect CLI timestamp parameters.
+
+    Prints an error and exits if a maximum timestamp is provided that is smaller
+    than the provided minimum timestamp.
+
+    Args:
+        args: Command-line arguments to process.
+
+    """
+    min_timestamp = args.minimum_timestamp
+    max_timestamp = args.maximum_timestamp
+
+    if (
+        min_timestamp is not None
+        and max_timestamp is not None
+        and max_timestamp < min_timestamp
+    ):
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Maximum timestamp '{max_timestamp}' is smaller than minimum "
+            f"timestamp '{min_timestamp}'. Exiting."
+        )
+
+
+def _prepare_event_broker(event_broker: "EventBroker") -> None:
+    """Prepares event broker to export tracker events.
+
+    Sets `should_keep_unpublished_messages` flag to `False` if
+    `self.event_broker` is a `PikaEventBroker`.
+
+    If publishing of events fails, the `PikaEventBroker` instance should not keep a
+    list of unpublished messages, so we can retry publishing them. This is because
+    the instance is launched as part of this short-lived export script, meaning the
+    object is destroyed before it might be published.
+
+    In addition, wait until the event broker reports a `ready` state.
+
+    """
+    if isinstance(event_broker, PikaEventBroker):
+        event_broker.should_keep_unpublished_messages = False
+        event_broker.raise_on_failure = True
+
+    if not event_broker.is_ready():
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Event broker of type '{type(event_broker)}' is not ready. Exiting."
+        )
+
+
+def export_trackers(args: argparse.Namespace) -> None:
+    """Export events for a connected tracker store using an event broker.
+
+    Args:
+        args: Command-line arguments to process.
+    """
+    asyncio.run(_export_trackers(args))
+
+
+async def _export_trackers(args: argparse.Namespace) -> None:
+    _assert_max_timestamp_is_greater_than_min_timestamp(args)
+
+    endpoints = rasa.core.utils.read_endpoints_from_path(args.endpoints)
+    tracker_store = _get_tracker_store(endpoints)
+    event_broker = await _get_event_broker(endpoints)
+    _prepare_event_broker(event_broker)
+    requested_conversation_ids = _get_requested_conversation_ids(args.conversation_ids)
+
+    from rasa.core.exporter import Exporter
+
+    exporter = Exporter(
+        tracker_store,
+        event_broker,
+        args.endpoints,
+        requested_conversation_ids,
+        args.minimum_timestamp,
+        args.maximum_timestamp,
+        args.offset_timestamps_by_seconds,
+    )
+
+    try:
+        published_events = await exporter.publish_events()
+        telemetry.track_tracker_export(published_events, tracker_store, event_broker)
+        rasa.shared.utils.cli.print_success(
+            f"Done! Successfully published {published_events} events 🎉"
+        )
+
+    except PublishingError as e:
+        command = _get_continuation_command(exporter, e.timestamp)
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Encountered error while publishing event with timestamp '{e}'. To "
+            f"continue where I left off, run the following command:"
+            f"\n\n\t{command}\n\nExiting."
+        )
+
+    except RasaException as e:
+        rasa.shared.utils.cli.print_error_and_exit(str(e))
+
+
+def _get_continuation_command(exporter: "Exporter", timestamp: float) -> Text:
+    """Build CLI command to continue 'rasa export' where it was interrupted.
+
+    Called when event publishing stops due to an error.
+
+    Args:
+        exporter: Exporter object containing objects relevant for this export.
+        timestamp: Timestamp of the last event attempted to be published.
+
+    """
+    # build CLI command command based on supplied timestamp and options
+    command = "rasa export"
+
+    if exporter.endpoints_path is not None:
+        command += f" --endpoints {exporter.endpoints_path}"
+
+    command += f" --minimum-timestamp {timestamp}"
+
+    if exporter.maximum_timestamp is not None:
+        command += f" --maximum-timestamp {exporter.maximum_timestamp}"
+
+    if exporter.requested_conversation_ids:
+        command += (
+            f" --conversation-ids {','.join(exporter.requested_conversation_ids)}"
+        )
+
+    if exporter.offset_timestamps_by_seconds is not None:
+        command += (
+            f" --offset-timestamps-by-seconds {exporter.offset_timestamps_by_seconds}"
+        )
+
+    return command

rasa/cli/inspect.py

@@ -0,0 +1,75 @@
+import argparse
+import webbrowser
+from asyncio import AbstractEventLoop
+from typing import List, Text
+
+from sanic import Sanic
+
+from rasa import telemetry
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments import shell as arguments
+from rasa.core import constants
+from rasa.utils.cli import remove_argument_from_parser
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all inspect parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    inspect_parser = subparsers.add_parser(
+        "inspect",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help=(
+            "Loads your trained model and lets you talk to your "
+            "assistant in the browser."
+        ),
+    )
+    inspect_parser.set_defaults(func=inspect)
+    arguments.set_shell_arguments(inspect_parser)
+
+    # additional argument for voice
+    inspect_parser.add_argument(
+        "--voice", help="Enable voice", action="store_true", default=False
+    )
+
+    # it'd be confusing to expose those arguments to the user,
+    # so we remove them
+    remove_argument_from_parser(inspect_parser, "--credentials")
+    remove_argument_from_parser(inspect_parser, "--connector")
+    remove_argument_from_parser(inspect_parser, "--enable-api")
+
+
+async def open_inspector_in_browser(server_url: Text, voice: bool = False) -> None:
+    """Opens the rasa inspector in the default browser."""
+    channel = "socketio" if not voice else "browser_audio"
+    webbrowser.open(f"{server_url}/webhooks/{channel}/inspect.html")
+
+
+def inspect(args: argparse.Namespace) -> None:
+    """Inspect the bot using the most recent model."""
+    import rasa.cli.run
+
+    async def after_start_hook_open_inspector(_: Sanic, __: AbstractEventLoop) -> None:
+        """Hook to open the browser on server start."""
+        server_url = constants.DEFAULT_SERVER_FORMAT.format("http", args.port)
+        await open_inspector_in_browser(server_url, args.voice)
+
+    # the following arguments are not exposed to the user
+    if args.voice:
+        args.connector = "browser_audio"
+    else:
+        args.connector = "rasa.core.channels.socketio.SocketIOInput"
+    args.enable_api = True
+    args.inspect = True
+    args.credentials = None
+    args.server_listeners = [(after_start_hook_open_inspector, "after_server_start")]
+
+    telemetry.track_inspect_started(args.connector)
+    rasa.cli.run.run(args)