rasa-pro 3.15.0a1__py3-none-any.whl → 3.15.0a3__py3-none-any.whl

rasa/builder/evaluator/response_classification/langfuse_runner.py

@@ -0,0 +1,463 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import langfuse
+import structlog
+import yaml  # type: ignore[import-untyped]
+from langfuse import Evaluation
+from langfuse._client.datasets import DatasetClient
+from langfuse.experiment import (
+    ExperimentItem,
+    ExperimentItemResult,
+    ExperimentResult,
+)
+
+from rasa.builder.copilot.models import (
+    ResponseCategory,
+)
+from rasa.builder.evaluator.constants import (
+    DEFAULT_RESPONSE_CLASSIFICATION_EVALUATION_TEXT_OUTPUT_FILENAME,
+    RESPONSE_CLASSIFICATION_EVALUATION_RESULTS_DIR,
+    RESPONSE_CLASSIFICATION_EVALUATION_YAML_OUTPUT_FILENAME,
+)
+from rasa.builder.evaluator.copilot_executor import (
+    CopilotRunResult,
+    run_copilot_with_response_handler,
+)
+from rasa.builder.evaluator.dataset.models import DatasetEntry
+from rasa.builder.evaluator.response_classification.constants import (
+    EXPERIMENT_DESCRIPTION,
+    EXPERIMENT_NAME,
+    MACRO_F1_DESCRIPTION,
+    MACRO_F1_METRIC,
+    MACRO_PRECISION_DESCRIPTION,
+    MACRO_PRECISION_METRIC,
+    MACRO_RECALL_DESCRIPTION,
+    MACRO_RECALL_METRIC,
+    MICRO_F1_DESCRIPTION,
+    MICRO_F1_METRIC,
+    MICRO_PRECISION_DESCRIPTION,
+    MICRO_PRECISION_METRIC,
+    MICRO_RECALL_DESCRIPTION,
+    MICRO_RECALL_METRIC,
+    PER_CLASS_F1_DESCRIPTION,
+    PER_CLASS_F1_METRIC_TEMPLATE,
+    PER_CLASS_PRECISION_DESCRIPTION,
+    PER_CLASS_PRECISION_METRIC_TEMPLATE,
+    PER_CLASS_RECALL_DESCRIPTION,
+    PER_CLASS_RECALL_METRIC_TEMPLATE,
+    PER_CLASS_SUPPORT_DESCRIPTION,
+    PER_CLASS_SUPPORT_METRIC_TEMPLATE,
+    SKIP_COUNT_DESCRIPTION,
+    SKIP_COUNT_METRIC,
+    WEIGHTED_F1_DESCRIPTION,
+    WEIGHTED_F1_METRIC,
+    WEIGHTED_PRECISION_DESCRIPTION,
+    WEIGHTED_PRECISION_METRIC,
+    WEIGHTED_RECALL_DESCRIPTION,
+    WEIGHTED_RECALL_METRIC,
+)
+from rasa.builder.evaluator.response_classification.evaluator import (
+    ResponseClassificationEvaluator,
+)
+from rasa.builder.evaluator.response_classification.models import (
+    ClassificationResult,
+    MetricsSummary,
+)
+
+structlogger = structlog.get_logger()
+
+
+class ResponseClassificationLangfuseRunner:
+    """Main class for running Langfuse evaluations on the classification evaluator."""
+
+    def __init__(self, dataset_name: str, output_dir: Optional[str] = None):
+        self._langfuse = langfuse.get_client()
+        self._dataset = self._retrieve_dataset(dataset_name)
+        self._output_dir = (
+            Path(output_dir)
+            if output_dir
+            else RESPONSE_CLASSIFICATION_EVALUATION_RESULTS_DIR
+        )
+
+    def _retrieve_dataset(self, dataset_name: str) -> DatasetClient:
+        """Get the dataset."""
+        try:
+            return self._langfuse.get_dataset(dataset_name)
+        except Exception as e:
+            structlogger.error(
+                "langfuse_runner.init.dataset_not_found",
+                event_info=f"Failed to get dataset '{dataset_name}'",
+                dataset_name=dataset_name,
+                error=str(e),
+            )
+            raise
+
+    def run_experiment(self) -> ExperimentResult:
+        """Run the experiment."""
+        result = self._dataset.run_experiment(
+            name=EXPERIMENT_NAME,
+            description=EXPERIMENT_DESCRIPTION,
+            task=self._run_copilot_task,
+            run_evaluators=[self._run_classification_metrics_evaluator],
+        )
+        self._report_run_results_to_txt_file(result)
+        self._langfuse.flush()
+        return result
+
+    async def _run_copilot_task(
+        self,
+        *,
+        item: ExperimentItem,
+        **kwargs: Dict[str, Any],
+    ) -> Optional[CopilotRunResult]:
+        """Copilot task function that processes each dataset item.
+
+        Follows the languse.experiment.TaskFunction protocol. The function mimics the
+        functionality of the `/copilot` endpoint.
+
+        Args:
+            item: The dataset item to process.
+            kwargs: Additional keyword arguments.
+
+        Returns:
+            A tuple containing the complete response and the generation context.
+        """
+        # Try to create the copilot context used for generating the response from the
+        # dataset item, if the context cannot be created, skip the evaluation by
+        # returning None.
+        try:
+            dataset_entry = DatasetEntry.from_raw_data(
+                id=item.id,  # type: ignore[union-attr]
+                input_data=item.input,  # type: ignore[union-attr]
+                expected_output_data=item.expected_output,  # type: ignore[union-attr]
+                metadata_data=item.metadata,  # type: ignore[union-attr]
+            )
+            context = dataset_entry.to_copilot_context()
+        except Exception as e:
+            structlogger.error(
+                "langfuse_runner._task_function_run_copilot.context_creation_failed",
+                event_info=(
+                    f"Failed to create CopilotContext from dataset item with id: "
+                    f"{item.id}. The Copilot cannot be run without a valid "  # type: ignore[union-attr]
+                    f"CopilotContext. Skipping evaluation."
+                ),
+                item_id=item.id,  # type: ignore[union-attr]
+                item_input=item.input,  # type: ignore[union-attr]
+                item_expected_output=item.expected_output,  # type: ignore[union-attr]
+                error=str(e),
+            )
+            return None
+
+        # Run the evalution. If the task fails, skip the evaluation by returning None.
+        try:
+            return await run_copilot_with_response_handler(context)
+        except Exception as e:
+            structlogger.error(
+                "langfuse_runner._task_function_run_copilot.copilot_run_failed",
+                event_info=(
+                    f"Failed to run the copilot with response handler for dataset item "
+                    f"with id: {item.id}. Skipping evaluation."  # type: ignore[union-attr]
+                ),
+                item_id=item.id,  # type: ignore[union-attr]
+                item_input=item.input,  # type: ignore[union-attr]
+                item_expected_output=item.expected_output,  # type: ignore[union-attr]
+                error=str(e),
+            )
+            return None
+
+    def _run_classification_metrics_evaluator(
+        self, *, item_results: List[ExperimentItemResult], **kwargs: Dict[str, Any]
+    ) -> List[Evaluation]:
+        """Main evaluator function that calculates classification metrics.
+
+        This function follows the languse.experiment.RunEvaluatorFunction protocol.
+        It will be called after the langfuse.experiment.TaskFunction has been called
+        for each item in the dataset.
+
+        Args:
+            item_results: The item results to evaluate.
+            kwargs: Additional keyword arguments.
+
+        Returns:
+            A list of Langfuse Evaluation objects.
+        """
+        # Create a list of ClassificationResult objects from item_results
+        classification_results, skip_count = (
+            self._create_classification_results_from_dataset_items(item_results)
+        )
+
+        # Log a warning if any items were skipped due to invalid data, as this will
+        # affect the overall metrics.
+        if skip_count > 0:
+            structlogger.warning(
+                "langfuse_runner._run_classification_metrics_evaluator.skipped_items",
+                event_info=(
+                    f"Skipped {skip_count} items due to invalid data. This will affect "
+                    f"the overall metrics."
+                ),
+                skipped_count=skip_count,
+                total_items=len(item_results),
+            )
+
+        # Run the response classification evaluator on the classification results and
+        # get the metrics summary
+        evaluator = ResponseClassificationEvaluator()  # type: ignore[no-untyped-call]
+        metrics_summary = evaluator.evaluate(classification_results)
+
+        # Record the metrics in Langfuse.
+        evaluations = self._create_langfuse_evaluation_objects(
+            metrics_summary, skip_count
+        )
+        self._report_yaml_structured_results(evaluations)
+        return evaluations
+
+    def _create_classification_results_from_dataset_items(
+        self, item_results: List[ExperimentItemResult]
+    ) -> Tuple[List[ClassificationResult], int]:
+        """Create a list of ClassificationResult objects from item results.
+
+        Args:
+            item_results: The item results to create ClassificationResult objects from.
+
+        Returns:
+            A tuple containing the list of ClassificationResult objects and the number
+            of items that were skipped due to missing predicted or expected categories.
+        """
+        classification_results: List[ClassificationResult] = []
+        skip_count = 0
+
+        # Try to create a ClassificationResult from the each item result. if either
+        # predicted or expected category is missing, skip the item.
+        for item_result in item_results:
+            # If the output is None, the task function resulted in an error, skip the
+            # item.
+            if (
+                item_result.output is None
+                or not isinstance(item_result.output, CopilotRunResult)
+                or item_result.output.response_category is None
+                or item_result.item.expected_output is None  # type: ignore[union-attr]
+                or not isinstance(item_result.item.expected_output, dict)  # type: ignore[union-attr]
+                or item_result.item.expected_output.get("response_category") is None  # type: ignore[union-attr]
+            ):
+                structlogger.error(
+                    "langfuse_runner._create_classification_results_from_dataset_items"
+                    ".invalid_item_result",
+                    event_info=(
+                        f"Cannot create a ClassificationResult from item result with "
+                        f"id: {item_result.item.id}. This item will not be used for "  # type: ignore[union-attr]
+                        f"evaluation."
+                    ),
+                    item_id=item_result.item.id,  # type: ignore[union-attr]
+                    item_output=item_result.output,
+                    item_expected_output=item_result.item.expected_output,  # type: ignore[union-attr]
+                )
+                skip_count += 1
+                continue
+
+            predicted_category = item_result.output.response_category.value
+            expected_category = item_result.item.expected_output["response_category"]  # type: ignore[union-attr]
+            classification_result = ClassificationResult(
+                prediction=ResponseCategory(predicted_category),
+                expected=ResponseCategory(expected_category),
+            )
+            classification_results.append(classification_result)
+
+        return classification_results, skip_count
+
+    def _create_langfuse_evaluation_objects(
+        self, metrics_summary: MetricsSummary, skip_count: int
+    ) -> List[Evaluation]:
+        """Create Langfuse Evaluation objects from metrics summary."""
+        evaluations: List[Evaluation] = []
+
+        # Overall metrics
+        evaluations.extend(
+            [
+                Evaluation(
+                    name=MICRO_PRECISION_METRIC,
+                    value=metrics_summary.overall.micro_precision,
+                    comment=MICRO_PRECISION_DESCRIPTION.format(
+                        value=metrics_summary.overall.micro_precision
+                    ),
+                ),
+                Evaluation(
+                    name=MACRO_PRECISION_METRIC,
+                    value=metrics_summary.overall.macro_precision,
+                    comment=MACRO_PRECISION_DESCRIPTION.format(
+                        value=metrics_summary.overall.macro_precision
+                    ),
+                ),
+                Evaluation(
+                    name=WEIGHTED_PRECISION_METRIC,
+                    value=metrics_summary.overall.weighted_avg_precision,
+                    comment=WEIGHTED_PRECISION_DESCRIPTION.format(
+                        value=metrics_summary.overall.weighted_avg_precision
+                    ),
+                ),
+                Evaluation(
+                    name=MICRO_RECALL_METRIC,
+                    value=metrics_summary.overall.micro_recall,
+                    comment=MICRO_RECALL_DESCRIPTION.format(
+                        value=metrics_summary.overall.micro_recall
+                    ),
+                ),
+                Evaluation(
+                    name=MACRO_RECALL_METRIC,
+                    value=metrics_summary.overall.macro_recall,
+                    comment=MACRO_RECALL_DESCRIPTION.format(
+                        value=metrics_summary.overall.macro_recall
+                    ),
+                ),
+                Evaluation(
+                    name=WEIGHTED_RECALL_METRIC,
+                    value=metrics_summary.overall.weighted_avg_recall,
+                    comment=WEIGHTED_RECALL_DESCRIPTION.format(
+                        value=metrics_summary.overall.weighted_avg_recall
+                    ),
+                ),
+                Evaluation(
+                    name=MICRO_F1_METRIC,
+                    value=metrics_summary.overall.micro_f1,
+                    comment=MICRO_F1_DESCRIPTION.format(
+                        value=metrics_summary.overall.micro_f1
+                    ),
+                ),
+                Evaluation(
+                    name=MACRO_F1_METRIC,
+                    value=metrics_summary.overall.macro_f1,
+                    comment=MACRO_F1_DESCRIPTION.format(
+                        value=metrics_summary.overall.macro_f1
+                    ),
+                ),
+                Evaluation(
+                    name=WEIGHTED_F1_METRIC,
+                    value=metrics_summary.overall.weighted_avg_f1,
+                    comment=WEIGHTED_F1_DESCRIPTION.format(
+                        value=metrics_summary.overall.weighted_avg_f1
+                    ),
+                ),
+            ]
+        )
+
+        # Per-class metrics
+        for category, per_class_metrics in metrics_summary.per_class.items():
+            category_name = category.value.lower()
+            evaluations.extend(
+                [
+                    Evaluation(
+                        name=PER_CLASS_PRECISION_METRIC_TEMPLATE.format(
+                            category=category_name
+                        ),
+                        value=per_class_metrics.precision,
+                        comment=PER_CLASS_PRECISION_DESCRIPTION.format(
+                            category=category.value,
+                            value=per_class_metrics.precision,
+                        ),
+                    ),
+                    Evaluation(
+                        name=PER_CLASS_RECALL_METRIC_TEMPLATE.format(
+                            category=category_name
+                        ),
+                        value=per_class_metrics.recall,
+                        comment=PER_CLASS_RECALL_DESCRIPTION.format(
+                            category=category.value,
+                            value=per_class_metrics.recall,
+                        ),
+                    ),
+                    Evaluation(
+                        name=PER_CLASS_F1_METRIC_TEMPLATE.format(
+                            category=category_name
+                        ),
+                        value=per_class_metrics.f1,
+                        comment=PER_CLASS_F1_DESCRIPTION.format(
+                            category=category.value,
+                            value=per_class_metrics.f1,
+                        ),
+                    ),
+                    Evaluation(
+                        name=PER_CLASS_SUPPORT_METRIC_TEMPLATE.format(
+                            category=category_name
+                        ),
+                        value=float(per_class_metrics.support),
+                        comment=PER_CLASS_SUPPORT_DESCRIPTION.format(
+                            category=category.value,
+                            value=per_class_metrics.support,
+                        ),
+                    ),
+                ]
+            )
+
+        # Record the number of items that were skipped due to invalid data
+        evaluations.append(
+            Evaluation(
+                name=SKIP_COUNT_METRIC,
+                value=skip_count,
+                comment=SKIP_COUNT_DESCRIPTION.format(value=skip_count),
+            )
+        )
+
+        return evaluations
+
+    def _report_run_results_to_txt_file(self, result: ExperimentResult) -> None:
+        result_str = result.format().replace("\\n", "\n")
+        self._output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Add timestamp prefix to filename
+        current_date = datetime.now().strftime("%Y%m%d_%H%M%S")
+        timestamped_filename = (
+            f"{current_date}_"
+            f"{DEFAULT_RESPONSE_CLASSIFICATION_EVALUATION_TEXT_OUTPUT_FILENAME}"
+        )
+        output_path = self._output_dir / timestamped_filename
+
+        with open(str(output_path), "w") as f:
+            f.write(result_str)
+        structlogger.info(
+            "langfuse_runner._report_run_results.exported",
+            event_info="Evaluation results exported to text file",
+            text_file=output_path,
+        )
+
+    def _report_yaml_structured_results(self, evaluations: list[Evaluation]) -> None:
+        """Export evaluation results to a YAML file with structured data."""
+        # Ensure results directory exists
+        self._output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Add timestamp prefix to filename
+        current_date = datetime.now().strftime("%Y%m%d_%H%M%S")
+        timestamped_filename = (
+            f"{current_date}_"
+            f"{RESPONSE_CLASSIFICATION_EVALUATION_YAML_OUTPUT_FILENAME}"
+        )
+        output_path = self._output_dir / timestamped_filename
+        # Convert evaluations to structured data
+        structured_data: Dict[str, Any] = {
+            "experiment": {
+                "name": EXPERIMENT_NAME,
+                "description": EXPERIMENT_DESCRIPTION,
+                "timestamp": datetime.now().isoformat(),
+            },
+            "metrics": [],
+        }
+
+        # Add each evaluation as a metric
+        for evaluation in evaluations:
+            metric_data: Dict[str, Any] = {
+                "name": evaluation.name,
+                "value": evaluation.value,
+                "description": evaluation.comment,
+            }
+            structured_data["metrics"].append(metric_data)
+
+        # Write to YAML file
+        with open(str(output_path), "w") as f:
+            yaml.dump(structured_data, f, default_flow_style=False, sort_keys=False)
+
+        structlogger.info(
+            "langfuse_runner._report_yaml_structured_results.exported",
+            event_info="Evaluation results exported to YAML file",
+            yaml_file=output_path,
+            metrics_count=len(evaluations),
+        )

rasa/builder/evaluator/response_classification/models.py

@@ -0,0 +1,61 @@
+from typing import Dict
+
+from pydantic import BaseModel, Field
+
+from rasa.builder.copilot.models import ResponseCategory
+
+
+class ClassificationResult(BaseModel):
+    prediction: ResponseCategory
+    expected: ResponseCategory
+
+
+class PerClassMetrics(BaseModel):
+    """Metrics for a single response category."""
+
+    precision: float = Field(ge=0.0, le=1.0, description="Precision score")
+    recall: float = Field(ge=0.0, le=1.0, description="Recall score")
+    f1: float = Field(ge=0.0, le=1.0, description="F1 score")
+
+    support: int = Field(ge=0, description="Number of actual occurrences.")
+
+    true_positives: int = Field(ge=0, description="Number of true positives.")
+    false_positives: int = Field(ge=0, description="Number of false positives.")
+    false_negatives: int = Field(ge=0, description="Number of false negatives.")
+
+
+class OverallClassificationMetrics(BaseModel):
+    """Overall evaluation metrics."""
+
+    micro_precision: float = Field(
+        ge=0.0, le=1.0, description="Micro-averaged Precision"
+    )
+    macro_precision: float = Field(
+        ge=0.0, le=1.0, description="Macro-averaged Precision"
+    )
+    weighted_avg_precision: float = Field(
+        ge=0.0, le=1.0, description="Weighted Precision"
+    )
+
+    micro_recall: float = Field(ge=0.0, le=1.0, description="Micro-averaged Recall")
+    macro_recall: float = Field(ge=0.0, le=1.0, description="Macro-averaged Recall")
+    weighted_avg_recall: float = Field(ge=0.0, le=1.0, description="Weighted Recall")
+
+    micro_f1: float = Field(ge=0.0, le=1.0, description="Micro-averaged F1 score")
+    macro_f1: float = Field(ge=0.0, le=1.0, description="Macro-averaged F1 score")
+    weighted_avg_f1: float = Field(ge=0.0, le=1.0, description="Weighted F1 score")
+
+    support: int = Field(ge=0, description="Total number of occurrences.")
+
+    true_positives: int = Field(ge=0, description="Total number of true positives.")
+    false_positives: int = Field(ge=0, description="Total number of false positives.")
+    false_negatives: int = Field(ge=0, description="Total number of false negatives.")
+
+
+class MetricsSummary(BaseModel):
+    """Complete metrics summary with per-class and overall metrics."""
+
+    per_class: Dict[ResponseCategory, PerClassMetrics] = Field(
+        description="Per-class metrics"
+    )
+    overall: OverallClassificationMetrics = Field(description="Overall metrics")

rasa/builder/evaluator/scripts/run_response_classification_evaluator.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Response Classification Evaluator CLI.
+
+A command-line tool for running response classification evaluation experiments using
+Langfuse.
+
+This script runs experiments on datasets and provides links to the results.
+"""
+
+import argparse
+import asyncio
+import os
+import sys
+from typing import Optional
+
+import structlog
+from langfuse.experiment import ExperimentResult
+
+from rasa.builder.evaluator.constants import (
+    DEFAULT_RESPONSE_CLASSIFICATION_EVALUATION_TEXT_OUTPUT_FILENAME,
+    RESPONSE_CLASSIFICATION_EVALUATION_YAML_OUTPUT_FILENAME,
+)
+from rasa.builder.evaluator.response_classification.langfuse_runner import (
+    ResponseClassificationLangfuseRunner,
+)
+
+# Configure structured logging
+structlogger = structlog.get_logger()
+
+
+def validate_environment() -> None:
+    """Validate that all required environment variables are set."""
+    required_vars = [
+        "OPENAI_API_KEY",
+        "INKEEP_API_KEY",
+        "LANGFUSE_HOST",
+        "LANGFUSE_PUBLIC_KEY",
+        "LANGFUSE_SECRET_KEY",
+    ]
+
+    missing_vars = []
+    for var in required_vars:
+        if not os.getenv(var):
+            missing_vars.append(var)
+
+    if missing_vars:
+        structlogger.error(
+            "main.validate_environment.missing_variables",
+            event_info=(
+                "Missing required environment variables. Please set the following "
+                f"environment variables: {missing_vars}."
+            ),
+            missing_variables=missing_vars,
+        )
+        sys.exit(1)
+
+
+async def run_experiment(
+    dataset_name: str, output_file: Optional[str] = None
+) -> Optional[ExperimentResult]:
+    """Run the response classification evaluation experiment."""
+    try:
+        structlogger.info(
+            "main.run_experiment.starting",
+            event_info="Starting response classification evaluation experiment",
+            dataset_name=dataset_name,
+        )
+
+        # Initialize and run the experiment
+        runner = ResponseClassificationLangfuseRunner(
+            dataset_name=dataset_name, output_dir=output_file
+        )
+        result = runner.run_experiment()
+
+        structlogger.info(
+            "main.run_experiment.completed",
+            event_info=(
+                "Response classification evaluation experiment completed successfully",
+            ),
+            dataset_name=dataset_name,
+        )
+
+        structlogger.info("✅ Experiment completed successfully!")
+
+        return result
+
+    except Exception as e:
+        structlogger.error(
+            "main.run_experiment.failed",
+            event_info="Response classification evaluation experiment failed",
+            error=str(e),
+            dataset_name=dataset_name,
+        )
+        structlogger.error(f"❌ Error running experiment: {e}")
+        return None
+
+
+def main() -> int:
+    """Main entry point for the CLI."""
+    parser = argparse.ArgumentParser(
+        description="Run response classification evaluation experiments using Langfuse",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="Examples:\npython run_response_classification_evaluator.py my_dataset",
+    )
+
+    parser.add_argument(
+        "--dataset-name",
+        help="Name of the dataset to evaluate",
+        required=True,
+    )
+    parser.add_argument(
+        "--output-file",
+        help=(
+            "(Optional) Directory to write experiment results. Two files are created "
+            "with a timestamp prefix (YYYYMMDD_HHMMSS_):\n"
+            f"- {DEFAULT_RESPONSE_CLASSIFICATION_EVALUATION_TEXT_OUTPUT_FILENAME} "
+            f"from Langfuse, and"
+            f"- {RESPONSE_CLASSIFICATION_EVALUATION_YAML_OUTPUT_FILENAME} "
+            "from the classifier."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    # Validate environment variables
+    validate_environment()
+
+    structlogger.info(f"🔍 Dataset: {args.dataset_name}")
+    structlogger.info("🚀 Starting evaluation...")
+
+    # Run the experiment
+    result = asyncio.run(run_experiment(args.dataset_name, args.output_file))
+
+    if result is None:
+        sys.exit(1)
+
+    # Get experiment link
+    structlogger.info(
+        "✨ Evaluation complete!",
+        dataset_run_id=result.dataset_run_id,
+        dataset_run_url=result.dataset_run_url,
+    )
+
+    # Print formatted results:
+    result_str = result.format().replace("\\n", "\n")
+    structlogger.info(result_str)
+
+    return 0
+
+
+if __name__ == "__main__":
+    main()