crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/vision_language/image2structure/webpage/driver.py

@@ -0,0 +1,84 @@
+from typing import Tuple, Dict, Any
+
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from selenium import webdriver
+    import selenium.common.exceptions
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["image2structure"])
+
+
+def init_driver(url: str, resolution: Tuple[int, int] = (1920, 1080)) -> webdriver.Chrome:
+    """Initialize the WebDriver
+
+    Args:
+        url (str): The URL of the website. Usually "http://localhost:{port}".
+        resolution (tuple[int, int], optional): The resolution of the WebDriver. Defaults to (1920, 1080).
+
+    Returns:
+        webdriver.Chrome: The Chrome WebDriver
+    """
+    options = webdriver.ChromeOptions()
+    options.add_argument(f"--window-size={resolution[0]},{resolution[1]}")
+    options.add_argument("--headless")  # Optional: run in headless mode
+    options.add_argument("--no-sandbox")  # Optional: for certain environments
+    options.add_argument("--disable-dev-shm-usage")  # Optional: overcome limited resource problems
+    driver = webdriver.Chrome(options=options)
+    driver.get(url)
+    return driver
+
+
+def close_driver(driver: webdriver.Chrome):
+    """Close the WebDriver
+
+    Args:
+        driver (webdriver.Chrome): The Chrome WebDriver
+    """
+    driver.quit()
+
+
+class ScreenshotOptions:
+    """A class to store the parameters for taking a screenshot"""
+
+    """The resolution of the screenshot"""
+    resolution: Tuple[int, int] = (1920, 1080)
+
+    """The delay between each action in milliseconds"""
+    delay_between_each_action_ms: int = 1000
+
+
+def save_random_screenshot(path: str, port: int, options: ScreenshotOptions = ScreenshotOptions()) -> Dict[str, Any]:
+    """Save a screenshot of a random page
+
+    Args:
+        path (str): The path to save the screenshot
+        port (int): The port to use for the website.
+        options (ScreenshotOptions, optional): The options to use for taking the screenshot.
+            Defaults to ScreenshotOptions().
+
+    Returns:
+        infos (Dict[str, Any]): Additional information about the screenshot
+
+    Raises:
+        ValueError: If the path does not end with .png
+    """
+    if not path.endswith(".png"):
+        raise ValueError("The path should end with .png")
+
+    driver: webdriver.Chrome
+    try:
+        driver = init_driver(url=f"http://localhost:{port}", resolution=options.resolution)
+    except selenium.common.exceptions.WebDriverException as e:
+        raise Exception(f"Failed to initialize the driver: {e}")
+    except Exception as e:
+        raise Exception(f"An unknown error occurred while initializing the driver: {e}")
+
+    # Extract the HTML of the page
+    html = driver.page_source
+
+    # Take a screenshot of the page
+    driver.save_screenshot(path)
+    close_driver(driver)
+
+    return {"html": html}

helm/benchmark/scenarios/vision_language/image2structure/webpage/jekyll_server.py

@@ -0,0 +1,182 @@
+import subprocess
+import os
+import signal
+from typing import Optional
+import time
+import socket
+import threading
+
+from helm.common.hierarchical_logger import hlog
+
+
+class JekyllServer:
+    """A class to start and stop a Jekyll server in a separate process."""
+
+    def __init__(self, repo_path: str, port: int, verbose: bool = False):
+        self.repo_path: str = repo_path
+        self.verbose: bool = verbose
+        self.port: int = port
+        self.process: Optional[subprocess.Popen] = None
+        self.success: bool = False  # Shared flag to indicate if the server started successfully
+
+    def __del__(self):
+        self.stop()
+        if JekyllServer.is_port_in_use(self.port):
+            if self.verbose:
+                hlog(f"Port {self.port} is in use. Attempting to free it.")
+            self.kill_process_using_port(self.port)
+        if self.verbose:
+            hlog("JekyllServer object deleted.")
+
+    def setup_gemfile(self):
+        # Check if Gemfile exists, if not, copy Gemfile.default to Gemfile
+        if not os.path.exists(f"{self.repo_path}/Gemfile"):
+            default_gemfile_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "Gemfile.default")
+            os.system(f"cp {default_gemfile_path} {self.repo_path}/Gemfile")
+            if self.verbose:
+                hlog("Copied Gemfile.default to Gemfile")
+            return
+
+        # Gemfile exists, check if it has the jekyll gem
+        if "jekyll" in open(f"{self.repo_path}/Gemfile").read():
+            # TODO: figure out if we need to do anything here
+            return
+
+        # Gemfile exists, but doesn't have jekyll gem
+        with open(f"{self.repo_path}/Gemfile", "a") as file:
+            file.write('gem "jekyll", "~> 4.3.3"')
+            if self.verbose:
+                hlog("Added jekyll gem to Gemfile")
+
+    def setup_config(self):
+        # Check if _config.yml exists, if not, copy _config.default.yml to _config.yml
+        if not os.path.exists(f"{self.repo_path}/_config.yml"):
+            default_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "_config.default.yml")
+            os.system(f"cp {default_config_path} {self.repo_path}/_config.yml")
+            if self.verbose:
+                hlog("Copied _config.default.yml to _config.yml")
+        # Search for line starting with "port:" and replace it with "port: <port>"
+        with open(f"{self.repo_path}/_config.yml", "r") as file:
+            lines = file.readlines()
+        with open(f"{self.repo_path}/_config.yml", "w") as file:
+            for line in lines:
+                if line.startswith("port"):
+                    file.write(f"port: {self.port}\n")
+                else:
+                    file.write(line)
+
+    @staticmethod
+    def is_port_in_use(port: int) -> bool:
+        """Check if a port is in use on localhost."""
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex(("localhost", port)) == 0
+
+    def kill_process_using_port(self, port: int):
+        """Find and kill the process using the specified port."""
+        command = f"lsof -ti:{port} | grep '[0-9]' | xargs -r kill -9"
+        os.system(command)
+        if self.verbose:
+            hlog(f"Killed process using port {port}.")
+
+    def stream_output(self, process: subprocess.Popen):
+        """Read from stdout and stderr streams and hlog."""
+        assert process.stdout is not None
+        assert process.stderr is not None
+        while True:
+            output = process.stdout.readline()
+            if not output:
+                err = process.stderr.readline()
+                if err:
+                    decoded_line = err.decode("utf-8").strip()
+                    if self.verbose:
+                        hlog(f"\t> \033[91mStderr: {decoded_line}\033[0m")
+                    self.success = False
+                    break
+                else:
+                    # No more output
+                    break
+            else:
+                decoded_line = output.decode("utf-8").strip()
+                if self.verbose:
+                    hlog(f"\t> Stdout: {decoded_line}")
+                if "Server running... press ctrl-c to stop." in decoded_line:
+                    self.success = True
+                    break
+
+    def start(self, timeout: int = 30) -> bool:
+        """Start the Jekyll server in a separate process and monitor the output."""
+        if JekyllServer.is_port_in_use(self.port):
+            if self.verbose:
+                hlog(f"Port {self.port} is in use. Attempting to free it.")
+            self.kill_process_using_port(self.port)
+
+        self.setup_gemfile()
+        self.setup_config()
+        command_install = f"cd {self.repo_path} && bundle install"
+        subprocess.run(command_install, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+        command_serve = f"cd {self.repo_path} && bundle exec jekyll serve --port {self.port}"
+        self.process = subprocess.Popen(
+            command_serve,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            preexec_fn=os.setsid,
+        )
+
+        # Start thread to read output
+        output_thread = threading.Thread(target=self.stream_output, args=(self.process,))
+        output_thread.start()
+
+        # Wait for the thread to complete or timeout
+        output_thread.join(timeout=timeout)
+
+        if output_thread.is_alive():
+            # If the thread is still alive after the timeout, the server did not start
+            # successfully within the timeout period
+            hlog("Timeout reached without detecting server start.")
+            self.process.terminate()  # Terminate the process if it's still running
+            output_thread.join()  # Ensure the thread is cleaned up
+            return False
+        else:
+            if self.verbose:
+                if self.success:
+                    hlog("Jekyll server started successfully.")
+                else:
+                    hlog("Jekyll server failed to start.")
+            return self.success  # Return the success flag
+
+    def stop(self, timeout=5):
+        """Stop the Jekyll server and terminate the process with a timeout.
+
+        Args:
+            timeout (int, optional): Time to wait for the server to gracefully shut down. Defaults to 5 seconds.
+        """
+        if self.process:
+            # Try to terminate the process group gracefully
+            os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
+            self.process.terminate()
+
+            # Wait for the process to end, checking periodically
+            try:
+                # Wait up to `timeout` seconds for process to terminate
+                for _ in range(timeout):
+                    if self.process.poll() is not None:  # Process has terminated
+                        break
+                    time.sleep(1)  # Wait a bit before checking again
+                else:
+                    # If the process is still alive after the timeout, kill it
+                    os.killpg(os.getpgid(self.process.pid), signal.SIGKILL)
+                    self.process.kill()
+                    self.process.wait()  # Wait for process to be killed
+                    if self.verbose:
+                        hlog("Jekyll server forcefully stopped.")
+            except Exception as e:
+                if self.verbose:
+                    hlog(f"Error stopping the Jekyll server: {e}")
+
+            self.process = None
+            if self.verbose:
+                hlog("Jekyll server stopped.")
+        elif self.verbose:
+            hlog("Jekyll server is not running.")

helm/benchmark/scenarios/vision_language/image2structure/webpage/utils.py

@@ -0,0 +1,31 @@
+import re
+
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from html2text import HTML2Text
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["image2structure"])
+
+
+def convert_html_to_text(handler: HTML2Text, html: str) -> str:
+    """Convert HTML to text
+
+    Args:
+        handler (HTML2Text): The HTML2Text handler
+        html (str): The HTML to convert
+
+    Returns:
+        str: The text
+    """
+    text: str = handler.handle(html)
+    # Normalize space sequences to a single space globally
+    text = re.sub(r" +", " ", text)
+    # Replace tabs with a single space
+    text = re.sub(r"\t", " ", text)
+    # Remove leading and trailing spaces on each line
+    text = re.sub(r"^[ \t]+|[ \t]+$", "", text, flags=re.MULTILINE)
+    # Remove unnecessary whitespace - multiple empty lines and tabulations
+    text = re.sub(r"\n\s*\n", "\n", text)
+
+    return text.strip()

helm/benchmark/scenarios/vision_language/image2structure/webpage_scenario.py

@@ -0,0 +1,225 @@
+from typing import Dict, List, Any
+
+from helm.benchmark.scenarios.scenario import VALID_SPLIT
+from helm.benchmark.scenarios.vision_language.image2structure.image2structure_scenario import (
+    Image2StructureScenario,
+    PROCESSED,
+)
+from helm.benchmark.scenarios.vision_language.image2structure.webpage.jekyll_server import JekyllServer
+from helm.benchmark.scenarios.vision_language.image2structure.webpage.driver import (
+    save_random_screenshot,
+    ScreenshotOptions,
+)
+from helm.benchmark.scenarios.vision_language.image2structure.webpage.utils import convert_html_to_text
+from helm.common.general import ensure_directory_exists
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from html2text import HTML2Text
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["image2structure"])
+
+
+import base64
+import os
+import threading
+import tarfile
+import shutil
+import time
+import pickle
+
+
+def extract_repo(base64_encoding: str, repo_path: str) -> None:
+    # Decode the base64 string which corresponds to an archive
+    # and extract the files to the repo_path
+    try:
+        archive = base64.b64decode(base64_encoding)
+        # Write to .tar file
+        with open(os.path.join(repo_path, "repo.tar.gz"), "wb") as f:
+            f.write(archive)
+        # Extract
+        with tarfile.open(os.path.join(repo_path, "repo.tar.gz"), "r:gz") as tar:
+            tar.extractall(repo_path)
+    except Exception as e:
+        raise ValueError(f"Failed to decode and extract the base64 archive: {e}")
+
+
+def list_assets(repo_path: str, extensions: List[str]) -> List[str]:
+    asset_paths: List[str] = []
+    for root, _, files in os.walk(repo_path):
+        for file in files:
+            if file.split(".")[-1].lower() in extensions:
+                # Remove repo_path from the file path
+                # ignore paths that start with a dot or in _site
+                if not root.startswith(os.path.join(repo_path, "_site")) and not root.startswith(
+                    os.path.join(repo_path, ".")
+                ):
+                    asset_paths.append(os.path.relpath(os.path.join(root, file), repo_path))
+    return asset_paths
+
+
+def serve_and_take_screenshot(
+    repo_path: str,
+    destination_path: str,
+    screenshot_options: ScreenshotOptions = ScreenshotOptions(),
+    max_tries: int = 5,
+) -> Dict[str, Any]:
+    # Start the Jekyll server
+    # Select a unique port per thread
+    port: int = 4000 + int(threading.get_ident()) % 1000
+    server = JekyllServer(repo_path, port=port, verbose=False)
+    success: bool = server.start()
+    if not success:
+        # This runs on examples that are not expected to fail
+        server.stop()
+        raise ValueError(f"Jekyll server failed to start: {repo_path}")
+
+    # Take a screenshot of a random page
+    success = False
+    error: Exception
+    for _ in range(max_tries):
+        try:
+            infos: Dict[str, Any] = save_random_screenshot(destination_path, port=port, options=screenshot_options)
+            success = True
+            break
+        except Exception as e:
+            if "net::ERR_CONNECTION_REFUSED" in str(e):
+                error = e
+                server.stop()
+                time.sleep(0.5)
+                server.start()
+                time.sleep(0.5)
+            else:
+                # Do not retry
+                break
+    if not success:
+        raise ValueError(f"Failed to take a screenshot: {error}")
+
+    # Stop the server
+    server.stop()
+    time.sleep(0.1)
+
+    return infos
+
+
+class WebpageScenario(Image2StructureScenario):
+    BASE_PROMPT = (
+        "Please generate the source code to generate a webpage that looks like this image as much as feasibly possible.\n"  # noqa: E501
+        "You should output a json object associating each file name with its content.\n\n"
+        "Here is a simple example of the expected structure (that does not correspond to the image)."
+        " In this example, 3 files are created: index.html, style.css and script.js.\n"
+        "[\n"
+        "  {\n"
+        '    "filename": "index.html",\n'
+        '    "content": "<!DOCTYPE html>\\n<html>\\n<head>\\n<title>Title of the document</title>\\n</head>\\n<body>\\n\\n<p>Content of the document......</p>\\n\\n</body>\\n</html>"\n'  # noqa: E501
+        "  },\n"
+        "  {\n"
+        '    "filename": "style.css",\n'
+        '    "content": "body {\\n  background-color: lightblue;\\n}\\nh1 {\\n  color: white;\\n  text-align: center;\\n}"\n'  # noqa: E501
+        "  },\n"
+        "  {\n"
+        '    "filename": "script.js",\n'
+        '    "content": "document.getElementById(\\"demo\\").innerHTML = \\"Hello JavaScript!\\";"\n'
+        "  }\n"
+        "]\n"
+        "You do not have to create files with the same names. Create as many files as you need, you can even use directories if necessary,"  # noqa: E501
+        " they will be created for you automatically. Try to write some realistic code keeping in mind that is should"
+        " look like the image as much as feasibly possible."
+    )
+
+    HUGGINGFACE_DATASET_NAME = "stanford-crfm/i2s-webpage"
+    SUBSETS = ["css", "html", "javascript"]
+    MAX_TRIES: int = 5
+    ASSETS_EXTENSIONS: List[str] = ["png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp", "tiff"]
+
+    name = "image2webpage"
+    description = "Evaluate multimodal models on webpage generation to recreate a provided image"
+
+    def __init__(
+        self,
+        subset: str,
+        recompile_prompt: bool = True,
+        split: str = VALID_SPLIT,
+        screenshot_options: ScreenshotOptions = ScreenshotOptions(),
+    ):
+        super().__init__(subset, recompile_prompt, split)
+        self._screenshot_options = screenshot_options
+        self._html2text = HTML2Text()
+        self._html2text.ignore_links = True
+
+    def preprocess_row(self, row: Dict[str, Any], assets_path: str) -> Dict[str, Any]:
+        """Extract the base64 encoding of the repo from the row and return it."""
+        # No need to reprocess if the assets are already saved
+        assets_save_path: str = os.path.join(assets_path, str(row["uuid"].replace('"', "")))
+        if os.path.exists(assets_save_path):
+            try:
+                with open(os.path.join(assets_save_path, "assets_paths.pkl"), "rb") as f:
+                    row["assets_paths"] = pickle.load(f)
+                with open(os.path.join(assets_save_path, "assets_names.pkl"), "rb") as f:
+                    row["assets_names"] = pickle.load(f)
+                del row["assets"]
+                row["structure"] = PROCESSED
+                return row
+            except Exception:
+                # There was an issue when loading the assets, reprocess
+                shutil.rmtree(assets_save_path)
+        ensure_directory_exists(assets_save_path)
+
+        # Structure is a base64 encoding of the repo
+        if self._output_path is None:
+            raise ValueError("Output path not set")
+        repo_path = os.path.join(self._output_path, f"tmp{threading.get_ident()}_{self._subset}")
+        ensure_directory_exists(repo_path)
+
+        # Decode the base64 string which corresponds to an archive
+        # and extract the files to the repo_path
+        structure: str = row["structure"]
+        extract_repo(structure, repo_path)
+        row["structure"] = PROCESSED
+        row["repo_path"] = repo_path  # Stored for cleanup
+
+        # Process the assets
+        asset_paths: List[str] = list_assets(repo_path, self.ASSETS_EXTENSIONS)
+        del row["assets"]
+        row["assets_paths"] = []
+        row["assets_names"] = []
+        # Copy each asset to a unique persistent path
+        for i, asset_local_path in enumerate(asset_paths):
+            asset_name: str = asset_local_path
+            asset_dest_path = os.path.join(assets_save_path, f"{i}.{asset_local_path.split('.')[-1]}")
+            shutil.copyfile(os.path.join(row["repo_path"], asset_local_path), asset_dest_path)
+            row["assets_paths"].append(asset_dest_path)
+            row["assets_names"].append(asset_name)
+
+        # Save both assets_paths and assets_names as files than can be loaded
+        with open(os.path.join(assets_save_path, "assets_paths.pkl"), "wb") as f:
+            pickle.dump(row["assets_paths"], f)
+        with open(os.path.join(assets_save_path, "assets_names.pkl"), "wb") as f:
+            pickle.dump(row["assets_names"], f)
+
+        return row
+
+    def build_prompt(self, row: Dict[str, Any]) -> str:
+        prompt: str = self.BASE_PROMPT
+        assert "assets_paths" in row, "No assets paths in the row"
+        assert "assets_names" in row, "No assets names in the row"
+        assert len(row["assets_paths"]) == len(row["assets_names"])
+        if len(row["assets_names"]) > 0:
+            prompt += "\nYou have access to the following assets:\n"
+            for asset_local_path in row["assets_names"]:
+                prompt += f"- {asset_local_path}\n"
+        return prompt
+
+    def compile_and_save(self, structure: str, assets_path: str, destination_path: str) -> str:
+        # Structure is the path to the repo
+        # Serve and take screenshot
+        repo_path: str = structure
+        infos: Dict[str, Any] = serve_and_take_screenshot(repo_path, destination_path, self._screenshot_options)
+        text: str = convert_html_to_text(self._html2text, infos["html"])
+        return text
+
+    def finalize(self, row: Dict[str, Any]) -> None:
+        """Perform cleanup operations after the instance has been generated."""
+        if "repo_path" in row:
+            repo_path: str = row["repo_path"]
+            shutil.rmtree(repo_path)

helm/benchmark/scenarios/vision_language/mementos_scenario.py

@@ -0,0 +1,124 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class MementosScenario(Scenario):
+    """
+    Mementos: A Comprehensive Benchmark for Multimodal Large Language Model Reasoning over Image Sequences
+
+    We introduces Mementos, a new benchmark designed to assess MLLMs' sequential image reasoning abilities. Mementos
+    features 4,761 diverse image sequences with varying lengths.
+
+    @misc{wang2024mementos,
+    title={Mementos: A Comprehensive Benchmark for Multimodal Large Language Model Reasoning over Image Sequences},
+    author={Xiyao Wang and Yuhang Zhou and Xiaoyu Liu and Hongjin Lu and Yuancheng Xu and Feihong He and Jaehong Yoon
+    and Taixi Lu and Gedas Bertasius and Mohit Bansal and Huaxiu Yao and Furong Huang},
+    year={2024},
+    eprint={2401.10529},
+    archivePrefix={arXiv},
+    primaryClass={cs.CV}
+    }
+
+    Paper: https://arxiv.org/abs/2401.10529
+    """
+
+    MEMENTOS_HUGGINGFACE_DATASET_NAME: str = "shenmishajing/unofficial_mementos_dataset"
+
+    IMAGE_URL: str = (
+        "https://huggingface.co/datasets/shenmishajing/unofficial_mementos_dataset/resolve/main/"
+        + "{subject}/{split}/{file_name}?download=true"
+    )
+
+    DATA_FILES: str = "{subject}/{split}/metadata.csv"
+
+    QUESTION_PROMPT: str = (
+        "Write a description for the given image sequence in a single paragraph, what is happening in this episode?"
+    )
+
+    SUBJECTS: List[str] = ["comics", "dailylife", "robotics"]
+
+    name = "mementos"
+    description = (
+        "A Comprehensive Benchmark for Multimodal Large Language Model Reasoning over Image Sequences"
+        " ([paper](https://arxiv.org/abs/2401.10529))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        subject_output_path: str = os.path.join(output_path, self._subject)
+        ensure_directory_exists(subject_output_path)
+
+        data_files = {
+            split: self.DATA_FILES.format(subject=self._subject, split=split) for split in [TRAIN_SPLIT, TEST_SPLIT]
+        }
+        instances: List[Instance] = []
+
+        for split in [TRAIN_SPLIT, TEST_SPLIT]:
+            cur_output_path = os.path.join(subject_output_path, split)
+            ensure_directory_exists(cur_output_path)
+
+            # Process the test set
+            for row in tqdm(
+                load_dataset(
+                    self.MEMENTOS_HUGGINGFACE_DATASET_NAME.format(subject=self._subject),
+                    data_files=data_files,
+                    split=split,
+                    cache_dir=cur_output_path,
+                )
+            ):
+                # Download the image
+                file_name: str = row["file_name"]
+                local_image_path: str = os.path.join(cur_output_path, file_name)
+                ensure_file_downloaded(
+                    source_url=self.IMAGE_URL.format(subject=self._subject, split=split, file_name=file_name),
+                    target_path=local_image_path,
+                    unpack=False,
+                )
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/png"),
+                    MediaObject(text=self.QUESTION_PROMPT, content_type="text/plain"),
+                ]
+                answer: str = row["description"]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+            print()
+
+        return instances
+
+
+def main():
+    scenario = MementosScenario("robotics")
+    instances = scenario.get_instances("output")
+    print(instances)
+
+
+if __name__ == "__main__":
+    main()