notte-eval 0.0.dev0__tar.gz

notte_eval-0.0.dev0/.gitignore

@@ -0,0 +1,179 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+ignore.*
+llm_usage.jsonl
+llm_parsing_error.jsonl
+traces/
+
+**/__pycache__/**
+.DS_Store
+**/.DS_Store
+old
+notebook

notte_eval-0.0.dev0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: notte-eval
+Version: 0.0.dev0
+Summary: The evaluation for Notte
+Author-email: Notte Team  <hello@notte.cc>
+Requires-Python: >=3.11
+Requires-Dist: notte-agent==0.0.dev
+Requires-Dist: notte-browser==0.0.dev
+Requires-Dist: notte-core==0.0.dev
+Requires-Dist: notte-integrations==0.0.dev
+Requires-Dist: notte-sdk==0.0.dev
+Provides-Extra: api
+Requires-Dist: fastapi>=0.115.8; extra == 'api'
+Requires-Dist: uvicorn>=0.29.0; extra == 'api'
+Provides-Extra: browser-use
+Requires-Dist: browser-use>=0.1.40; extra == 'browser-use'
+Requires-Dist: langchain-google-genai>=2.1.1; extra == 'browser-use'
+Provides-Extra: browserbase
+Requires-Dist: browserbase>=1.2.0; extra == 'browserbase'
+Provides-Extra: camoufox
+Requires-Dist: camoufox[geoip]>=0.4.11; extra == 'camoufox'
+Provides-Extra: convergence
+Requires-Dist: proxy-lite; extra == 'convergence'
+Provides-Extra: discord
+Requires-Dist: discord-py<2.5.0,>=2.3.0; extra == 'discord'
+Provides-Extra: embedding
+Requires-Dist: sentence-transformers>=3.4.1; extra == 'embedding'
+Provides-Extra: server
+Requires-Dist: litellm[proxy]>=1.61.16; extra == 'server'
+Provides-Extra: slack
+Requires-Dist: slack-sdk>=3.34.0; extra == 'slack'
+Provides-Extra: vault
+Requires-Dist: hvac>=2.3.0; extra == 'vault'

notte_eval-0.0.dev0/pyproject.toml

@@ -0,0 +1,118 @@
+[project]
+name = "notte-eval"
+version = "0.0.dev"
+description = "The evaluation for Notte"
+readme = "README.md"
+authors = [
+    { name = "Notte Team ", email = "hello@notte.cc" }
+]
+packages = [
+    { include = "notte_eval", from = "src" },
+]
+
+
+requires-python = ">=3.11"
+dependencies = [
+    "notte-agent==0.0.dev",
+    "notte-browser==0.0.dev",
+    "notte-core==0.0.dev",
+    "notte-sdk==0.0.dev",
+    "notte-integrations==0.0.dev",
+]
+
+[project.optional-dependencies]
+vault = [
+    "hvac>=2.3.0",
+]
+server = [
+    "litellm[proxy]>=1.61.16",
+]
+embedding = [
+    "sentence-transformers>=3.4.1",
+]
+api = [
+    "fastapi>=0.115.8",
+    "uvicorn>=0.29.0",
+]
+discord = [
+    "discord-py>=2.3.0,<2.5.0",
+]
+slack = [
+    "slack-sdk>=3.34.0",
+]
+
+browserbase = [
+    "browserbase>=1.2.0",
+]
+camoufox = [
+    "camoufox[geoip]>=0.4.11",
+]
+browser-use = [
+    "browser-use>=0.1.40",
+    "langchain-google-genai>=2.1.1",
+]
+convergence = [
+    "proxy-lite",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+
+dev = [
+    "aiomultiprocess>=0.9.1",
+    "basedpyright>=1.27.1",
+    "cloudpickle>=3.1.1",
+    "joblib>=1.4.2",
+    "jupyter>=1.1.1",
+    "pandas",
+    "pebble>=5.1.1",
+    "pre-commit>=4.1.0",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.3",
+    "pytest-mock>=3.14.0",
+]
+lint = [
+    "ruff>=0.9.7",
+]
+
+[tool.pytest.ini_options]
+asyncio_default_fixture_loop_scope = "function"
+testpaths = ["tests"]
+timeout = 60
+asyncio_mode = "strict"
+log_cli = true
+log_cli_level = "INFO"
+filterwarnings = [
+    "ignore::DeprecationWarning:sklearn.utils.fixes:",
+    "ignore::DeprecationWarning:pandas.core.common:",
+    "ignore::pydantic.warnings.PydanticDeprecatedSince20:",
+    "ignore::DeprecationWarning:importlib.resources._legacy:",
+    "ignore::DeprecationWarning:litellm.utils:",
+    "ignore:open_text is deprecated*:DeprecationWarning",
+    "ignore:distutils Version classes are deprecated. Use packaging.version instead.*:DeprecationWarning",
+    'ignore:configuration option "asyncio_default_fixture_loop_scope" is unset',
+    "ignore:Valid config keys have changed in V2*:UserWarning"
+]
+
+[tool.ruff]
+line-length = 120
+indent-width = 4
+exclude = [".venv"]
+
+[tool.ruff.lint]
+extend-select = ["I"]
+
+[tool.basedpyright]
+exclude = [".venv", "uv-cache", "**/site-packages/**", "**/dist-packages/**", "tests", "old", "notebook", "dist"]
+enableTypeIgnoreComments = true
+reportIgnoreCommentWithoutRule = false
+failOnWarnings = true
+reportAny = false
+reportExplicitAny = false
+
+[tool.uv.sources]
+maincontentextractor = { git = "https://github.com/HawkClaws/main_content_extractor", rev = "7c3ed7f6ed7f6c10223a3357d43ab741663bc812" }
+proxy-lite = { git = "https://github.com/leo-notte/proxy-lite" }

notte_eval-0.0.dev0/src/notte_eval/__init__.py

@@ -0,0 +1,3 @@
+from notte_core import check_notte_version
+
+__version__ = check_notte_version("notte_eval")

notte_eval-0.0.dev0/src/notte_eval/agent_handlers/__init__.py

@@ -0,0 +1,58 @@
+import importlib
+from enum import StrEnum
+from typing import Any, NamedTuple
+
+from pydantic import BaseModel
+
+
+class PoolEnum(StrEnum):
+    NONE = "None"
+    ANCHOR = "Anchor"
+    STEEL = "Steel"
+    BROWSERBASE = "BrowserBase"
+    CAMOUFOX = "Camoufox"
+
+
+class Proxy(BaseModel):
+    server: str
+    username: str
+    password: str
+
+
+def fetch_handler(key: str) -> tuple[type, type]:
+    """
+    Import specific module based on key and return input and handler types
+    """
+    if key not in HANDLERS_DICT:
+        raise ValueError(f"Unknown handler key: {key}")
+
+    handler = HANDLERS_DICT[key]
+    module = importlib.import_module(f"{__package__}.{handler.module_name}")
+
+    input_type = getattr(module, handler.input_name)
+    handler_type = getattr(module, handler.handler_name)
+
+    return input_type, handler_type
+
+
+class HandlerTuple(NamedTuple):
+    module_name: str
+    input_name: str
+    handler_name: str
+
+
+HANDLERS_DICT = {
+    "Falco": HandlerTuple("falco", "FalcoInput", "FalcoBench"),
+    "BrowserUse": HandlerTuple("browseruse", "BrowserUseInput", "BrowserUseBench"),
+    "BrowserUseAPI": HandlerTuple("browseruse_api", "BrowserUseAPIInput", "BrowserUseAPIBench"),
+    "Convergence": HandlerTuple("convergence", "ConvergenceInput", "ConvergenceBench"),
+}
+
+
+def trim_image_messages(input_content: list[dict[Any, Any]]) -> None:
+    # trim down: remove images in the message history
+    for msg in input_content:
+        if "content" in msg and isinstance(msg["content"], list):
+            for submsg in msg["content"]:  # type: ignore
+                if "type" in submsg and submsg["type"] == "image_url" and "image_url" in submsg:
+                    submsg["image_url"] = "benchmark: removed"

notte_eval-0.0.dev0/src/notte_eval/agent_handlers/browseruse.py

@@ -0,0 +1,201 @@
+import json
+import logging
+import os
+import typing
+
+# posthog seems to deadlock tasks otherwise
+os.environ["ANONYMIZED_TELEMETRY"] = "false"
+
+from notte_core.utils.webp_replay import ScreenshotReplay
+from pydantic import BaseModel, SecretStr, ValidationError
+from typing_extensions import override
+
+from notte_eval.agent_handlers import Proxy, trim_image_messages
+from notte_eval.data.load_data import BenchmarkTask
+from notte_eval.patcher import AgentPatcher, FunctionLog
+from notte_eval.task_types import AgentBenchmark, LLMCall, Step, TaskResult
+
+try:
+    from browser_use import Agent as BrowserUseAgent  # type: ignore
+    from browser_use import AgentHistoryList, Browser, BrowserConfig  # type: ignore
+    from browser_use.controller.views import DoneAction  # type: ignore
+    from langchain_openai import ChatOpenAI
+except ImportError:
+    raise ImportError("Install notte[browseruse] to fix")
+
+
+# solely for simplicity of parsing response
+class BUAgentCurrentState(BaseModel):
+    evaluation_previous_goal: str
+    memory: str
+    next_goal: str
+
+
+class BUAgentArguments(BaseModel):
+    current_state: BUAgentCurrentState
+    action: list[dict[str, typing.Any]]
+
+
+# used for the io to the benchmark (toml)
+class BrowserUseInput(BaseModel):
+    use_vision: bool
+    model: str
+    headless: bool
+    max_steps: int
+    use_anchor: bool
+    proxy: Proxy | None = None
+
+
+class BrowserUseOutput(BaseModel):
+    logged_data: dict[str, list[FunctionLog]]
+    per_step_calls: list[tuple[FunctionLog, dict[str, list[FunctionLog]]]]
+    history: AgentHistoryList
+
+
+class BrowserUseBench(AgentBenchmark[BrowserUseInput, BrowserUseOutput]):
+    def __init__(self, params: BrowserUseInput):
+        super().__init__(params)
+
+    @override
+    async def run_agent(self, task: BenchmarkTask) -> BrowserUseOutput:
+        prompt = f"""You are a helpful web agent.
+        Now you are given the task: {task.question}.
+        Please interact with : {task.url or "the web"} to get the answer.
+        """
+
+        if self.params.proxy is not None:
+            proxy = self.params.proxy.model_dump()
+        else:
+            proxy = None
+
+        llm = ChatOpenAI(model=self.params.model, api_key=SecretStr(os.getenv("OPENAI_API_KEY", "")))
+
+        pool = None
+        wss_url = None
+        if self.params.use_anchor:
+            from notte_integrations.sessions.anchor import AnchorSessionsManager
+
+            pool = AnchorSessionsManager()
+            await pool.start()
+
+            session = pool.create_session_cdp()
+            wss_url = session.cdp_url
+
+        context = None
+        try:
+            browser = Browser(config=BrowserConfig(headless=self.params.headless, cdp_url=wss_url, proxy=proxy))  # type: ignore
+            context = await browser.new_context()
+            agent = BrowserUseAgent(  # type: ignore
+                browser=browser,
+                browser_context=context,
+                task=prompt,
+                llm=llm,
+                use_vision=self.params.use_vision,
+            )
+
+            patcher = AgentPatcher()
+            _ = patcher.log(agent.llm, ["invoke", "ainvoke"])
+            _ = patcher.log(agent, ["step", "run"])  # type: ignore
+
+            result = await agent.run(max_steps=self.params.max_steps)
+        finally:
+            if context is not None:
+                await context.close()
+            if pool is not None:
+                await pool.stop()
+
+        return BrowserUseOutput(
+            logged_data=patcher.logged_data,
+            per_step_calls=patcher.find_encompassed_events("Agent.step"),
+            history=result,
+        )
+
+    @override
+    async def process_output(self, task: BenchmarkTask, out: BrowserUseOutput) -> TaskResult:
+        len_steps = len(out.per_step_calls)
+        len_history = len(out.history.history)
+
+        if len_steps != len_history:
+            logging.error(
+                "Number of step calls isn't the same as the length in history:"
+                + f"{len_steps=}, {len_history=}.\n"
+                + "There will likely be a mismatch."
+            )
+
+        steps: list[Step] = []
+        screenshots: list[str] = []
+        for (step, in_step_calls), hist in zip(out.per_step_calls, out.history.history):
+            screen = hist.state.screenshot
+            if screen is not None:
+                screenshots.append(screen)
+
+            llm_calls: list[LLMCall] = []
+            llm_calls_logs = in_step_calls["BaseChatModel.ainvoke"]
+            for llm_call_log in llm_calls_logs:
+                input_content = json.loads(llm_call_log.input_data)
+
+                input_content = input_content["input"]
+
+                # trim down images
+                trim_image_messages(input_content)
+
+                output_content = json.loads(llm_call_log.output_data)
+                response = output_content["additional_kwargs"]
+                tokens = output_content["response_metadata"]["token_usage"]
+
+                message = ""
+                try:
+                    for tool_call in response["tool_calls"]:
+                        if "function" not in tool_call or "arguments" not in tool_call["function"]:
+                            continue
+
+                        args = BUAgentArguments.model_validate_json(tool_call["function"]["arguments"])
+
+                        message += f"🔎 {args.current_state.evaluation_previous_goal}\n"
+                        message += f"🧠 {args.current_state.memory}\n"
+                        message += f"🎯 {args.current_state.next_goal}\n"
+                        message += "🛠️ Actions: \n"
+                        for action in args.action:
+                            message += f" - {action}\n"
+                except ValidationError:
+                    pass
+
+                llm_calls.append(
+                    LLMCall(
+                        input_tokens=tokens["prompt_tokens"],
+                        output_tokens=tokens["completion_tokens"],
+                        messages_in=input_content,
+                        message_out=response,
+                        pretty_out=message,
+                    )
+                )
+
+            # for llm_call in llm_calls:
+            step = Step(
+                url=hist.state.url,
+                duration_in_s=step.duration_in_s,
+                llm_calls=llm_calls,
+            )
+            steps.append(step)
+
+        last_out = out.history.history[-1].model_output
+
+        # default to the full string of the last output, otherwise pick out the answer if we can
+        answer = str(last_out)
+        try:
+            if last_out is not None:
+                for action in last_out.action:
+                    if hasattr(action, "done"):
+                        answer = typing.cast(DoneAction, getattr(action, "done")).text
+                        break
+        except Exception:
+            answer = str(last_out)
+
+        return TaskResult(
+            success=out.history.is_successful() or False,
+            duration_in_s=out.logged_data["Agent.run"][0].duration_in_s,
+            agent_answer=answer,
+            task=task,
+            steps=steps,
+            screenshots=ScreenshotReplay.from_base64(screenshots),
+        )