lumivor 0.1.7__py3-none-any.whl

lumivor/browser/tests/screenshot_test.py

@@ -0,0 +1,38 @@
+import base64
+
+import pytest
+
+from lumivor.browser.browser import Browser, BrowserConfig
+
+
+@pytest.fixture
+async def browser():
+    browser_service = Browser(config=BrowserConfig(headless=True))
+    yield browser_service
+
+    await browser_service.close()
+
+
+# @pytest.mark.skip(reason='takes too long')
+def test_take_full_page_screenshot(browser):
+    # Go to a test page
+    browser.go_to_url('https://example.com')
+
+    # Take full page screenshot
+    screenshot_b64 = browser.take_screenshot(full_page=True)
+
+    # Verify screenshot is not empty and is valid base64
+    assert screenshot_b64 is not None
+    assert isinstance(screenshot_b64, str)
+    assert len(screenshot_b64) > 0
+
+    # Test we can decode the base64 string
+    try:
+        base64.b64decode(screenshot_b64)
+    except Exception as e:
+        pytest.fail(f'Failed to decode base64 screenshot: {str(e)}')
+
+
+if __name__ == '__main__':
+    test_take_full_page_screenshot(
+        Browser(config=BrowserConfig(headless=False)))

lumivor/browser/tests/test_clicks.py

@@ -0,0 +1,77 @@
+import asyncio
+import json
+
+import pytest
+
+from lumivor.browser.browser import Browser, BrowserConfig
+from lumivor.dom.views import ElementTreeSerializer
+from lumivor.utils import time_execution_sync
+
+
+# run with: pytest lumivor/browser/tests/test_clicks.py
+@pytest.mark.asyncio
+async def test_highlight_elements():
+    browser = Browser(config=BrowserConfig(
+        headless=False, disable_security=True))
+
+    async with await browser.new_context() as context:
+        page = await context.get_current_page()
+        # await page.goto('https://immobilienscout24.de')
+        # await page.goto('https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/service-plans')
+        # await page.goto('https://google.com/search?q=elon+musk')
+        # await page.goto('https://kayak.com')
+        # await page.goto('https://www.w3schools.com/tags/tryit.asp?filename=tryhtml_iframe')
+        # await page.goto('https://dictionary.cambridge.org')
+        # await page.goto('https://github.com')
+        await page.goto('https://huggingface.co/')
+
+        await asyncio.sleep(1)
+
+        while True:
+            try:
+                # await asyncio.sleep(10)
+                state = await context.get_state()
+
+                with open('./tmp/page.json', 'w') as f:
+                    json.dump(
+                        ElementTreeSerializer.dom_element_node_to_json(
+                            state.element_tree),
+                        f,
+                        indent=1,
+                    )
+
+                # await time_execution_sync('highlight_selector_map_elements')(
+                # 	browser.highlight_selector_map_elements
+                # )(state.selector_map)
+
+                # Find and print duplicate XPaths
+                xpath_counts = {}
+                if not state.selector_map:
+                    continue
+                for selector in state.selector_map.values():
+                    xpath = selector.xpath
+                    if xpath in xpath_counts:
+                        xpath_counts[xpath] += 1
+                    else:
+                        xpath_counts[xpath] = 1
+
+                print('\nDuplicate XPaths found:')
+                for xpath, count in xpath_counts.items():
+                    if count > 1:
+                        print(f'XPath: {xpath}')
+                        print(f'Count: {count}\n')
+
+                print(list(state.selector_map.keys()), 'Selector map keys')
+                print(state.element_tree.clickable_elements_to_string())
+                action = input('Select next action: ')
+
+                await time_execution_sync('remove_highlight_elements')(context.remove_highlights)()
+
+                node_element = state.selector_map[int(action)]
+
+                # check if index of selector map are the same as index of items in dom_items
+
+                await context._click_element_node(node_element)
+
+            except Exception as e:
+                print(e)

lumivor/browser/views.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+from lumivor.dom.history_tree_processor.service import DOMHistoryElement
+from lumivor.dom.views import DOMState
+
+
+# Pydantic
+class TabInfo(BaseModel):
+    """Represents information about a browser tab"""
+
+    page_id: int
+    url: str
+    title: str
+
+
+@dataclass
+class BrowserState(DOMState):
+    url: str
+    title: str
+    tabs: list[TabInfo]
+    screenshot: Optional[str] = None
+
+
+@dataclass
+class BrowserStateHistory:
+    url: str
+    title: str
+    tabs: list[TabInfo]
+    interacted_element: list[DOMHistoryElement | None] | list[None]
+    screenshot: Optional[str] = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data = {}
+        data['tabs'] = [tab.model_dump() for tab in self.tabs]
+        data['screenshot'] = self.screenshot
+        data['interacted_element'] = [
+            el.to_dict() if el else None for el in self.interacted_element
+        ]
+        data['url'] = self.url
+        data['title'] = self.title
+        return data
+
+
+class BrowserError(Exception):
+    """Base class for all browser errors"""

lumivor/controller/registry/service.py

@@ -0,0 +1,140 @@
+import asyncio
+from inspect import iscoroutinefunction, signature
+from typing import Any, Callable, Optional, Type
+
+from pydantic import BaseModel, create_model
+
+from lumivor.browser.context import BrowserContext
+from lumivor.controller.registry.views import (
+    ActionModel,
+    ActionRegistry,
+    RegisteredAction,
+)
+from lumivor.telemetry.service import ProductTelemetry
+from lumivor.telemetry.views import (
+    ControllerRegisteredFunctionsTelemetryEvent,
+    RegisteredFunction,
+)
+
+
+class Registry:
+    """Service for registering and managing actions"""
+
+    def __init__(self):
+        self.registry = ActionRegistry()
+        self.telemetry = ProductTelemetry()
+
+    def _create_param_model(self, function: Callable) -> Type[BaseModel]:
+        """Creates a Pydantic model from function signature"""
+        sig = signature(function)
+        params = {
+            name: (param.annotation, ... if param.default ==
+                   param.empty else param.default)
+            for name, param in sig.parameters.items()
+            if name != 'browser'
+        }
+        # TODO: make the types here work
+        return create_model(
+            f'{function.__name__}Params',
+            __base__=ActionModel,
+            **params,  # type: ignore
+        )
+
+    def action(
+            self,
+            description: str,
+            param_model: Optional[Type[BaseModel]] = None,
+            requires_browser: bool = False,
+    ):
+        """Decorator for registering actions"""
+
+        def decorator(func: Callable):
+            # Create param model from function if not provided
+            actual_param_model = param_model or self._create_param_model(func)
+
+            # Wrap sync functions to make them async
+            if not iscoroutinefunction(func):
+
+                async def async_wrapper(*args, **kwargs):
+                    return await asyncio.to_thread(func, *args, **kwargs)
+
+                # Copy the signature and other metadata from the original function
+                async_wrapper.__signature__ = signature(func)
+                async_wrapper.__name__ = func.__name__
+                async_wrapper.__annotations__ = func.__annotations__
+                wrapped_func = async_wrapper
+            else:
+                wrapped_func = func
+
+            action = RegisteredAction(
+                name=func.__name__,
+                description=description,
+                function=wrapped_func,
+                param_model=actual_param_model,
+                requires_browser=requires_browser,
+            )
+            self.registry.actions[func.__name__] = action
+            return func
+
+        return decorator
+
+    async def execute_action(
+            self, action_name: str, params: dict, browser: Optional[BrowserContext] = None
+    ) -> Any:
+        """Execute a registered action"""
+        if action_name not in self.registry.actions:
+            raise ValueError(f'Action {action_name} not found')
+
+        action = self.registry.actions[action_name]
+        try:
+            # Create the validated Pydantic model
+            validated_params = action.param_model(**params)
+
+            # Check if the first parameter is a Pydantic model
+            sig = signature(action.function)
+            parameters = list(sig.parameters.values())
+            is_pydantic = parameters and issubclass(
+                parameters[0].annotation, BaseModel)
+
+            # Prepare arguments based on parameter type
+            if action.requires_browser:
+                if not browser:
+                    raise ValueError(
+                        f'Action {
+                            action_name} requires browser but none provided. This has to be used in combination of `requires_browser=True` when registering the action.'
+                    )
+                if is_pydantic:
+                    return await action.function(validated_params, browser=browser)
+                return await action.function(**validated_params.model_dump(), browser=browser)
+
+            if is_pydantic:
+                return await action.function(validated_params)
+            return await action.function(**validated_params.model_dump())
+
+        except Exception as e:
+            raise RuntimeError(f'Error executing action {
+                               action_name}: {str(e)}') from e
+
+    def create_action_model(self) -> Type[ActionModel]:
+        """Creates a Pydantic model from registered actions"""
+        fields = {
+            name: (Optional[action.param_model], None)
+            for name, action in self.registry.actions.items()
+        }
+
+        self.telemetry.capture(
+            ControllerRegisteredFunctionsTelemetryEvent(
+                registered_functions=[
+                    RegisteredFunction(
+                        name=name, params=action.param_model.model_json_schema())
+                    for name, action in self.registry.actions.items()
+                ]
+            )
+        )
+
+        # type:ignore
+        return create_model('ActionModel', __base__=ActionModel, **fields)
+
+    def get_prompt_description(self) -> str:
+        """Get a description of all actions for the prompt"""
+        return self.registry.get_prompt_description()

lumivor/controller/registry/views.py

@@ -0,0 +1,71 @@
+from typing import Callable, Dict, Type
+
+from pydantic import BaseModel, ConfigDict
+
+
+class RegisteredAction(BaseModel):
+	"""Model for a registered action"""
+
+	name: str
+	description: str
+	function: Callable
+	param_model: Type[BaseModel]
+	requires_browser: bool = False
+
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	def prompt_description(self) -> str:
+		"""Get a description of the action for the prompt"""
+		skip_keys = ['title']
+		s = f'{self.description}: \n'
+		s += '{' + str(self.name) + ': '
+		s += str(
+			{
+				k: {sub_k: sub_v for sub_k, sub_v in v.items() if sub_k not in skip_keys}
+				for k, v in self.param_model.schema()['properties'].items()
+			}
+		)
+		s += '}'
+		return s
+
+
+class ActionModel(BaseModel):
+	"""Base model for dynamically created action models"""
+
+	# this will have all the registered actions, e.g.
+	# click_element = param_model = ClickElementParams
+	# done = param_model = None
+	#
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	def get_index(self) -> int | None:
+		"""Get the index of the action"""
+		# {'clicked_element': {'index':5}}
+		params = self.model_dump(exclude_unset=True).values()
+		if not params:
+			return None
+		for param in params:
+			if param is not None and 'index' in param:
+				return param['index']
+		return None
+
+	def set_index(self, index: int):
+		"""Overwrite the index of the action"""
+		# Get the action name and params
+		action_data = self.model_dump(exclude_unset=True)
+		action_name = next(iter(action_data.keys()))
+		action_params = getattr(self, action_name)
+
+		# Update the index directly on the model
+		if hasattr(action_params, 'index'):
+			action_params.index = index
+
+
+class ActionRegistry(BaseModel):
+	"""Model representing the action registry"""
+
+	actions: Dict[str, RegisteredAction] = {}
+
+	def get_prompt_description(self) -> str:
+		"""Get a description of all actions for the prompt"""
+		return '\n'.join([action.prompt_description() for action in self.actions.values()])