optexity 0.1.5.2__tar.gz → 0.1.5.3__tar.gz

{optexity-0.1.5.2 → optexity-0.1.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optexity
-Version: 0.1.5.2
+Version: 0.1.5.3
 Summary: Optexity is a platform for building and running browser and computer agents.
 Author-email: Optexity <founders@optexity.com>
 Requires-Python: >=3.11

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity/inference/core/logging.py

@@ -364,7 +364,14 @@ async def save_latest_memory_state_locally(
             await f.write(json.dumps(task.input_parameters, indent=4))
 
         async with aiofiles.open(step_directory / "secure_parameters.json", "w") as f:
-            await f.write(json.dumps(task.secure_parameters, indent=4))
+            secure_parameters = {
+                key: [
+                    a.model_dump(exclude_none=True, exclude_defaults=True)
+                    for a in value
+                ]
+                for key, value in task.secure_parameters.items()
+            }
+            await f.write(json.dumps(secure_parameters, indent=4))
 
         async with aiofiles.open(step_directory / "generated_variables.json", "w") as f:
             await f.write(json.dumps(memory.variables.generated_variables, indent=4))

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity/inference/core/run_automation.py

@@ -54,11 +54,14 @@ async def run_automation(task: Task, child_process_id: int):
     logger.info(f"Task {task.task_id} started running")
     memory = None
     browser = None
+
     try:
         await start_task_in_server(task)
         memory = Memory()
+
         browser = Browser(
             memory=memory,
+            user_data_dir=f"/tmp/userdata_{task.task_id}",
             headless=False,
             channel=task.automation.browser_channel,
             debug_port=9222 + child_process_id,
@@ -66,9 +69,12 @@ async def run_automation(task: Task, child_process_id: int):
             proxy_session_id=task.proxy_session_id(
                 settings.PROXY_PROVIDER if task.use_proxy else None
             ),
+            is_dedicated=task.is_dedicated,
         )
         await browser.start()
 
+        browser.memory = memory
+
         automation = task.automation
 
         memory.automation_state.step_index = -1

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity/inference/infra/browser.py

@@ -2,35 +2,48 @@ import asyncio
 import base64
 import json
 import logging
+import pathlib
 import re
+import shutil
 from typing import Literal
 from uuid import uuid4
 
+import patchright.async_api
+import playwright.async_api
 from browser_use import Agent, BrowserSession, ChatGoogle
 from browser_use.browser.views import BrowserStateSummary
 from patchright._impl._errors import TimeoutError as PatchrightTimeoutError
 from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
 from playwright.async_api import Download, Locator, Page, Request, Response
 
+from optexity.inference.infra.utils import _download_extension, _extract_extension
 from optexity.schema.memory import Memory, NetworkRequest, NetworkResponse
 from optexity.utils.settings import settings
 
 logger = logging.getLogger(__name__)
 
+_global_playwright: (
+    playwright.async_api.Playwright | patchright.async_api.Playwright | None
+) = None
+_global_context: (
+    playwright.async_api.BrowserContext | patchright.async_api.BrowserContext | None
+) = None
+
 
 class Browser:
     def __init__(
         self,
         memory: Memory,
-        user_data_dir: str = None,
+        user_data_dir: str,
         headless: bool = False,
-        proxy: str = None,
+        proxy: str | None = None,
         stealth: bool = True,
         backend: Literal["browser-use", "browserbase"] = "browser-use",
         debug_port: int = 9222,
         channel: Literal["chromium", "chrome"] = "chromium",
         use_proxy: bool = False,
         proxy_session_id: str | None = None,
+        is_dedicated: bool = False,
     ):
 
         if proxy:
@@ -44,9 +57,15 @@ class Browser:
         self.debug_port = debug_port
         self.use_proxy = use_proxy
         self.proxy_session_id = proxy_session_id
-        self.playwright = None
+        self.playwright: (
+            playwright.async_api.Playwright | patchright.async_api.Playwright | None
+        ) = None
         self.browser = None
-        self.context = None
+        self.context: (
+            playwright.async_api.BrowserContext
+            | patchright.async_api.BrowserContext
+            | None
+        ) = None
         self.page = None
         self.cdp_url = f"http://localhost:{self.debug_port}"
         self.backend_agent = None
@@ -54,16 +73,108 @@ class Browser:
         self.memory = memory
         self.page_to_target_id = []
         self.previous_total_pages = 0
-
+        self.is_dedicated = is_dedicated
         self.active_downloads = 0
         self.all_active_downloads_done = asyncio.Event()
         self.all_active_downloads_done.set()
 
         self.network_calls: list[NetworkResponse | NetworkRequest] = []
 
+        self.extensions = [
+            # {
+            #     "name": "optexity recorder",
+            #     "id": "pbaganbicadeoacahamnbgohafchgakp",
+            #     "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dpbaganbicadeoacahamnbgohafchgakp%26uc",
+            # },
+            {
+                "name": "I still don't care about cookies",
+                "id": "edibdbjcniadpccecjdfdjjppcpchdlm",
+                "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dedibdbjcniadpccecjdfdjjppcpchdlm%26uc",
+            },
+            # {
+            #     "name": "popupoff",
+            #     "id": "kiodaajmphnkcajieajajinghpejdjai",
+            #     "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dkiodaajmphnkcajieajajinghpejdjai%26uc",
+            # },
+            {
+                "name": "ublock origin",
+                "id": "ddkjiahejlhfcafbddmgiahcphecmpfh",
+                "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc",
+            },
+        ]
+
     async def start(self):
+        global _global_playwright, _global_context
         logger.debug("Starting browser")
         try:
+            cache_dir = pathlib.Path("/tmp/extensions")
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            extension_paths = []
+            loaded_extension_names = []
+            for ext in self.extensions:
+                ext_dir = cache_dir / ext["id"]
+                crx_file = cache_dir / f'{ext["id"]}.crx'
+
+                # Check if extension is already extracted
+                if ext_dir.exists() and (ext_dir / "manifest.json").exists():
+                    logger.info(
+                        f'✅ Using cached {ext["name"]} extension from {ext_dir}'
+                    )
+                    extension_paths.append(str(ext_dir))
+                    loaded_extension_names.append(ext["name"])
+                    continue
+
+                try:
+                    # Download extension if not cached
+                    if not crx_file.exists():
+                        logger.info(f'📦 Downloading {ext["name"]} extension...')
+                        _download_extension(ext["url"], crx_file)
+                    else:
+                        logger.info(f'📦 Found cached {ext["name"]} .crx file')
+
+                    # Extract extension
+                    logger.info(f'📂 Extracting {ext["name"]} extension...')
+                    _extract_extension(crx_file, ext_dir)
+
+                    extension_paths.append(str(ext_dir))
+                    loaded_extension_names.append(ext["name"])
+                    logger.info(f'✅ Successfully loaded {ext["name"]}')
+
+                except Exception as e:
+                    logger.error(
+                        f'❌ Failed to setup {ext["name"]} extension: {e}',
+                        exc_info=True,
+                    )
+                    continue
+
+            if not extension_paths:
+                logger.error("⚠️ No extensions were loaded successfully!")
+
+            logger.info(f"Loaded extensions: {', '.join(loaded_extension_names)}")
+
+            args = [
+                "--disable-site-isolation-trials",
+                "--disable-web-security",
+                "--disable-features=IsolateOrigins,site-per-process",
+                "--allow-running-insecure-content",
+                "--ignore-certificate-errors",
+                "--ignore-ssl-errors",
+                "--ignore-certificate-errors-spki-list",
+                "--enable-extensions",
+                "--disable-extensions-file-access-check",
+                "--disable-extensions-http-throttling",
+            ]
+
+            if extension_paths:
+                disable_except = (
+                    f'--disable-extensions-except={",".join(extension_paths)}'
+                )
+                load_extension = f'--load-extension={",".join(extension_paths)}'
+                args.append(disable_except)
+                args.append(load_extension)
+                logger.info(f"Extension args: {disable_except}")
+                logger.info(f"Extension args: {load_extension}")
+
             if self.playwright is not None:
                 await self.playwright.stop()
 
@@ -98,44 +209,64 @@ class Browser:
                 if settings.PROXY_PASSWORD is not None:
                     proxy["password"] = settings.PROXY_PASSWORD
 
-            self.playwright = await async_playwright().start()
-            self.browser = await self.playwright.chromium.launch(
-                channel=self.channel,
-                headless=self.headless,
-                proxy=proxy,
-                args=[
-                    "--start-fullscreen",
-                    "--disable-popup-blocking",
-                    "--window-size=1920,1080",
-                    f"--remote-debugging-port={self.debug_port}",
-                    "--disable-gpu",
-                    "--disable-extensions",
-                    "--disable-background-networking",
-                ],
-                chromium_sandbox=False,
-            )
+            if (
+                _global_playwright is None
+                or _global_context is None
+                or not self.is_dedicated
+            ):
+                self.playwright = await async_playwright().start()
+                self.context = await self.playwright.chromium.launch_persistent_context(
+                    channel=self.channel,
+                    user_data_dir=self.user_data_dir,
+                    headless=self.headless,
+                    proxy=proxy,
+                    args=[
+                        # "--start-fullscreen",
+                        "--disable-popup-blocking",
+                        "--window-size=1920,1080",
+                        f"--remote-debugging-port={self.debug_port}",
+                        "--disable-gpu",
+                        "--disable-background-networking",
+                    ]
+                    + args,
+                    chromium_sandbox=False,
+                    no_viewport=True,
+                )
+                _global_playwright = self.playwright
+                _global_context = self.context
 
-            self.context = await self.browser.new_context(
-                no_viewport=True, ignore_https_errors=True
-            )
+                async def log_request(req: Request):
+                    await self.log_request(req)
 
-            async def log_request(req: Request):
-                await self.log_request(req)
+                async def handle_random_download(download: Download):
+                    await self.handle_random_download(download)
 
-            async def handle_random_download(download: Download):
-                await self.handle_random_download(download)
+                async def handle_random_url_downloads(resp: Response):
+                    await self.handle_random_url_downloads(resp)
 
-            async def handle_random_url_downloads(resp: Response):
-                await self.handle_random_url_downloads(resp)
+                self.context.on("request", log_request)
+                self.context.on("response", handle_random_url_downloads)
 
-            self.context.on("request", log_request)
-            self.context.on("response", handle_random_url_downloads)
+                self.context.on(
+                    "page", lambda p: (p.on("download", handle_random_download))
+                )
 
-            self.context.on(
-                "page", lambda p: (p.on("download", handle_random_download))
-            )
+            elif self.is_dedicated:
+                self.context = _global_context
+                self.playwright = _global_playwright
+                for i in range(len(self.context.pages) - 1, 0, -1):
+                    await self.context.pages[i].close()
+            else:
+                raise ValueError(
+                    "Browser is not dedicated and global playwright and context are not set"
+                )
 
-            self.page = await self.context.new_page()
+            # self.context = await self.browser.new_context(
+            #     no_viewport=True, ignore_https_errors=True
+            # )
+
+            # self.page = await self.context.new_page()
+            self.page = self.context.pages[0]
 
             browser_session = BrowserSession(cdp_url=self.cdp_url, keep_alive=True)
 
@@ -162,7 +293,7 @@ class Browser:
             raise e
 
     async def stop(self):
-        logger.debug("Stopping full system")
+        logger.debug("Stopping backend agent")
         if self.backend_agent is not None:
             logger.debug("Stopping backend agent")
             self.backend_agent.stop()
@@ -174,21 +305,25 @@ class Browser:
                 logger.debug("Browser session reset")
             self.backend_agent = None
 
-        if self.context is not None:
-            logger.debug("Stopping context")
-            await self.context.close()
-            self.context = None
-
-        if self.browser is not None:
-            logger.debug("Stopping browser")
-            await self.browser.close()
-            self.browser = None
-
-        if self.playwright is not None:
-            logger.debug("Stopping playwright")
-            await self.playwright.stop()
-            self.playwright = None
-        logger.debug("Full system stopped")
+        if not self.is_dedicated:
+            logger.debug("Stopping context and playwright and browser as not dedicated")
+            if self.context is not None:
+                logger.debug("Stopping context")
+                await self.context.close()
+                self.context = None
+
+            if self.browser is not None:
+                logger.debug("Stopping browser")
+                await self.browser.close()
+                self.browser = None
+
+            if self.playwright is not None:
+                logger.debug("Stopping playwright")
+                await self.playwright.stop()
+                self.playwright = None
+            shutil.rmtree(self.user_data_dir, ignore_errors=True)
+        else:
+            logger.debug("browser not stopped as dedicated")
 
     async def get_current_page(self) -> Page | None:
         if self.context is None:

optexity-0.1.5.3/optexity/inference/infra/utils.py

@@ -0,0 +1,98 @@
+import json
+import logging
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def _download_extension(url: str, output_path: Path) -> None:
+    """Download extension .crx file."""
+    import urllib.request
+
+    try:
+        logger.info(f"Downloading from: {url}")
+        with urllib.request.urlopen(url) as response:
+            content = response.read()
+            logger.info(f"Downloaded {len(content)} bytes")
+            with open(output_path, "wb") as f:
+                f.write(content)
+        logger.info(f"Saved to: {output_path}")
+    except Exception as e:
+        raise Exception(f"Failed to download extension: {e}")
+
+
+def _extract_extension(crx_path: Path, extract_dir: Path) -> None:
+    """Extract .crx file to directory."""
+    import os
+    import shutil
+    import zipfile
+
+    # Remove existing directory
+    if extract_dir.exists():
+        shutil.rmtree(extract_dir)
+
+    extract_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        # CRX files are ZIP files with a header, try to extract as ZIP
+        with zipfile.ZipFile(crx_path, "r") as zip_ref:
+            zip_ref.extractall(extract_dir)
+
+        # Verify manifest exists
+        if not (extract_dir / "manifest.json").exists():
+            raise Exception("No manifest.json found in extension")
+
+        logger.info("✅ Extracted as regular ZIP file")
+
+    except zipfile.BadZipFile:
+        logger.info("📦 Processing CRX header...")
+        # CRX files have a header before the ZIP data
+        with open(crx_path, "rb") as f:
+            # Read CRX header to find ZIP start
+            magic = f.read(4)
+            if magic != b"Cr24":
+                raise Exception(f"Invalid CRX file format. Magic: {magic}")
+
+            version = int.from_bytes(f.read(4), "little")
+            logger.info(f"CRX version: {version}")
+
+            if version == 2:
+                pubkey_len = int.from_bytes(f.read(4), "little")
+                sig_len = int.from_bytes(f.read(4), "little")
+                f.seek(16 + pubkey_len + sig_len)
+            elif version == 3:
+                header_len = int.from_bytes(f.read(4), "little")
+                f.seek(12 + header_len)
+            else:
+                raise Exception(f"Unsupported CRX version: {version}")
+
+            # Extract ZIP data
+            zip_data = f.read()
+            logger.info(f"ZIP data size: {len(zip_data)} bytes")
+
+        # Write ZIP data to temp file and extract
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp_zip:
+            temp_zip.write(zip_data)
+            temp_zip.flush()
+
+            with zipfile.ZipFile(temp_zip.name, "r") as zip_ref:
+                zip_ref.extractall(extract_dir)
+
+            os.unlink(temp_zip.name)
+
+    # Remove 'key' from manifest if present (can cause issues)
+    manifest_path = extract_dir / "manifest.json"
+    if manifest_path.exists():
+        data = json.loads(manifest_path.read_text())
+        logger.info(f"Manifest version: {data.get('manifest_version')}")
+        logger.info(f"Extension name: {data.get('name')}")
+
+        if "key" in data:
+            logger.info("Removing 'key' field from manifest")
+            del data["key"]
+            manifest_path.write_text(json.dumps(data, indent=2))
+    else:
+        raise Exception("manifest.json not found after extraction")

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity/schema/task.py

@@ -72,6 +72,7 @@ class Task(BaseModel):
     max_retries: int = 1
     api_key: str
     callback_url: CallbackUrl | None = None
+    is_dedicated: bool = False
 
     class Config:
         json_encoders = {datetime: lambda v: v.isoformat() if v is not None else None}

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity/utils/utils.py

@@ -3,6 +3,7 @@ import logging
 import os
 from pathlib import Path
 from typing import List, Optional
+from urllib.parse import urlparse
 
 import aiofiles
 import pyotp
@@ -74,3 +75,16 @@ async def get_onepassword_value(vault_name: str, item_name: str, field_name: str
     )
 
     return str_value
+
+
+def clean_url(url: str) -> str:
+    if not url.startswith(("http://", "https://")):
+        url = "http://" + url  # needed for urlparse
+
+    parsed = urlparse(url)
+    domain = parsed.netloc.lower()
+
+    if domain.startswith("www."):
+        domain = domain[4:]
+
+    return domain

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optexity
-Version: 0.1.5.2
+Version: 0.1.5.3
 Summary: Optexity is a platform for building and running browser and computer agents.
 Author-email: Optexity <founders@optexity.com>
 Requires-Python: >=3.11

{optexity-0.1.5.2 → optexity-0.1.5.3}/optexity.egg-info/SOURCES.txt

@@ -61,6 +61,7 @@ optexity/inference/core/two_factor_auth/__init__.py
 optexity/inference/infra/__init__.py
 optexity/inference/infra/browser.py
 optexity/inference/infra/browser_extension.py
+optexity/inference/infra/utils.py
 optexity/inference/models/__init__.py
 optexity/inference/models/gemini.py
 optexity/inference/models/human.py

{optexity-0.1.5.2 → optexity-0.1.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "optexity"
-version = "0.1.5.2"
+version = "0.1.5.3"
 readme = "README.md"
 description = "Optexity is a platform for building and running browser and computer agents."
 authors = [{ name = "Optexity", email = "founders@optexity.com" }]