webpilot-engine 2.0.0__py3-none-any.whl

core/__init__.py

@@ -0,0 +1,45 @@
+"""Core package for universal web applier."""
+
+from core.models import (
+    ProxyConfig,
+    DeviceProfile,
+    InputField,
+    DropdownField,
+    ChoiceGroup,
+    FileUploadField,
+    ActionButton,
+    InspectionResult,
+    FillSummary,
+    ReactiveClickOutcome,
+)
+from core.exceptions import (
+    UniversalApplierError,
+    ConfigurationError,
+    ValidationError,
+    AutomationError,
+    ElementNotFoundError,
+    InteractionTimeoutError,
+    DocumentUploadError,
+    AuthenticationError,
+)
+
+__all__ = [
+    "ProxyConfig",
+    "DeviceProfile",
+    "InputField",
+    "DropdownField",
+    "ChoiceGroup",
+    "FileUploadField",
+    "ActionButton",
+    "InspectionResult",
+    "FillSummary",
+    "ReactiveClickOutcome",
+    "UniversalApplierError",
+    "ConfigurationError",
+    "ValidationError",
+    "AutomationError",
+    "ElementNotFoundError",
+    "InteractionTimeoutError",
+    "DocumentUploadError",
+    "AuthenticationError",
+]

core/exceptions.py

@@ -0,0 +1,33 @@
+"""Typed exception hierarchy for Universal Web Applier."""
+
+class UniversalApplierError(Exception):
+    """Base exception for all universal web applier domain errors."""
+    pass
+
+class ConfigurationError(UniversalApplierError):
+    """Raised when configuration values, files, or device profiles are missing or invalid."""
+    pass
+
+class ValidationError(UniversalApplierError):
+    """Raised when input data, payloads, or schema validation fails."""
+    pass
+
+class AutomationError(UniversalApplierError):
+    """Raised when web automation actions fail unexpectedly."""
+    pass
+
+class ElementNotFoundError(AutomationError):
+    """Raised when an expected DOM element cannot be located."""
+    pass
+
+class InteractionTimeoutError(AutomationError):
+    """Raised when an action or wait condition exceeds its deadline."""
+    pass
+
+class DocumentUploadError(AutomationError):
+    """Raised when document attachment fails."""
+    pass
+
+class AuthenticationError(AutomationError):
+    """Raised when login or credential submission fails."""
+    pass

core/models.py

@@ -0,0 +1,273 @@
+"""Shared domain Transfer Objects (DTOs) and data models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple, Union
+from core.exceptions import ValidationError
+
+
+@dataclass
+class ProxyConfig:
+    """Network proxy configuration loaded from settings."""
+    server: str
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+    def to_playwright_dict(self) -> Dict[str, str]:
+        """Convert to Playwright-compatible proxy dictionary."""
+        d: Dict[str, str] = {"server": self.server}
+        if self.username:
+            d["username"] = self.username
+        if self.password:
+            d["password"] = self.password
+        return d
+
+
+@dataclass
+class DeviceProfile:
+    """Device & browser profile parameters for browser context emulation."""
+    device_name: str
+    user_agent: str
+    viewport_width: int
+    viewport_height: int
+    device_scale_factor: float = 1.0
+    is_mobile: bool = False
+    has_touch: bool = False
+    locale: str = "en-US"
+    timezone_id: str = "UTC"
+    geolocation_latitude: Optional[float] = None
+    geolocation_longitude: Optional[float] = None
+    geolocation_accuracy: Optional[float] = None
+    permissions: List[str] = field(default_factory=list)
+    platform: str = "Win32"
+    vendor: str = "Google Inc."
+
+    def viewport_dict(self) -> Dict[str, int]:
+        """Returns the viewport dimensions dictionary."""
+        return {"width": self.viewport_width, "height": self.viewport_height}
+
+    def geolocation_dict(self) -> Optional[Dict[str, float]]:
+        """Returns geolocation dictionary if valid coordinates are present."""
+        if self.geolocation_latitude is not None and self.geolocation_longitude is not None:
+            return {
+                "latitude": float(self.geolocation_latitude),
+                "longitude": float(self.geolocation_longitude),
+                "accuracy": float(self.geolocation_accuracy or 10.0),
+            }
+        return None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "DeviceProfile":
+        """Instantiate a DeviceProfile strictly from parsed dictionary data."""
+        if not isinstance(data, dict):
+            raise ValidationError(f"Expected dict for DeviceProfile, got {type(data).__name__}")
+
+        viewport = data.get("viewport") or {}
+        vp_width = data.get("viewport_width") or viewport.get("width")
+        vp_height = data.get("viewport_height") or viewport.get("height")
+
+        geo = data.get("geolocation") or {}
+        geo_lat = data.get("geolocation_latitude", geo.get("latitude"))
+        geo_lon = data.get("geolocation_longitude", geo.get("longitude"))
+        geo_acc = data.get("geolocation_accuracy", geo.get("accuracy"))
+
+        return cls(
+            device_name=str(data.get("device_name", "")),
+            user_agent=str(data.get("user_agent", "")),
+            viewport_width=int(vp_width) if vp_width is not None else 0,
+            viewport_height=int(vp_height) if vp_height is not None else 0,
+            device_scale_factor=float(data.get("device_scale_factor", 1.0)),
+            is_mobile=bool(data.get("is_mobile", False)),
+            has_touch=bool(data.get("has_touch", False)),
+            locale=str(data.get("locale", "en-US")),
+            timezone_id=str(data.get("timezone_id", "UTC")),
+            geolocation_latitude=float(geo_lat) if geo_lat is not None else None,
+            geolocation_longitude=float(geo_lon) if geo_lon is not None else None,
+            geolocation_accuracy=float(geo_acc) if geo_acc is not None else None,
+            permissions=list(data.get("permissions", [])),
+            platform=str(data.get("platform", "Win32")),
+            vendor=str(data.get("vendor", "Google Inc.")),
+        )
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert DeviceProfile to serializable dictionary."""
+        d: Dict[str, Any] = {
+            "device_name": self.device_name,
+            "user_agent": self.user_agent,
+            "viewport": self.viewport_dict(),
+            "device_scale_factor": self.device_scale_factor,
+            "is_mobile": self.is_mobile,
+            "has_touch": self.has_touch,
+            "locale": self.locale,
+            "timezone_id": self.timezone_id,
+            "permissions": self.permissions,
+            "platform": self.platform,
+            "vendor": self.vendor,
+        }
+        geo = self.geolocation_dict()
+        if geo:
+            d["geolocation"] = geo
+        return d
+
+
+@dataclass
+class InputField:
+    """Inspected text or numeric input element."""
+    id: str
+    label: str
+    type: str = "text"
+    required: bool = False
+    value: str = ""
+
+
+@dataclass
+class DropdownField:
+    """Inspected select dropdown or combobox picklist."""
+    id: str
+    label: str
+    type: str = "select"
+    required: bool = False
+    value: str = ""
+    trigger_id: str = ""
+    sample_options: List[str] = field(default_factory=list)
+    options_count: int = 0
+
+
+@dataclass
+class ChoiceGroup:
+    """Inspected radio or checkbox group for a question."""
+    question: str
+    type: str = "radio"
+    required: bool = False
+    options: List[str] = field(default_factory=list)
+    selected: Optional[str] = None
+
+
+@dataclass
+class FileUploadField:
+    """Inspected file upload zone or attachment input."""
+    id: str
+    label: str
+    required: bool = False
+
+
+@dataclass
+class ActionButton:
+    """Inspected form action button."""
+    id: str
+    text: str
+    action: str = "action"
+
+
+@dataclass
+class InspectionResult:
+    """Consolidated form inspection schema."""
+    title: str = ""
+    url: str = ""
+    sections: List[str] = field(default_factory=list)
+    inputs: List[InputField] = field(default_factory=list)
+    dropdowns: List[DropdownField] = field(default_factory=list)
+    choices: List[ChoiceGroup] = field(default_factory=list)
+    file_uploads: List[FileUploadField] = field(default_factory=list)
+    already_uploaded_files: List[str] = field(default_factory=list)
+    buttons: List[ActionButton] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to JSON-serializable dictionary matching legacy schema format."""
+        return {
+            "title": self.title,
+            "url": self.url,
+            "sections": self.sections,
+            "inputs": [vars(i) for i in self.inputs],
+            "dropdowns": [vars(d) for d in self.dropdowns],
+            "choices": [vars(c) for c in self.choices],
+            "file_uploads": [vars(f) for f in self.file_uploads],
+            "already_uploaded_files": self.already_uploaded_files,
+            "buttons": [vars(b) for b in self.buttons],
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "InspectionResult":
+        """Instantiate DTO from raw inspection dictionary."""
+        return cls(
+            title=data.get("title", ""),
+            url=data.get("url", ""),
+            sections=list(data.get("sections", [])),
+            inputs=[InputField(**i) if isinstance(i, dict) else i for i in data.get("inputs", [])],
+            dropdowns=[DropdownField(**d) if isinstance(d, dict) else d for d in data.get("dropdowns", [])],
+            choices=[ChoiceGroup(**c) if isinstance(c, dict) else c for c in data.get("choices", [])],
+            file_uploads=[FileUploadField(**f) if isinstance(f, dict) else f for f in data.get("file_uploads", [])],
+            already_uploaded_files=list(data.get("already_uploaded_files", [])),
+            buttons=[ActionButton(**b) if isinstance(b, dict) else b for b in data.get("buttons", [])],
+        )
+
+
+@dataclass
+class FillSummary:
+    """Summary record for batch sequential form filling."""
+    confirmed: List[Tuple[str, Any]] = field(default_factory=list)
+    unconfirmed: List[Tuple[str, Any]] = field(default_factory=list)
+    failed: List[Tuple[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, List[Tuple[str, Any]]]:
+        """Return standard dictionary format matching legacy API."""
+        return {
+            "confirmed": list(self.confirmed),
+            "unconfirmed": list(self.unconfirmed),
+            "failed": list(self.failed),
+        }
+
+
+@dataclass
+class ReactiveClickOutcome:
+    """Outcome and reactive environment mutations triggered by a button press."""
+    success: bool
+    target: str
+    redirected: bool = False
+    new_window: bool = False
+    new_url: Optional[str] = None
+    modal_opened: bool = False
+    modal_text: str = ""
+    new_inputs_count: int = 0
+    schema: Optional[Dict[str, Any]] = None
+    alerts: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to standard dictionary format matching legacy API."""
+        return {
+            "success": self.success,
+            "target": self.target,
+            "redirected": self.redirected,
+            "new_window": self.new_window,
+            "new_url": self.new_url,
+            "modal_opened": self.modal_opened,
+            "modal_text": self.modal_text,
+            "new_inputs_count": self.new_inputs_count,
+            "schema": self.schema,
+            "alerts": self.alerts,
+        }
+
+
+@dataclass
+class ApplyExecutionResult:
+    """Structured result returned upon completion of an apply flow."""
+    confirmed_fields: List[Tuple[str, Any]] = field(default_factory=list)
+    unconfirmed_fields: List[Tuple[str, Any]] = field(default_factory=list)
+    failed_fields: List[Tuple[str, Any]] = field(default_factory=list)
+    final_url: str = ""
+    validation_errors: List[str] = field(default_factory=list)
+    submitted: bool = False
+    confirmation_screenshot: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary matching standard serialization."""
+        return {
+            "confirmed_fields": list(self.confirmed_fields),
+            "unconfirmed_fields": list(self.unconfirmed_fields),
+            "failed_fields": list(self.failed_fields),
+            "final_url": self.final_url,
+            "validation_errors": list(self.validation_errors),
+            "submitted": self.submitted,
+            "confirmation_screenshot": self.confirmation_screenshot,
+        }

engine.py

@@ -0,0 +1,199 @@
+"""Universal Form Engine Facade.
+
+Maintains 100% backwards-compatible external behavior while delegating to
+cohesive domain services and adapters.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+from playwright.sync_api import Browser, BrowserContext, Page
+
+from adapters.browser_adapter import BrowserAdapter
+from adapters.upload_adapter import UploadAdapter
+from config.settings import Settings, get_settings, load_device_profile
+from core.models import DeviceProfile
+from services.auth_navigation_service import AuthNavigationService
+from services.field_interaction_service import FieldInteractionService
+from services.form_filler_service import FormFillerService
+from services.inspection_formatter_service import InspectionFormatterService
+from services.inspection_service import InspectionService
+from services.reactive_interaction_service import ReactiveInteractionService
+from validators.form_validator import FormValidator
+
+
+class UniversalFormEngine:
+    """Universal Form Engine facade coordinating browser lifecycle, inspection, and interaction services."""
+
+    def __init__(
+        self,
+        headless: Optional[bool] = None,
+        settings: Optional[Settings] = None,
+        device_profile: Optional[DeviceProfile] = None,
+        viewport: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+    ):
+        self.settings = settings or get_settings()
+        self.device_profile = device_profile or load_device_profile()
+
+        if viewport:
+            self.device_profile.viewport_width = viewport.get("width", self.device_profile.viewport_width)
+            self.device_profile.viewport_height = viewport.get("height", self.device_profile.viewport_height)
+        if user_agent:
+            self.device_profile.user_agent = user_agent
+
+        self.headless = headless if headless is not None else self.settings.headless
+        self.viewport = self.device_profile.viewport_dict()
+        self.user_agent = self.device_profile.user_agent
+
+        self.browser_adapter = BrowserAdapter(
+            headless=self.headless,
+            settings=self.settings,
+            device_profile=self.device_profile,
+        )
+
+        # Service instances initialized upon start()
+        self.auth_service: Optional[AuthNavigationService] = None
+        self.inspection_service: Optional[InspectionService] = None
+        self.field_service: Optional[FieldInteractionService] = None
+        self.filler_service: Optional[FormFillerService] = None
+        self.upload_adapter: Optional[UploadAdapter] = None
+        self.validator: Optional[FormValidator] = None
+        self.reactive_service: Optional[ReactiveInteractionService] = None
+
+    @property
+    def browser(self) -> Optional[Browser]:
+        """Underlying Playwright Browser."""
+        return self.browser_adapter.browser
+
+    @property
+    def context(self) -> Optional[BrowserContext]:
+        """Active Playwright BrowserContext."""
+        return self.browser_adapter.context
+
+    @property
+    def page(self) -> Optional[Page]:
+        """Active Playwright Page."""
+        return self.browser_adapter.page
+
+    @page.setter
+    def page(self, p: Page) -> None:
+        self.browser_adapter.page = p
+        if self._services_ready():
+            self._bind_services(p)
+
+    def _services_ready(self) -> bool:
+        return self.auth_service is not None
+
+    def _bind_services(self, active_page: Page) -> None:
+        """Binds active page to services and adapters."""
+        ctx = self.context
+        assert ctx is not None
+        self.auth_service = AuthNavigationService(active_page, ctx)
+        self.inspection_service = InspectionService(active_page)
+        self.field_service = FieldInteractionService(active_page)
+        self.filler_service = FormFillerService(active_page, self.field_service)
+        self.upload_adapter = UploadAdapter(active_page)
+        self.validator = FormValidator(active_page)
+        self.reactive_service = ReactiveInteractionService(
+            page=active_page,
+            context=ctx,
+            inspection_service=self.inspection_service,
+            form_validator=self.validator,
+            on_active_page_change=self._on_active_page_change,
+        )
+
+    def _on_active_page_change(self, new_page: Page) -> None:
+        self.browser_adapter.page = new_page
+        self._bind_services(new_page)
+
+    def start(self, cookies_path: Optional[str] = None) -> None:
+        """Starts Playwright browser instance, initializes context, and prepares services."""
+        self.browser_adapter.start(cookies_path=cookies_path)
+        assert self.page is not None
+        self._bind_services(self.page)
+
+    def stop(self) -> None:
+        """Closes browser context and Playwright instance."""
+        self.browser_adapter.stop()
+
+    def navigate(self, url: str, wait_timeout: Optional[int] = None) -> None:
+        """Navigates to URL and waits for DOM content loaded."""
+        self.browser_adapter.navigate(url=url, wait_timeout=wait_timeout)
+
+    def handle_login_if_needed(self, email: str, password: str, cookies_save_path: Optional[str] = None) -> bool:
+        """Detects if page is a login/sign-in page and completes authentication automatically."""
+        assert self.auth_service is not None, "Engine not started."
+        return self.auth_service.handle_login_if_needed(email, password, cookies_save_path)
+
+    def bypass_job_landing_portal(self) -> bool:
+        """Generic helper: Clicks Apply Now on portal landing pages to enter ATS application."""
+        assert self.auth_service is not None, "Engine not started."
+        res = self.auth_service.bypass_job_landing_portal()
+        if self.page != self.auth_service.page:
+            self.page = self.auth_service.page
+        return res
+
+    def wait_for_form_ready(self, timeout_ms: Optional[int] = None) -> None:
+        """Waits for form elements to be rendered in DOM, dismissing 'Loading...' state."""
+        assert self.auth_service is not None, "Engine not started."
+        effective_timeout = timeout_ms if timeout_ms is not None else self.settings.form_ready_timeout_ms
+        self.auth_service.wait_for_form_ready(timeout_ms=effective_timeout)
+
+    def expand_all_sections(self) -> int:
+        """Safely expands any collapsed form sections without toggling already-open ones."""
+        assert self.auth_service is not None, "Engine not started."
+        return self.auth_service.expand_all_sections(wait_ms=self.settings.expand_section_wait_ms)
+
+    def fill_form_sequential(self, field_mappings: List[Tuple[str, Any]]) -> Dict[str, List[Tuple[str, Any]]]:
+        """Universal sequential form filler (section-by-section + global fallback)."""
+        assert self.filler_service is not None, "Engine not started."
+        return self.filler_service.fill_form_sequential(field_mappings)
+
+    def inspect(self, unpack_options: bool = True) -> Dict[str, Any]:
+        """Deep inspection of the active page form components and schema."""
+        assert self.auth_service is not None and self.inspection_service is not None, "Engine not started."
+        self.wait_for_form_ready()
+        self.expand_all_sections()
+        return self.inspection_service.inspect(unpack_options=unpack_options)
+
+    def format_inspection_summary(self, data: Dict[str, Any]) -> str:
+        """Returns a concise, human-readable summary of the inspected form."""
+        return InspectionFormatterService.format_summary(data)
+
+    def set_field(self, target_identifier: str, value: Any) -> bool:
+        """Sets form field value across text, picklist, radio, checkbox widgets."""
+        assert self.field_service is not None, "Engine not started."
+        return self.field_service.set_field(target_identifier, value)
+
+    def _playwright_click_fallback(self, target: str, value: Any) -> bool:
+        """Fallback setter using Playwright native locators (click / fill)."""
+        assert self.field_service is not None, "Engine not started."
+        return self.field_service._playwright_click_fallback(target, value)
+
+    def verify_field_value(self, target: str, expected: Any) -> bool:
+        """Reads back current DOM value of field and checks if it matches expected."""
+        assert self.field_service is not None, "Engine not started."
+        return self.field_service.verify_field_value(target, expected)
+
+    def upload_document(self, file_path: str, document_type_keyword: str = "") -> bool:
+        """Universally attaches a document to file inputs or custom ATS chooser triggers."""
+        assert self.upload_adapter is not None, "Engine not started."
+        return self.upload_adapter.upload_document(file_path, document_type_keyword)
+
+    def click_button(self, button_text_or_purpose: str) -> Dict[str, Any]:
+        """Intelligently clicks a button/link and monitors reactive environmental mutations."""
+        assert self.reactive_service is not None, "Engine not started."
+        outcome = self.reactive_service.click_button(button_text_or_purpose)
+        if outcome.get("new_window") and self.page != self.reactive_service.page:
+            self.page = self.reactive_service.page
+        return outcome
+
+    def get_validation_errors(self) -> List[str]:
+        """Collects any error alerts, validation messages, or warning texts currently shown."""
+        assert self.validator is not None, "Engine not started."
+        return self.validator.get_validation_errors()
+
+    def capture_screenshot(self, output_path: str, full_page: bool = False) -> None:
+        """Saves a screenshot of the current page."""
+        self.browser_adapter.capture_screenshot(output_path=output_path, full_page=full_page)

exceptions.py

@@ -0,0 +1,23 @@
+"""Backwards compatibility shim for exceptions."""
+
+from core.exceptions import (
+    UniversalApplierError,
+    ConfigurationError,
+    ValidationError,
+    AutomationError,
+    ElementNotFoundError,
+    InteractionTimeoutError,
+    DocumentUploadError,
+    AuthenticationError,
+)
+
+__all__ = [
+    "UniversalApplierError",
+    "ConfigurationError",
+    "ValidationError",
+    "AutomationError",
+    "ElementNotFoundError",
+    "InteractionTimeoutError",
+    "DocumentUploadError",
+    "AuthenticationError",
+]