qoder-autopilot 0.2.1__py3-none-any.whl

qoder_autopilot/captcha/opencv_detect.py

@@ -0,0 +1,184 @@
+"""
+Qoder Autopilot — OpenCV Captcha Gap Detection
+================================================
+4-method computer vision approach to find the gap position in
+Aliyun slide CAPTCHAs without requiring an AI API:
+
+1. Column brightness drop analysis
+2. Edge density analysis
+3. Masked template matching (TM_CCOEFF_NORMED)
+4. Masked SQDIFF matching (TM_SQDIFF_NORMED)
+
+Results are combined via voting to find the most likely gap position.
+"""
+
+import base64
+import urllib.request
+
+import numpy as np
+
+from ..logger import log
+
+
+async def detect_gap_position(page) -> float | None:
+    """Use OpenCV to detect the puzzle gap position in the captcha image.
+
+    Args:
+        page: Playwright/Camoufox page object with the captcha loaded.
+
+    Returns:
+        The X offset scaled to track width, or None on failure.
+    """
+    try:
+        import cv2
+
+        # Get image URLs from the page
+        img_urls = await page.evaluate("""() => {
+            const bg = document.querySelector('#aliyunCaptcha-img');
+            const pz = document.querySelector('#aliyunCaptcha-puzzle');
+            return {
+                bgSrc: bg?.src || '',
+                pzSrc: pz?.src || '',
+                bgW: bg?.getBoundingClientRect()?.width || 300,
+                bgH: bg?.getBoundingClientRect()?.height || 200,
+                pzW: pz?.getBoundingClientRect()?.width || 52,
+            };
+        }""")
+
+        if not img_urls.get("bgSrc") or not img_urls.get("pzSrc"):
+            return None
+
+        # Parse base64 data URIs
+        def parse_data_uri(uri: str) -> bytes:
+            if uri.startswith("data:"):
+                _, b64data = uri.split(",", 1)
+                return base64.b64decode(b64data)
+            else:
+                return urllib.request.urlopen(uri).read()
+
+        bg_data = parse_data_uri(img_urls["bgSrc"])
+        pz_data = parse_data_uri(img_urls["pzSrc"])
+
+        bg_img = cv2.imdecode(np.frombuffer(bg_data, np.uint8), cv2.IMREAD_COLOR)
+        pz_img = cv2.imdecode(np.frombuffer(pz_data, np.uint8), cv2.IMREAD_UNCHANGED)
+
+        if bg_img is None or pz_img is None:
+            return None
+
+        # Convert puzzle to grayscale + edge detection
+        if pz_img.shape[2] == 4:
+            pz_gray = pz_img[:, :, 3]  # Use alpha channel
+            _, pz_mask = cv2.threshold(pz_gray, 127, 255, cv2.THRESH_BINARY)
+            pz_bgr = pz_img[:, :, :3]
+        else:
+            pz_gray = cv2.cvtColor(pz_img, cv2.COLOR_BGR2GRAY)
+            pz_mask = None
+            pz_bgr = pz_img
+
+        bg_gray = cv2.cvtColor(bg_img, cv2.COLOR_BGR2GRAY)
+        bg_h, bg_w = bg_gray.shape
+        pz_h, pz_w = pz_bgr.shape[:2]
+
+        # ─── Method 1: Column brightness drop analysis ───
+        col_brightness = np.mean(bg_gray, axis=0)
+        kernel_size = max(3, pz_w // 4)
+        if kernel_size % 2 == 0:
+            kernel_size += 1
+        smoothed = cv2.GaussianBlur(col_brightness.reshape(1, -1), (kernel_size, 1), 0).flatten()
+
+        window = pz_w
+        min_brightness = float("inf")
+        min_col = bg_w // 2
+
+        start_col = int(bg_w * 0.15)  # Skip first 15% (puzzle piece start area)
+        for col in range(start_col, bg_w - pz_w):
+            left = max(0, col - window // 2)
+            right = min(bg_w, col + window // 2)
+            region_brightness = np.mean(smoothed[left:right])
+
+            other_left = smoothed[:left] if left > 0 else np.array([])
+            other_right = smoothed[right:] if right < bg_w else np.array([])
+            other = np.concatenate([other_left, other_right])
+            if len(other) > 0:
+                other_brightness = np.mean(other)
+                drop = other_brightness - region_brightness
+                if drop > (np.mean(smoothed) - min_brightness):
+                    min_brightness = region_brightness
+                    min_col = col
+
+        gap_x_method1 = min_col
+
+        # ─── Method 2: Edge density analysis ───
+        bg_edges = cv2.Canny(bg_gray, 50, 150)
+        col_edge_density = np.sum(bg_edges > 0, axis=0).astype(float)
+        edge_smoothed = cv2.GaussianBlur(
+            col_edge_density.reshape(1, -1), (kernel_size, 1), 0
+        ).flatten()
+
+        max_edge = 0
+        max_edge_col = bg_w // 2
+        for col in range(start_col, bg_w - pz_w):
+            left = max(0, col - window // 2)
+            right = min(bg_w, col + window // 2)
+            edge_sum = np.sum(edge_smoothed[left:right])
+            if edge_sum > max_edge:
+                max_edge = edge_sum
+                max_edge_col = col
+
+        gap_x_method2 = max_edge_col
+
+        # ─── Method 3: Masked template matching ───
+        pz_gray_tmpl = cv2.cvtColor(pz_bgr, cv2.COLOR_BGR2GRAY)
+        tm_mask = (
+            pz_mask if pz_mask is not None else np.ones(pz_bgr.shape[:2], dtype=np.uint8) * 255
+        )
+
+        result_masked = cv2.matchTemplate(bg_gray, pz_gray_tmpl, cv2.TM_CCOEFF_NORMED, mask=tm_mask)
+        _, max_val_masked, _, max_loc_masked = cv2.minMaxLoc(result_masked)
+        gap_x_method3 = max_loc_masked[0]
+
+        # ─── Method 4: SQDIFF masked ───
+        result_sqdiff = cv2.matchTemplate(bg_gray, pz_gray_tmpl, cv2.TM_SQDIFF_NORMED, mask=tm_mask)
+        _, _, min_loc_sq, _ = cv2.minMaxLoc(result_sqdiff)
+        gap_x_method4 = min_loc_sq[0]
+
+        # ─── Combine via voting ───
+        methods = {
+            "brightness": gap_x_method1,
+            "edge": gap_x_method2,
+            "maskedTM": gap_x_method3,
+            "sqdiffTM": gap_x_method4,
+        }
+
+        best_votes = {}
+        for name, x in methods.items():
+            votes = sum(1 for other_x in methods.values() if abs(x - other_x) < 25)
+            best_votes[name] = votes
+
+        winner = max(best_votes.keys(), key=lambda k: k and best_votes[k])
+        gap_x = methods[winner]
+
+        agreeing = [x for x in methods.values() if abs(x - gap_x) < 25]
+        if len(agreeing) >= 2:
+            gap_x = int(np.mean(agreeing))
+            confidence = min(0.9, len(agreeing) * 0.25)
+        else:
+            gap_x = gap_x_method3
+            confidence = max_val_masked
+
+        method_str = f"winner={winner}(votes={best_votes[winner]}) | {methods}"
+        log(f"   🔍 Gap: x={gap_x}px | {method_str}")
+
+        bg_img_width = bg_img.shape[1]
+        track_width = img_urls["bgW"]
+        scaled_x = (gap_x / bg_img_width) * track_width
+
+        log(
+            f"   🔍 Gap: x={gap_x}px (img {bg_img_width}px) "
+            f"→ track {scaled_x:.0f}/{track_width:.0f}px, conf={confidence:.2f}"
+        )
+        return scaled_x
+
+    except Exception as e:
+        log(f"   ⚠️  Gap detection failed: {e}")
+        return None

qoder_autopilot/captcha/slider.py

@@ -0,0 +1,86 @@
+"""
+Qoder Autopilot — Human-like Slider Movement
+===============================================
+Simulates realistic human mouse movement for sliding captcha puzzle pieces.
+Includes variable speed, slight vertical wobble, and overshoot correction.
+"""
+
+import asyncio
+import random
+
+from ..logger import log
+
+
+async def slide_puzzle(
+    page,
+    target_x: float,
+    track_width: float,
+) -> None:
+    """Slide the puzzle piece to the target position with human-like movement.
+
+    Args:
+        page: Playwright/Camoufox page object.
+        target_x: Target X position on the track.
+        track_width: Total width of the sliding track.
+    """
+    # Find the slider element
+    slider = await page.evaluate("""() => {
+        const sels = ['#aliyunCaptcha-sliding-slider',
+                      '#aliyunCaptcha-sliding .aliyunCaptcha-sliding-btn',
+                      '.nc_iconfont.btn_slide', '.slide-verify-slider',
+                      '[class*="slider-move"]', '[id*="sliding-slider"]',
+                      '[class*="slider"] [class*="btn"]', '[class*="drag"]',
+                      '[class*="slide"] [class*="handle"]'];
+        for (const s of sels) {
+            const el = document.querySelector(s);
+            if (el) { const r = el.getBoundingClientRect();
+                return {x: r.x, y: r.y, w: r.width, h: r.height}; }
+        }
+        return null;
+    }""")
+
+    if not slider:
+        log("   ⚠️  No slider found")
+        return
+
+    sx = slider["x"] + slider["w"] / 2
+    sy = slider["y"] + slider["h"] / 2
+
+    log(f"   🖱️  Sliding {target_x:.0f}/{track_width:.0f}px")
+
+    # Move to slider start position
+    await page.mouse.move(sx, sy)
+    await asyncio.sleep(random.uniform(0.3, 0.5))
+    await page.mouse.down()
+    await asyncio.sleep(random.uniform(0.1, 0.2))
+
+    # ─── Main slide with variable speed ───
+    cx = 0
+    steps = random.randint(25, 45)
+
+    for i in range(steps):
+        p = (i + 1) / steps
+        # Fast in the middle, slow at start and end
+        if p < 0.7:
+            speed = random.uniform(3, 8)
+        elif p < 0.9:
+            speed = random.uniform(1, 3)
+        else:
+            speed = random.uniform(0.3, 1)
+        dx = min(speed, target_x - cx)
+        cx += dx
+        # Add slight vertical wobble
+        await page.mouse.move(
+            sx + cx,
+            sy + random.uniform(-1.5, 1.5),
+            steps=1,
+        )
+        await asyncio.sleep(random.uniform(0.01, 0.04))
+
+    # ─── Overshoot + correct (mimics human behavior) ───
+    await page.mouse.move(sx + cx + random.uniform(3, 10), sy, steps=2)
+    await asyncio.sleep(random.uniform(0.1, 0.3))
+    await page.mouse.move(sx + target_x, sy, steps=3)
+    await asyncio.sleep(random.uniform(0.1, 0.2))
+    await page.mouse.up()
+    await asyncio.sleep(2)

qoder_autopilot/captcha/solver.py

@@ -0,0 +1,165 @@
+"""
+Qoder Autopilot — Captcha Solver Orchestrator
+===============================================
+Coordinates captcha detection, solving strategies, and retry logic.
+
+Strategies (in priority order):
+    1. AI Vision (if API key configured) — most accurate
+    2. OpenCV (always available) — 4-method fallback
+    3. Manual (if requested) — user solves in browser
+
+Usage:
+    solver = CaptchaSolver(manual=False)
+    ok = await solver.solve(page)
+"""
+
+import asyncio
+import random
+import time
+
+from .. import config
+from ..logger import log, log_err, log_ok
+from .ai_vision import gemini_detect_gap
+from .manual import handle_captcha_manual
+from .opencv_detect import detect_gap_position
+from .slider import slide_puzzle
+
+# CSS selectors for detecting captcha presence
+_CAPTCHA_DETECT_SELECTORS = """
+    const sels = ['#aliyunCaptcha-sliding', '.aliyunCaptcha', '#nc_1_wrapper',
+                  '.nc-container', 'iframe[src*="captcha"]', '.slide-verify',
+                  '[class*="captcha"]', '[id*="captcha"]'];
+    for (const s of sels) {
+        const el = document.querySelector(s);
+        if (el && el.offsetParent !== null) return true;
+    }
+    return document.querySelectorAll(
+        'iframe[src*="captcha"], iframe[src*="aliyun"]'
+    ).length > 0;
+"""
+
+_CAPTCHA_STILL_SELECTORS = """
+    const sels = ['#aliyunCaptcha-sliding-slider', '#aliyunCaptcha-window-float',
+                  '#aliyunCaptcha-sliding', '.aliyunCaptcha', '#captcha-element',
+                  '#nc_1_wrapper', '.nc-container'];
+    for (const s of sels) {
+        const el = document.querySelector(s);
+        if (el && el.offsetParent !== null) return true;
+    }
+    return false;
+"""
+
+
+class CaptchaSolver:
+    """Orchestrates captcha solving with multiple strategies and retries."""
+
+    def __init__(
+        self,
+        manual: bool = False,
+        use_ai: bool = True,
+        max_attempts: int | None = None,
+    ):
+        self.manual = manual
+        self.use_ai = use_ai and bool(config.AI_API_KEY)
+        self.max_attempts = max_attempts or config.MAX_CAPTCHA_ATTEMPTS
+
+    async def solve(self, page) -> bool:
+        """Detect and solve the captcha on the current page.
+
+        Args:
+            page: Playwright/Camoufox page object.
+
+        Returns:
+            True if captcha was solved (or not present), False on failure.
+        """
+        if self.manual:
+            return await handle_captcha_manual(page)
+
+        await asyncio.sleep(2)
+
+        # Check if captcha is present
+        has_captcha = await page.evaluate(f"() => {{ {_CAPTCHA_DETECT_SELECTORS} }}")
+        if not has_captcha:
+            log("   ℹ️  No captcha detected")
+            return True
+
+        log("   🔒 Captcha detected! Solving...")
+        config.SCREENSHOTS_DIR.mkdir(exist_ok=True)
+        await page.screenshot(path=str(config.SCREENSHOTS_DIR / f"captcha_{int(time.time())}.png"))
+
+        tried_positions: list[float] = []
+
+        for attempt in range(self.max_attempts):
+            # Wait for captcha image to load
+            img_ready = {"ready": False, "src": ""}
+            for _ in range(30):  # up to 6 seconds
+                img_ready = await page.evaluate("""() => {
+                    const bg = document.querySelector('#aliyunCaptcha-img');
+                    if (!bg || !bg.src) return {ready: false, src: ''};
+                    const isLoaded = bg.src.startsWith('data:')
+                        ? bg.complete
+                        : (bg.complete && bg.naturalWidth > 0);
+                    return {ready: isLoaded, src: bg.src.substring(0, 80)};
+                }""")
+                if img_ready.get("ready"):
+                    break
+                await asyncio.sleep(0.2)
+            else:
+                log(f"   ⚠️ Image not loaded after 6s, src='{img_ready.get('src', 'N/A')}'")
+            await asyncio.sleep(0.5)
+
+            # Get track width
+            track_w = await page.evaluate("""() => {
+                const sels = ['#aliyunCaptcha-sliding-body',
+                              '.aliyunCaptcha-sliding-track',
+                              '.nc-container .nc_scale', '.slide-verify-track',
+                              '[class*="track"]'];
+                for (const s of sels) {
+                    const el = document.querySelector(s);
+                    if (el && el.getBoundingClientRect().width > 50)
+                        return el.getBoundingClientRect().width;
+                }
+                return 300;
+            }""")
+            if track_w < 50:
+                track_w = 300
+
+            # ─── Detect gap position ───
+            detected_x = None
+            if self.use_ai:
+                detected_x = await gemini_detect_gap(page)
+            if detected_x is None:
+                detected_x = await detect_gap_position(page)
+
+            base = detected_x if detected_x is not None else track_w * 0.6
+
+            # ─── Calculate target with retry offset strategy ───
+            if attempt == 0:
+                offset = random.uniform(-2, 2)
+            elif attempt <= 2:
+                offset = random.uniform(-5, 5)
+            else:
+                offset = random.choice([-15, -10, -5, 5, 10, 15]) + random.uniform(-3, 3)
+            target = max(10, min(track_w - 10, base + offset))
+            tried_positions.append(target)
+
+            if detected_x is not None:
+                log(f"   🎯 Target: {target:.0f}/{track_w:.0f}px")
+            else:
+                log(f"   🎯 Fallback target: {target:.0f}/{track_w:.0f}px")
+
+            log(f"   🖱️  Attempt {attempt + 1}: sliding {target:.0f}/{track_w:.0f}px")
+
+            # ─── Execute the slide ───
+            await slide_puzzle(page, target, track_w)
+
+            # ─── Check if captcha is gone ───
+            still = await page.evaluate(f"() => {{ {_CAPTCHA_STILL_SELECTORS} }}")
+            if not still:
+                log_ok("Captcha solved!")
+                return True
+
+            log(f"   ⚠️  Captcha still present (attempt {attempt + 1}/{self.max_attempts})")
+
+        log_err(f"Failed to solve captcha after {self.max_attempts} attempts")
+        return False