onecrawler 0.1.0__py3-none-any.whl

onecrawler/__init__.py

@@ -0,0 +1,7 @@
+from .crawler.link.classifier import LinkClassifierPipeline
+from .crawler.link.engine import LinkExtractionEngine
+from .crawler.scraper.engine import ScraperEngine
+from .map.sitemap import SiteMap, SitemapStats, UniversalSiteMap
+from .settings import *
+
+__version__ = "0.1.0"

onecrawler/browser.py

@@ -0,0 +1,93 @@
+from contextlib import suppress
+
+from playwright.async_api import async_playwright
+
+from .settings.browser import BrowserSettings
+
+
+class GoogleChrome:
+    def __init__(self, settings: BrowserSettings):
+        self.settings = settings
+        self.playwright = None
+        self.browser = None
+        self.context = None
+        self._started = False
+        self._closed = False
+
+    async def start(self):
+        if self._started:
+            return
+
+        self.playwright = await async_playwright().start()
+
+        launch = self.settings.launch
+        context = self.settings.context
+
+        self.browser = await self.playwright.chromium.launch(
+            headless=launch.headless,
+            slow_mo=launch.slow_mo,
+            args=launch.args,
+            executable_path=launch.executable_path,
+            channel=launch.channel,
+            env=launch.env,
+        )
+
+        self.context = await self.browser.new_context(
+            viewport=context.viewport,
+            screen=context.screen,
+            no_viewport=context.no_viewport,
+            locale=context.locale,
+            timezone_id=context.timezone_id,
+            user_agent=context.user_agent,
+            java_script_enabled=context.java_script_enabled,
+            bypass_csp=context.bypass_csp,
+            ignore_https_errors=context.ignore_https_errors,
+            extra_http_headers=context.extra_http_headers,
+            offline=context.offline,
+            geolocation=context.geolocation,
+            permissions=context.permissions,
+            storage_state=context.storage_state,
+            base_url=context.base_url,
+            proxy=self.settings.proxy.as_playwright() if self.settings.proxy else None,
+        )
+
+        self._started = True
+        self._closed = False
+
+    async def new_page(self):
+        if not self._started:
+            await self.start()
+
+        page = await self.context.new_page()
+
+        runtime = self.settings.runtime
+        page.set_default_timeout(runtime.action_timeout)
+        page.set_default_navigation_timeout(runtime.navigation_timeout)
+
+        return page
+
+    async def close(self):
+        if self._closed:
+            return
+
+        self._closed = True
+
+        # Close context safely
+        if self.context:
+            with suppress(Exception):
+                await self.context.close()
+            self.context = None
+
+        # Close browser (YOU WERE MISSING THIS)
+        if self.browser:
+            with suppress(Exception):
+                await self.browser.close()
+            self.browser = None
+
+        # Stop playwright
+        if self.playwright:
+            with suppress(Exception):
+                await self.playwright.stop()
+            self.playwright = None
+
+        self._started = False

onecrawler/crawler/base.py

@@ -0,0 +1,54 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Optional, Type
+
+
+class BaseEngine(ABC):
+    def __init__(self):
+        self._closed: bool = True
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    # ===== Context Manager =====
+    async def __aenter__(self):
+        self._closed = False
+        await self.start()
+        self.logger.debug("Engine started")
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        tb,
+    ):
+        try:
+            await self.close()
+        finally:
+            self._closed = True
+            self.logger.debug("Engine closed")
+
+    # ===== Lifecycle Hooks =====
+    async def start(self):
+        """Override to initialize resources."""
+        pass
+
+    async def close(self):
+        """Override to cleanup resources."""
+        pass
+
+    # ===== REQUIRED API =====
+    @abstractmethod
+    async def run(self, *args, **kwargs):
+        """Main execution method for the engine."""
+        raise NotImplementedError
+
+    # ===== Safety =====
+    def _ensure_open(self):
+        if self._closed:
+            raise RuntimeError(
+                f"{self.__class__.__name__} is closed. Use 'async with'."
+            )
+
+    @property
+    def is_closed(self) -> bool:
+        return self._closed

onecrawler/crawler/link/classifier.py

@@ -0,0 +1,116 @@
+import asyncio
+import warnings
+from functools import lru_cache
+from typing import List
+from urllib.parse import unquote
+
+CLASSIFIER_AVAILABLE = True
+try:
+    import torch
+    from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
+except ImportError:
+    CLASSIFIER_AVAILABLE = False
+
+
+if CLASSIFIER_AVAILABLE:
+    repo_id = "SayedShaun/distilbert-link-type-classifier"
+
+    torch.set_grad_enabled(False)
+
+    model = AutoModelForSequenceClassification.from_pretrained(repo_id)
+    tokenizer = AutoTokenizer.from_pretrained(repo_id)
+
+    def get_classifier(device: str = "cpu"):
+        device_id = 0 if device == "cuda" else -1
+        return pipeline(
+            "text-classification",
+            model=model,
+            tokenizer=tokenizer,
+            device=device_id,
+        )
+
+    clf = get_classifier("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def cheap_filter(url: str) -> bool:
+    if not url:
+        return False
+
+    bad_patterns = (
+        "javascript:",
+        "mailto:",
+        "tel:",
+        "#",
+    )
+
+    return not any(p in url for p in bad_patterns)
+
+
+@lru_cache(maxsize=10000)
+def _cached_single_prediction(url: str) -> str:
+    result = clf(url, truncation=True, max_length=128)[0]
+    return result["label"]
+
+
+class LinkClassifierPipeline:
+    def __init__(
+        self,
+        confidence_threshold: float = 0.8,
+    ):
+        self.threshold = confidence_threshold
+
+        if not CLASSIFIER_AVAILABLE:
+            warnings.warn(
+                "Classifier enabled but transformers/torch not installed. Disabling."
+            )
+            self.available = False
+        else:
+            self.available = True
+
+    async def classify_batch(self, urls: List[str]) -> List[bool]:
+        if not self.available:
+            return [True] * len(urls)
+
+        filtered_urls = []
+        index_map = []
+
+        for i, url in enumerate(urls):
+            if cheap_filter(url):
+                filtered_urls.append(unquote(url))
+                index_map.append(i)
+
+        results = [False] * len(urls)
+
+        if not filtered_urls:
+            return results
+
+        try:
+            predictions = await asyncio.to_thread(
+                lambda: clf(filtered_urls, truncation=True, max_length=128)
+            )
+        except Exception:
+            return [True] * len(urls)
+
+        for idx, pred in zip(index_map, predictions):
+            label = pred["label"]
+            score = pred["score"]
+
+            if label == "content" and score >= self.threshold:
+                results[idx] = True
+            else:
+                results[idx] = False
+
+        return results
+
+    async def is_valid(self, url: str) -> bool:
+        if not self.available:
+            return True
+
+        if not cheap_filter(url):
+            return False
+
+        try:
+            label = await asyncio.to_thread(_cached_single_prediction, unquote(url))
+            return label != "section"
+        except Exception:
+            return True

onecrawler/crawler/link/deep.py

@@ -0,0 +1,235 @@
+import asyncio
+import logging
+from collections import deque
+from typing import Optional
+from urllib.parse import urldefrag
+
+from ...settings.simulation import HumanBehaviorSettings
+from .helper import human_delay, human_mouse_move, human_scroll, wildcard_link_match
+
+logger = logging.getLogger(__name__)
+
+
+class BFScheduler:
+    def __init__(self, base_url: str, max_queue_size: int = 5000):
+        self.queue = deque([base_url])
+        self.priority = deque()
+
+        self.visited = set()
+        self.in_queue = {base_url}
+
+        self.max_queue_size = max_queue_size
+        self.lock = asyncio.Lock()
+
+    async def has_next(self) -> bool:
+        async with self.lock:
+            return bool(self.queue or self.priority)
+
+    async def next(self) -> str | None:
+        async with self.lock:
+            if self.priority:
+                return self.priority.popleft()
+            if self.queue:
+                return self.queue.popleft()
+            return None
+
+    async def add(self, url: str, priority: bool = False):
+        async with self.lock:
+            if url in self.visited or url in self.in_queue:
+                return
+
+            if len(self.in_queue) >= self.max_queue_size:
+                return
+
+            self.in_queue.add(url)
+
+            if priority:
+                self.priority.append(url)
+            else:
+                self.queue.append(url)
+
+    def mark_visited(self, url: str):
+        self.visited.add(url)
+        self.in_queue.discard(url)
+
+
+class BrowserPool:
+    def __init__(self, browser, size: int):
+        self.browser = browser
+        self.size = size
+        self.pages = asyncio.Queue(maxsize=size)
+        self._closed = False
+
+    async def init(self):
+        # NOTE: browser.start() must already be called before BrowserPool.init()
+        # We only create pages here — do NOT call self.browser.start() again.
+        for _ in range(self.size):
+            page = await self.browser.new_page()
+            await self.pages.put(page)
+
+    async def acquire(self):
+        return await self.pages.get()
+
+    async def release(self, page):
+        if not self._closed:
+            await self.pages.put(page)
+
+    async def close(self):
+        self._closed = True
+        while not self.pages.empty():
+            page = await self.pages.get()
+            await page.close()
+        # NOTE: do NOT close the browser here — the engine owns it
+
+
+class LinkSpider:
+    def __init__(self, base_prefix: str):
+        self.base_prefix = base_prefix
+
+    async def parse(self, page):
+        raw = await page.eval_on_selector_all(
+            "a", "els => els.map(e => e.href).filter(Boolean)"
+        )
+        return [
+            urldefrag(link).url
+            for link in raw
+            if isinstance(link, str) and link.startswith(self.base_prefix)
+        ]
+
+
+class BFSRuntime:
+    def __init__(
+        self,
+        scheduler,
+        pool,
+        spider,
+        base_prefix: str,
+        max_links: int,
+        human_behavior_settings: HumanBehaviorSettings,
+        include_pattern: Optional[list] = None,
+        enable_human_behaviors: bool = False,
+        concurrency: int = 5,
+    ):
+        self.scheduler = scheduler
+        self.pool = pool
+        self.spider = spider
+
+        self.base_prefix = base_prefix
+        self.max_links = max_links
+        self.include_pattern = include_pattern
+        self.enable_human_behaviors = enable_human_behaviors
+        self.human_behavior_settings = human_behavior_settings
+        self.concurrency = concurrency
+
+        self.stop_event = asyncio.Event()
+
+        self.results = []
+        self.results_set = set()
+        self.lock = asyncio.Lock()
+
+        # Track how many workers are actively processing a URL
+        self._active_workers = 0
+        self._active_lock = asyncio.Lock()
+
+    async def worker(self):
+        while not self.stop_event.is_set():
+            url = await self.scheduler.next()
+
+            if url is None:
+                # Queue is empty — check if any other worker is still active.
+                # If not, all work is done and we can exit.
+                async with self._active_lock:
+                    if self._active_workers == 0:
+                        self.stop_event.set()
+                        return
+
+                # Another worker is still processing and may enqueue more URLs.
+                # Wait a bit and retry instead of busy-spinning.
+                await asyncio.sleep(0.2)
+                continue
+
+            if url in self.scheduler.visited:
+                continue
+
+            async with self._active_lock:
+                self._active_workers += 1
+
+            page = await self.pool.acquire()
+
+            try:
+                self.scheduler.mark_visited(url)
+
+                try:
+                    await page.goto(url, wait_until="domcontentloaded")
+                except Exception as e:
+                    logger.warning("Failed to load %s: %s", url, e)
+                    return
+
+                if self.enable_human_behaviors:
+                    await human_delay(
+                        self.human_behavior_settings.min_delay,
+                        self.human_behavior_settings.max_delay,
+                    )
+                    await human_scroll(
+                        page, max_scrolls=self.human_behavior_settings.max_scrolls
+                    )
+
+                links = await self.spider.parse(page)
+
+                if self.enable_human_behaviors:
+                    await human_mouse_move(
+                        page,
+                        min_mouse_moves=self.human_behavior_settings.min_mouse_moves,
+                        max_mouse_moves=self.human_behavior_settings.max_mouse_moves,
+                        mouse_width=self.human_behavior_settings.mouse_width,
+                        mouse_height=self.human_behavior_settings.mouse_height,
+                        min_mouse_steps=self.human_behavior_settings.min_mouse_steps,
+                        max_mouse_steps=self.human_behavior_settings.max_mouse_steps,
+                        min_mouse_sleep=self.human_behavior_settings.min_mouse_sleep,
+                        max_mouse_sleep=self.human_behavior_settings.max_mouse_sleep,
+                    )
+
+                for link in links:
+                    if self.stop_event.is_set():
+                        break
+
+                    if not link.startswith(self.base_prefix):
+                        continue
+
+                    if self.include_pattern:
+                        if not wildcard_link_match(
+                            link,
+                            self.base_prefix,
+                            self.include_pattern,
+                        ):
+                            continue
+
+                    await self.scheduler.add(link)
+
+                    async with self.lock:
+                        if link in self.results_set:
+                            continue
+
+                        self.results_set.add(link)
+                        self.results.append(link)
+
+                        logger.info(
+                            "Discovered %s/%s links; link=%s",
+                            len(self.results),
+                            self.max_links,
+                            link,
+                        )
+
+                        if len(self.results) >= self.max_links:
+                            self.stop_event.set()
+                            return
+
+            finally:
+                await self.pool.release(page)
+                async with self._active_lock:
+                    self._active_workers -= 1
+
+    async def run(self):
+        tasks = [asyncio.create_task(self.worker()) for _ in range(self.concurrency)]
+        await asyncio.gather(*tasks, return_exceptions=True)
+        return self.results

onecrawler/crawler/link/engine.py

@@ -0,0 +1,80 @@
+from urllib.parse import urlparse
+
+from ...browser import GoogleChrome
+from ..base import BaseEngine
+from .deep import BFScheduler, BFSRuntime, BrowserPool, LinkSpider
+from .shallow import extract_url_from_current_page
+
+
+class LinkExtractionEngine(BaseEngine):
+    def __init__(self, settings):
+        super().__init__()
+
+        self.settings = settings
+
+        # future-ready placeholders
+        self.session = None
+
+        self.logger.info("LinkExtractionEngine initialized")
+
+    async def start(self):
+        self.browser = GoogleChrome(self.settings.browser_settings)
+        await self.browser.start()
+
+    async def close(self):
+        if hasattr(self, "browser") and self.browser:
+            await self.browser.close()
+
+    async def run(self, url: str) -> dict:
+        self._ensure_open()
+
+        strategy = self.settings.link_extraction_strategy
+        self.logger.info(f"Running link extraction on {url} with strategy: {strategy}")
+
+        if strategy == "shallow":
+            return await self._run_shallow(url)
+
+        if strategy == "deep":
+            return await self._run_deep(url)
+
+        raise ValueError(f"Unknown strategy: {strategy}")
+
+    async def _run_shallow(self, url: str) -> dict:
+        self._ensure_open()
+
+        return await extract_url_from_current_page(
+            url=url,
+            browser=self.browser,
+            include_link_patterns=self.settings.include_link_patterns,
+            link_classification=self.settings.link_classification,
+            max_links=self.settings.link_extraction_limit,
+        )
+
+    async def _run_deep(self, url: str) -> dict:
+        self._ensure_open()
+
+        parsed = urlparse(url)
+        base_prefix = f"{parsed.scheme}://{parsed.netloc}"
+
+        scheduler = BFScheduler(url)
+        spider = LinkSpider(base_prefix)
+        pool = BrowserPool(self.browser, self.settings.concurrency)
+
+        await pool.init()
+
+        runtime = BFSRuntime(
+            scheduler=scheduler,
+            pool=pool,
+            spider=spider,
+            base_prefix=base_prefix,
+            max_links=self.settings.link_extraction_limit,
+            include_pattern=self.settings.include_link_patterns,
+            enable_human_behaviors=self.settings.enable_human_behaviors,
+            human_behavior_settings=self.settings.human_behavior_settings,
+            concurrency=self.settings.concurrency,
+        )
+
+        try:
+            return await runtime.run()
+        finally:
+            await pool.close()