web-queue2 0.1.0__tar.gz

web_queue2-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 AllenChou
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

web_queue2-0.1.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: web-queue2
+Version: 0.1.0
+Summary: Get web content from queue.
+License: MIT
+License-File: LICENSE
+Author: Allen Chou
+Author-email: f1470891079@gmail.com
+Requires-Python: >=3.11,<4
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: bs4
+Requires-Dist: cachetic
+Requires-Dist: dictpress
+Requires-Dist: fastapi
+Requires-Dist: httpx
+Requires-Dist: huey
+Requires-Dist: logfire
+Requires-Dist: logging_bullet_train
+Requires-Dist: openai (>=1,<2)
+Requires-Dist: openai-agents (>=0.1.0,<1.0.0)
+Requires-Dist: playwright
+Requires-Dist: pydantic (>=2)
+Requires-Dist: pydantic-settings
+Requires-Dist: str-or-none
+Requires-Dist: tiktoken
+Requires-Dist: yarl
+Project-URL: Homepage, https://github.com/allen2c/web-queue
+Project-URL: PyPI, https://pypi.org/project/web-queue/
+Project-URL: Repository, https://github.com/allen2c/web-queue
+Description-Content-Type: text/markdown
+
+# Web-Queue
+
+[![PyPI version](https://img.shields.io/pypi/v/web-queue.svg)](https://pypi.org/project/web-queue/)
+[![Python Version](https://img.shields.io/pypi/pyversions/web-queue.svg)](https://pypi.org/project/web-queue/)
+[![License](https://img.shields.io/pypi/l/web-queue.svg)](https://opensource.org/licenses/MIT)
+
+A web content pipeline library.
+
+## License
+
+MIT License
+

web_queue2-0.1.0/README.md

@@ -0,0 +1,11 @@
+# Web-Queue
+
+[![PyPI version](https://img.shields.io/pypi/v/web-queue.svg)](https://pypi.org/project/web-queue/)
+[![Python Version](https://img.shields.io/pypi/pyversions/web-queue.svg)](https://pypi.org/project/web-queue/)
+[![License](https://img.shields.io/pypi/l/web-queue.svg)](https://opensource.org/licenses/MIT)
+
+A web content pipeline library.
+
+## License
+
+MIT License

web_queue2-0.1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[project]
+authors = [{ name = "Allen Chou", email = "f1470891079@gmail.com" }]
+dependencies = [
+  "bs4",
+  "cachetic",
+  "dictpress",
+  "fastapi",
+  "httpx",
+  "huey",
+  "logfire",
+  "logging_bullet_train",
+  "openai (>=1,<2)",
+  "openai-agents (>=0.1.0,<1.0.0)",
+  "playwright",
+  "pydantic (>=2)",
+  "pydantic-settings",
+  "str-or-none",
+  "tiktoken",
+  "yarl",
+]
+description = "Get web content from queue."
+license = { text = "MIT" }
+name = "web-queue2"
+readme = "README.md"
+requires-python = ">=3.11,<4"
+version = "0.1.0"
+
+[project.urls]
+Homepage = "https://github.com/allen2c/web-queue"
+"PyPI" = "https://pypi.org/project/web-queue/"
+Repository = "https://github.com/allen2c/web-queue"
+
+[tool.poetry]
+packages = [{ include = "web_queue" }]
+
+[tool.poetry.extras]
+all = []
+
+[tool.poetry.group.dev.dependencies]
+black = { extras = ["jupyter"], version = "*" }
+isort = "*"
+poetry-plugin-export = "*"
+pytest = "*"
+pytest-asyncio = "*"
+pytest-cov = "*"
+pytest-env = "*"
+pytest-xdist = "*"
+rich = "*"
+rich-color-support = "*"
+setuptools = "*"
+twine = "*"
+
+[tool.isort]
+profile = "black"
+
+[tool.flake8]
+ignore = ["E203", "E704", "W503"]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+env = ["ENVIRONMENT=test", "PYTEST_IS_RUNNING=true"]
+
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=2.0.0,<3.0.0"]

web_queue2-0.1.0/web_queue/VERSION

@@ -0,0 +1 @@
+0.1.0

web_queue2-0.1.0/web_queue/__init__.py

@@ -0,0 +1,3 @@
+import pathlib
+
+__version__ = pathlib.Path(__file__).parent.joinpath("VERSION").read_text().strip()

web_queue2-0.1.0/web_queue/app.py

@@ -0,0 +1,63 @@
+import asyncio
+import logging
+import typing
+
+import huey
+import logfire
+import logging_bullet_train as lbt
+from huey.api import Task
+
+import web_queue.config
+
+if typing.TYPE_CHECKING:
+    from web_queue.client import WebQueueClient
+    from web_queue.types.fetch_html_message import FetchHTMLMessage
+    from web_queue.types.html_content import HTMLContent
+
+lbt.set_logger("web_queue")
+
+logfire.configure()
+logfire.instrument_openai()
+
+logger = logging.getLogger(__name__)
+
+logger.info("Web queue app starting...")
+
+web_queue_settings = web_queue.config.Settings()
+logger.info(f"Web queue connecting to redis: {web_queue_settings.web_queue_safe_url}")
+
+huey_app = huey.RedisExpireHuey(
+    web_queue_settings.WEB_QUEUE_NAME,
+    url=web_queue_settings.WEB_QUEUE_URL.get_secret_value(),
+    expire_time=24 * 60 * 60,  # 24 hours
+)
+
+
+@huey_app.task(
+    retries=1,
+    retry_delay=8,
+    expires=24 * 60 * 60,
+    context=True,
+)
+def fetch_html(
+    message: typing.Union["FetchHTMLMessage", str, bytes], task: Task
+) -> str:
+    from web_queue.types.fetch_html_message import FetchHTMLMessage
+
+    message = FetchHTMLMessage.from_any(message)
+    message.id = task.id
+
+    logger.info(f"Fetching HTML from {message.data.url}")
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    try:
+        wq_client: "WebQueueClient" = web_queue_settings.web_queue_client
+        html_content: "HTMLContent" = loop.run_until_complete(
+            wq_client.fetch(**message.data.model_dump())
+        )
+        return html_content.model_dump_json()
+
+    finally:
+        loop.close()

web_queue2-0.1.0/web_queue/client/__init__.py

@@ -0,0 +1,3 @@
+from web_queue.client._client import WebQueueClient
+
+__all__ = ["WebQueueClient"]

web_queue2-0.1.0/web_queue/client/_client.py

@@ -0,0 +1,95 @@
+import functools
+import typing
+
+import httpx
+import yarl
+
+if typing.TYPE_CHECKING:
+    from web_queue.client.ai import AI
+    from web_queue.client.clean import Clean
+    from web_queue.client.config import Settings
+    from web_queue.client.web import Web
+    from web_queue.types.html_content import HTMLContent
+
+
+class WebQueueClient:
+    def __init__(self, settings: typing.Optional["Settings"] = None):
+        from web_queue.client.config import Settings
+
+        self.settings = settings or Settings()
+
+    @functools.cached_property
+    def web(self) -> "Web":
+        from web_queue.client.web import Web
+
+        return Web(self)
+
+    @functools.cached_property
+    def clean(self) -> "Clean":
+        from web_queue.client.clean import Clean
+
+        return Clean(self)
+
+    @functools.cached_property
+    def ai(self) -> "AI":
+        from web_queue.client.ai import AI
+
+        return AI(self)
+
+    async def fetch(
+        self,
+        url: yarl.URL | httpx.URL | str,
+        *,
+        headless: bool = False,
+        goto_timeout: int = 4000,  # 4 seconds
+        circling_times: int = 2,
+        scrolling_times: int = 3,
+        human_delay_base_delay: float = 1.2,
+        dynamic_content_loading_delay: float = 2.0,
+    ) -> "HTMLContent":
+        from web_queue.types.html_content import HTMLContent
+        from web_queue.utils.html_to_str import htmls_to_str
+
+        # Fetch HTML
+        html = await self.web.fetch(
+            url,
+            headless=headless,
+            goto_timeout=goto_timeout,
+            circling_times=circling_times,
+            scrolling_times=scrolling_times,
+            human_delay_base_delay=human_delay_base_delay,
+            dynamic_content_loading_delay=dynamic_content_loading_delay,
+        )
+
+        # Clean HTML
+        html = self.clean.as_main_content(html)
+
+        # Extract content metadata
+        html_metadata = await self.ai.as_html_metadata(html)
+
+        if not html_metadata:
+            raise ValueError(f"Failed to retrieve content metadata for url: {url}")
+
+        # Extract content body
+        content_body_htmls = html.select(html_metadata.content_body_css_selector)
+        if not content_body_htmls:
+            raise ValueError(
+                "Failed to retrieve content body by css selector "
+                + f"'{html_metadata.content_body_css_selector}' "
+                + f"for url: '{url}'"
+            )
+
+        content_body_text = htmls_to_str(content_body_htmls)
+
+        html_content = HTMLContent(
+            title=html_metadata.title,
+            author=html_metadata.author,
+            chapter_id=html_metadata.chapter_id,
+            chapter_number=html_metadata.chapter_number,
+            content=content_body_text,
+            created_date=html_metadata.created_date,
+            updated_date=html_metadata.updated_date,
+        )
+
+        html_content._html = str(html)
+        return html_content

web_queue2-0.1.0/web_queue/client/ai/__init__.py

@@ -0,0 +1,3 @@
+from web_queue.client.ai._ai import AI
+
+__all__ = ["AI"]

web_queue2-0.1.0/web_queue/client/ai/_ai.py

@@ -0,0 +1,140 @@
+import asyncio
+import datetime
+import hashlib
+import logging
+import textwrap
+import typing
+import zoneinfo
+
+import logfire
+from rich.pretty import pretty_repr
+
+from web_queue.client import WebQueueClient
+from web_queue.types.html_metadata_response import HTMLMetadataResponse
+from web_queue.utils.compression import compress, decompress
+
+if typing.TYPE_CHECKING:
+    import bs4
+
+logger = logging.getLogger(__name__)
+
+
+class AI:
+    def __init__(self, client: WebQueueClient):
+        self.client = client
+
+    @logfire.instrument
+    async def as_html_metadata(
+        self, html: typing.Union["bs4.BeautifulSoup", typing.Text]
+    ) -> typing.Optional[HTMLMetadataResponse]:
+        """Extract content metadata and CSS selector from HTML.
+
+        Analyzes HTML to find content body selector and extract metadata values.
+        """
+        openai_client = self.client.settings.openai_client
+        model_name = self.client.settings.OPENAI_MODEL
+
+        html = str(html)
+
+        logger.info(f"AI is extracting content metadata from HTML: {html}")
+
+        cache_key = (
+            "retrieve_html_content_metadata:"
+            + f"{hashlib.md5(html.encode('utf-8')).hexdigest()}"
+        )
+
+        might_cached_data: typing.Text | None = await asyncio.to_thread(
+            self.client.settings.compressed_base64_cache.get, cache_key
+        )
+        if might_cached_data is not None:
+            logger.debug(
+                "Hit cache 'as_html_content_metadata':"
+                + f"{pretty_repr(html, max_string=32)}"
+            )
+            return HTMLMetadataResponse.model_validate_json(
+                decompress(might_cached_data)
+            )
+
+        # Get current time in Asia/Taipei timezone for relative date parsing
+        current_time = datetime.datetime.now(zoneinfo.ZoneInfo("Asia/Taipei"))
+        current_time_iso = current_time.isoformat()
+
+        system_prompt = textwrap.dedent(
+            f"""
+            You are an HTML structure analysis expert. Task: From the provided HTML, extract content metadata and identify CSS selectors.
+
+            Current time (Asia/Taipei timezone): {current_time_iso}
+
+            Instructions:
+            1. **content_body_css_selector**: Find the CSS selector for the main content body element containing ONLY the article text.
+               - Look for semantic tags like <article>, <main>, or <div> with classes/IDs like 'body', 'content', 'text', 'novel-body'.
+               - EXCLUDE elements containing metadata (title, author, dates, navigation, footer, ads, comments).
+               - Example: 'div.article-body', 'div#novel-content', 'div.p-novel__text'.
+               - Return empty string if not found.
+
+            2. **title**: Extract the actual title text (chapter title, article title).
+               - Look in <h1>, <h2>, or elements with class/id containing 'title', 'heading'.
+               - Return the text content, not the CSS selector.
+               - Return empty string if not found.
+
+            3. **author**: Extract the actual author name or username.
+               - Look in elements with class/id containing 'author', 'writer', 'username'.
+               - Return the text content.
+               - Return empty string if not found.
+
+            4. **chapter_id**: Extract the actual chapter identifier (e.g., '12345', 'ch-001').
+               - Look in data attributes, URLs, or element IDs.
+               - Return empty string if not found.
+
+            5. **chapter_number**: Extract the actual chapter number (e.g., '1', '42', 'Chapter 5').
+               - Return empty string if not found.
+
+            6. **created_date** and **updated_date**: Parse dates to ISO 8601 format with +08:00 timezone.
+               - For absolute dates: Convert to 'YYYY-MM-DDTHH:MM:SS+08:00' format.
+               - For relative dates ('2 days ago', '3 hours ago'): Calculate from current_time and format.
+               - Return empty string if not found.
+
+            Rules:
+            - If any field is not found or unclear, return empty string "".
+            - Do not guess or make up information.
+            - Focus on precision and accuracy.
+
+            Now, analyze the provided HTML and extract all available metadata.
+            """  # noqa: E501
+        ).strip()
+
+        try:
+            parsed_cmpl = await openai_client.chat.completions.parse(
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": html},
+                ],
+                model=model_name,
+                response_format=HTMLMetadataResponse,
+            )
+            response_msg = parsed_cmpl.choices[0].message
+            if response_msg.refusal:
+                logger.error(f"LLM refusal: {response_msg.refusal}")
+                return None
+
+            elif response_msg.parsed:
+                output: HTMLMetadataResponse = response_msg.parsed
+                output._html = html
+                logger.info(f"LLM response: {output}")
+
+                # Cache the response
+                await asyncio.to_thread(
+                    self.client.settings.compressed_base64_cache.set,
+                    cache_key,
+                    compress(output.model_dump_json()),
+                )
+
+                return output
+
+            else:
+                logger.error(f"LLM Error for message: {response_msg}")
+                return None
+
+        except Exception as e:
+            logger.error(f"Parsing failed: {e}")
+            return None

web_queue2-0.1.0/web_queue/client/clean/__init__.py

@@ -0,0 +1,3 @@
+from web_queue.client.clean._clean import Clean
+
+__all__ = ["Clean"]

web_queue2-0.1.0/web_queue/client/clean/_clean.py

@@ -0,0 +1,25 @@
+import logging
+import typing
+
+import bs4
+
+from web_queue.client import WebQueueClient
+from web_queue.utils.html_cleaner import HTMLCleaner
+
+logger = logging.getLogger(__name__)
+
+
+class Clean:
+    def __init__(self, client: WebQueueClient):
+        self.client = client
+
+    def as_main_content(self, html: bs4.BeautifulSoup | str) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+
+        logger.info(f"Cleaning HTML: {html}")
+        cleaned_html = HTMLCleaner.clean_as_main_content_html_str(html)
+        return bs4.BeautifulSoup(cleaned_html, "html.parser")

web_queue2-0.1.0/web_queue/client/config.py

@@ -0,0 +1,61 @@
+import functools
+import pathlib
+import typing
+
+import cachetic
+import openai
+import pydantic as pydantic
+import pydantic_settings
+
+
+class Settings(pydantic_settings.BaseSettings):
+    OPENAI_MODEL: str = pydantic.Field(default="gpt-4.1-nano")
+    OPENAI_API_KEY: pydantic.SecretStr = pydantic.SecretStr("")
+
+    # Cache
+    WEB_CACHE_PATH: typing.Text = pydantic.Field(default="./.cache/web.cache")
+    WEB_CACHE_EXPIRE_SECONDS: int = pydantic.Field(default=60 * 60 * 24)  # 1 day
+    WEB_SCREENSHOT_PATH: typing.Text = pydantic.Field(default="./data/screenshots")
+    WEB_PDF_PATH: typing.Text = pydantic.Field(default="./data/pdfs")
+    COMPRESSED_BASE64_CACHE_PATH: typing.Text = pydantic.Field(
+        default="./.cache/compressed_base64.cache"
+    )
+    COMPRESSED_BASE64_CACHE_EXPIRE_SECONDS: int = pydantic.Field(
+        default=60 * 60 * 24
+    )  # 1 day
+
+    @functools.cached_property
+    def openai_client(self) -> openai.AsyncOpenAI:
+        return openai.AsyncOpenAI(api_key=self.OPENAI_API_KEY.get_secret_value())
+
+    @functools.cached_property
+    def web_cache(self) -> "cachetic.Cachetic[typing.Text]":
+        import cachetic
+
+        return cachetic.Cachetic(
+            object_type=pydantic.TypeAdapter(typing.Text),
+            cache_url=pathlib.Path(self.WEB_CACHE_PATH),
+            default_ttl=self.WEB_CACHE_EXPIRE_SECONDS,
+        )
+
+    @functools.cached_property
+    def compressed_base64_cache(self) -> "cachetic.Cachetic[typing.Text]":
+        import cachetic
+
+        return cachetic.Cachetic(
+            object_type=pydantic.TypeAdapter(typing.Text),
+            cache_url=pathlib.Path(self.COMPRESSED_BASE64_CACHE_PATH),
+            default_ttl=self.COMPRESSED_BASE64_CACHE_EXPIRE_SECONDS,
+        )
+
+    @property
+    def web_screenshot_path(self) -> pathlib.Path:
+        _path = pathlib.Path(self.WEB_SCREENSHOT_PATH)
+        _path.mkdir(parents=True, exist_ok=True)
+        return _path
+
+    @property
+    def web_pdf_path(self) -> pathlib.Path:
+        _path = pathlib.Path(self.WEB_PDF_PATH)
+        _path.mkdir(parents=True, exist_ok=True)
+        return _path

web_queue2-0.1.0/web_queue/client/web/__init__.py

@@ -0,0 +1,3 @@
+from web_queue.client.web._web import Web
+
+__all__ = ["Web"]

web_queue2-0.1.0/web_queue/client/web/_web.py

@@ -0,0 +1,175 @@
+import asyncio
+import logging
+import secrets
+import time
+import typing
+
+import bs4
+import fastapi
+import httpx
+import yarl
+from playwright._impl._api_structures import ViewportSize
+from playwright.async_api import async_playwright
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+from str_or_none import str_or_none
+
+from web_queue.client import WebQueueClient
+from web_queue.utils.compression import compress, decompress
+from web_queue.utils.human_delay import human_delay
+from web_queue.utils.page_with_init_script import page_with_init_script
+from web_queue.utils.simulate_mouse_circling import simulate_mouse_circling
+from web_queue.utils.simulate_scrolling import simulate_scrolling
+
+logger = logging.getLogger(__name__)
+
+
+class Web:
+    USER_AGENTS: typing.ClassVar[typing.Tuple[typing.Text, ...]] = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",  # noqa: E501
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",  # noqa: E501
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",  # noqa: E501
+    )
+    VIEWPORT_SIZES: typing.ClassVar[typing.Tuple[typing.Tuple[int, int], ...]] = (
+        (1920, 1080),
+        (1366, 768),
+        (1440, 900),
+    )
+
+    def __init__(self, client: WebQueueClient):
+        self.client = client
+
+    async def fetch(
+        self,
+        url: typing.Text | yarl.URL | httpx.URL,
+        *,
+        headless: bool = True,
+        goto_timeout: int = 4000,  # 4 seconds
+        circling_times: int = 3,
+        scrolling_times: int = 3,
+        human_delay_base_delay: float = 1.2,
+        dynamic_content_loading_delay: float = 2.0,
+    ) -> bs4.BeautifulSoup:
+        _url = str_or_none(str(url))
+        if not _url:
+            raise fastapi.exceptions.HTTPException(status_code=400, detail="Empty URL")
+
+        html_content: typing.Text | None = None
+        h_delay = human_delay_base_delay
+        d_delay = dynamic_content_loading_delay
+
+        logger.info(f"Browser is fetching {_url}")
+        maybe_html_content = self.client.settings.web_cache.get(_url)
+        if maybe_html_content:
+            logger.debug(f"Hit web cache for {_url}")
+            html_content = await asyncio.to_thread(
+                decompress, maybe_html_content, format="zstd"
+            )
+            return bs4.BeautifulSoup(html_content, "html.parser")
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(
+                headless=headless,
+                args=[
+                    "--no-sandbox",
+                    "--disable-blink-features=AutomationControlled",
+                    "--disable-dev-shm-usage",
+                    "--disable-web-security",
+                    "--disable-features=VizDisplayCompositor",
+                ],
+            )
+
+            # Create context
+            _viewport_size = secrets.choice(self.VIEWPORT_SIZES)
+            _viewport = ViewportSize(width=_viewport_size[0], height=_viewport_size[1])
+            context = await browser.new_context(
+                user_agent=secrets.choice(self.USER_AGENTS),
+                viewport=_viewport,
+                locale="en-US",
+                timezone_id="Asia/Tokyo",
+                permissions=["geolocation"],
+                extra_http_headers={
+                    "Accept-Language": "en-US,en;q=0.9,ja;q=0.8",
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",  # noqa: E501
+                    "Accept-Encoding": "gzip, deflate, br",
+                    "Accept-Charset": "utf-8",
+                },
+            )
+
+            # Create new page
+            page = await context.new_page()
+
+            # Inject script to hide automation features
+            page = await page_with_init_script(page)
+
+            try:
+                # Navigate to URL
+                logger.debug(f"Navigating (timeout: {goto_timeout}ms) to {_url}")
+                try:
+                    await page.goto(
+                        _url, wait_until="domcontentloaded", timeout=goto_timeout
+                    )  # Wait for network idle
+                except PlaywrightTimeoutError:
+                    logger.info(f"Timeout for goto '{_url}', continuing...")
+                await human_delay(h_delay)  # Initial delay
+
+                # Wait for full page load (additional checks)
+                logger.debug(f"Waiting {h_delay}s for full page load")
+                await page.wait_for_load_state("domcontentloaded")
+                await human_delay(h_delay)
+
+                # Simulate smooth mouse circling three times
+                start_position = None
+                for i in range(circling_times):
+                    logger.debug(f"Simulating mouse circling {i+1} of {circling_times}")
+                    start_position = await simulate_mouse_circling(
+                        page, _viewport, start_position=start_position
+                    )
+                    await human_delay(h_delay)
+
+                # Simulate scrolling three times
+                for i in range(scrolling_times):
+                    logger.debug(f"Simulating scrolling {i+1} of {scrolling_times}")
+                    await simulate_scrolling(page, scroll_direction="down")
+                    await human_delay(h_delay)
+
+                # Extra delay for dynamic content loading
+                logger.debug(f"Delaying {d_delay}s for dynamic content loading")
+                await human_delay(d_delay)
+
+                # Get full HTML content
+                html_content = await page.content()
+                html_content = str_or_none(html_content)
+                html_content_size = len(html_content or " ")
+
+                logger.info(
+                    f"Fetched HTML content size: {html_content_size} for {_url}"
+                )
+
+                # Screenshot and PDF
+                snapshot_filename = f"{int(time.time()*1E3)}_{secrets.token_hex(2)}"
+                screenshot_path = self.client.settings.web_screenshot_path.joinpath(
+                    f"{snapshot_filename}.png"
+                )
+                screenshot_path.write_bytes(await page.screenshot())
+                logger.info(f"Screenshot saved to {screenshot_path}")
+                pdf_path = self.client.settings.web_pdf_path.joinpath(
+                    f"{snapshot_filename}.pdf"
+                )
+                await page.pdf(path=pdf_path, print_background=True)
+                logger.info(f"PDF saved to {pdf_path}")
+
+            finally:
+                await browser.close()
+
+        if not html_content:
+            raise fastapi.exceptions.HTTPException(
+                status_code=500, detail="Failed to fetch content"
+            )
+
+        await asyncio.to_thread(
+            self.client.settings.web_cache.set,
+            _url,
+            compress(html_content, format="zstd"),
+        )
+
+        return bs4.BeautifulSoup(html_content, "html.parser")

web_queue2-0.1.0/web_queue/config.py

@@ -0,0 +1,33 @@
+import functools
+import typing
+
+import pydantic
+import pydantic_settings
+import yarl
+from str_or_none import str_or_none
+
+if typing.TYPE_CHECKING:
+    from web_queue.client import WebQueueClient
+
+
+class Settings(pydantic_settings.BaseSettings):
+    WEB_QUEUE_NAME: str = pydantic.Field(default="web-queue")
+    WEB_QUEUE_URL: pydantic.SecretStr = pydantic.SecretStr("")
+
+    @pydantic.model_validator(mode="after")
+    def validate_values(self) -> typing.Self:
+        if str_or_none(self.WEB_QUEUE_NAME) is None:
+            raise ValueError("WEB_QUEUE_NAME is required")
+        if str_or_none(self.WEB_QUEUE_URL.get_secret_value()) is None:
+            raise ValueError("WEB_QUEUE_URL is required")
+        return self
+
+    @functools.cached_property
+    def web_queue_client(self) -> "WebQueueClient":
+        from web_queue.client import WebQueueClient
+
+        return WebQueueClient()
+
+    @property
+    def web_queue_safe_url(self) -> str:
+        return str(yarl.URL(self.WEB_QUEUE_URL.get_secret_value()).with_password("***"))

web_queue2-0.1.0/web_queue/types/fetch_html_message.py

@@ -0,0 +1,26 @@
+import typing
+
+import pydantic
+from str_or_none import str_or_none
+
+from web_queue.types.message import Message
+
+
+class FetchHTMLMessageRequest(pydantic.BaseModel):
+    url: str
+    headless: bool = False
+    goto_timeout: int = 4000
+    circling_times: int = 2
+    scrolling_times: int = 3
+    human_delay_base_delay: float = 1.2
+    dynamic_content_loading_delay: float = 2
+
+    @pydantic.model_validator(mode="after")
+    def validate_url(self) -> typing.Self:
+        if not str_or_none(self.url):
+            raise ValueError("URL is required")
+        return self
+
+
+class FetchHTMLMessage(Message):
+    data: FetchHTMLMessageRequest

web_queue2-0.1.0/web_queue/types/html_content.py

@@ -0,0 +1,18 @@
+import logging
+
+import pydantic
+
+logger = logging.getLogger(__name__)
+
+
+class HTMLContent(pydantic.BaseModel):
+    title: str = pydantic.Field(default="")
+    author: str = pydantic.Field(default="")
+    chapter_id: str = pydantic.Field(default="")
+    chapter_number: str = pydantic.Field(default="")
+    content: str = pydantic.Field(default="")
+    created_date: str = pydantic.Field(default="")
+    updated_date: str = pydantic.Field(default="")
+
+    # Private attributes
+    _html: str = pydantic.PrivateAttr(default="")

web_queue2-0.1.0/web_queue/types/html_metadata_response.py

@@ -0,0 +1,82 @@
+import logging
+
+import pydantic
+
+logger = logging.getLogger(__name__)
+
+
+class HTMLMetadataResponse(pydantic.BaseModel):
+    """Structured response for HTML content metadata and element locators.
+
+    Extracts content body CSS selector and metadata values.
+    """
+
+    title: str = pydantic.Field(
+        default="",
+        description=(
+            "The actual title text of the content "
+            "(e.g., chapter title, article title). "
+            "Return empty string if not found."
+        ),
+    )
+
+    author: str = pydantic.Field(
+        default="",
+        description=(
+            "The actual author name or username. " "Return empty string if not found."
+        ),
+    )
+
+    chapter_id: str = pydantic.Field(
+        default="",
+        description=(
+            "The actual chapter ID or identifier (e.g., '12345', 'ch-001'). "
+            "Return empty string if not found."
+        ),
+    )
+
+    chapter_number: str = pydantic.Field(
+        default="",
+        description=(
+            "The actual chapter number (e.g., '1', '42', 'Chapter 5'). "
+            "Return empty string if not found."
+        ),
+    )
+
+    content_body_css_selector: str = pydantic.Field(
+        default="",
+        description=(
+            "CSS selector for the main content body element "
+            "containing article text only. "
+            "Exclude metadata like title, author, dates. "
+            "Example: 'div.article-body', 'div#novel-content'. "
+            "Use standard CSS syntax. Return empty string if not found."
+        ),
+    )
+
+    created_date: str = pydantic.Field(
+        default="",
+        description=(
+            "The content creation date in ISO 8601 format "
+            "with Asia/Taipei timezone "
+            "(e.g., '2025-10-12T14:30:00+08:00'). "
+            "Parse relative dates like '2 days ago' "
+            "using the current_time provided in the system prompt. "
+            "Return empty string if not found."
+        ),
+    )
+
+    updated_date: str = pydantic.Field(
+        default="",
+        description=(
+            "The content last update date in ISO 8601 format "
+            "with Asia/Taipei timezone "
+            "(e.g., '2025-10-12T14:30:00+08:00'). "
+            "Parse relative dates like '2 days ago' "
+            "using the current_time provided in the system prompt. "
+            "Return empty string if not found."
+        ),
+    )
+
+    # Private attributes
+    _html: str = pydantic.PrivateAttr(default="")

web_queue2-0.1.0/web_queue/types/message.py

@@ -0,0 +1,34 @@
+import enum
+import json
+import typing
+
+import pydantic
+
+
+class MessageStatus(enum.StrEnum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class Message(pydantic.BaseModel):
+    id: str | None = None
+    data: typing.Any
+    status: MessageStatus = pydantic.Field(default=MessageStatus.PENDING)
+    total_steps: int = pydantic.Field(default=100)
+    completed_steps: int = pydantic.Field(default=0)
+    error: typing.Optional[str] = pydantic.Field(default=None)
+
+    @classmethod
+    def from_any(cls, any: typing.Union[pydantic.BaseModel, typing.Dict, str, bytes]):
+        if isinstance(any, pydantic.BaseModel):
+            return cls.model_validate_json(any.model_dump_json())
+        elif isinstance(any, typing.Dict):
+            return cls.model_validate_json(json.dumps(any))
+        elif isinstance(any, str):
+            return cls.model_validate_json(any)
+        elif isinstance(any, bytes):
+            return cls.model_validate_json(any)
+        else:
+            raise ValueError(f"Invalid type: {type(any)}")

web_queue2-0.1.0/web_queue/utils/compression.py

@@ -0,0 +1,29 @@
+import base64
+import typing
+
+import fastapi
+import zstandard
+
+
+def compress(
+    data: str, *, level: int = 9, format: typing.Literal["zstd"] = "zstd"
+) -> str:
+    if format == "zstd":
+        return base64.b64encode(
+            zstandard.compress(data.encode("utf-8"), level=level)
+        ).decode("utf-8")
+    else:
+        raise fastapi.exceptions.HTTPException(
+            status_code=400, detail=f"Invalid format: {format}"
+        )
+
+
+def decompress(data: str, *, format: typing.Literal["zstd"] = "zstd") -> str:
+    if format == "zstd":
+        return zstandard.decompress(base64.b64decode(data.encode("utf-8"))).decode(
+            "utf-8"
+        )
+    else:
+        raise fastapi.exceptions.HTTPException(
+            status_code=400, detail=f"Invalid format: {format}"
+        )

web_queue2-0.1.0/web_queue/utils/html_cleaner.py

@@ -0,0 +1,145 @@
+import re
+import typing
+
+import bs4
+
+DEFAULT_KEEP_TAGS: typing.Tuple[typing.Text, ...] = (
+    "a",
+    "article",
+    "body",
+    "br",
+    "div",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "hr",
+    "html",
+    "li",
+    "main",
+    "ol",
+    "p",
+    "section",
+    "table",
+    "tbody",
+    "td",
+    "th",
+    "tr",
+    "ul",
+)
+DEFAULT_KEEP_ATTRIBUTES: typing.Tuple[typing.Text, ...] = ("id", "class")
+DEFAULT_DROP_TAGS: typing.Tuple[typing.Text, ...] = ("script", "style", "iframe")
+
+
+class HTMLCleaner:
+    @staticmethod
+    def clean_as_main_content_html(
+        html: typing.Text | bs4.BeautifulSoup,
+    ) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+        html = HTMLCleaner.clean_all_comments(html)
+        html = HTMLCleaner.keep_only_tags(html)
+        html = HTMLCleaner.clean_tags(html)
+        html = HTMLCleaner.clean_attributes(html)
+        html = HTMLCleaner.keep_first_class_name(html)
+        return html
+
+    @staticmethod
+    def clean_as_main_content_html_str(
+        html: typing.Text | bs4.BeautifulSoup,
+    ) -> str:
+        html = HTMLCleaner.clean_as_main_content_html(html)
+        return re.sub(r">\s+<", "><", str(html))
+
+    @staticmethod
+    def keep_only_tags(
+        html: typing.Text | bs4.BeautifulSoup,
+        tags: typing.List[typing.Text] | None = None,
+    ) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+        tags = tags or list(DEFAULT_KEEP_TAGS)
+
+        # Find all tags that are not in the keep list and decompose them
+        for tag in html.find_all():
+            if tag.name not in tags:
+                tag.decompose()
+
+        return html
+
+    @staticmethod
+    def keep_first_class_name(
+        html: typing.Text | bs4.BeautifulSoup,
+    ) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+
+        # Keep only the first class name for elements with multiple classes
+        for tag in html.find_all(attrs={"class": True}):
+            class_attr = tag.get("class")
+            if isinstance(class_attr, list) and len(class_attr) > 1:
+                tag["class"] = class_attr[0]
+            elif isinstance(class_attr, str):
+                classes = class_attr.split()
+                if len(classes) > 1:
+                    tag["class"] = classes[0]
+
+        return html
+
+    @staticmethod
+    def clean_attributes(
+        html: typing.Text | bs4.BeautifulSoup,
+        attributes: typing.List[typing.Text] | None = None,
+    ) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+        attributes = attributes or list(DEFAULT_KEEP_ATTRIBUTES)
+        for tag in html.find_all():
+            for attribute in list(tag.attrs):
+                if attribute not in attributes:
+                    tag.attrs.pop(attribute, None)
+
+        return html
+
+    @staticmethod
+    def clean_all_comments(html: typing.Text | bs4.BeautifulSoup) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+        for comment in html.find_all(text=lambda text: isinstance(text, bs4.Comment)):
+            comment.decompose()
+        return html
+
+    @staticmethod
+    def clean_tags(
+        html: typing.Text | bs4.BeautifulSoup,
+        tags: typing.List[typing.Text] | None = None,
+    ) -> bs4.BeautifulSoup:
+        html = (
+            bs4.BeautifulSoup(html, "html.parser")
+            if isinstance(html, typing.Text)
+            else html
+        )
+        tags = tags or list(DEFAULT_DROP_TAGS)
+
+        for tag in html.find_all(tags):
+            tag.decompose()
+
+        return html

web_queue2-0.1.0/web_queue/utils/html_to_str.py

@@ -0,0 +1,21 @@
+import typing
+
+import bs4
+
+
+def html_to_str(html: bs4.BeautifulSoup | bs4.Tag | str) -> str:
+    html = bs4.BeautifulSoup(html, "html.parser") if isinstance(html, str) else html
+
+    full_text = ""
+    for p in html.find_all("p"):
+        content = p.get_text(separator="\n", strip=True)
+        full_text += content
+        full_text += "\n"
+
+    return full_text.strip()
+
+
+def htmls_to_str(
+    htmls: typing.List[bs4.BeautifulSoup | bs4.Tag | str] | bs4.ResultSet[bs4.Tag],
+) -> str:
+    return "\n\n".join(html_to_str(h) for h in htmls)

web_queue2-0.1.0/web_queue/utils/human_delay.py

@@ -0,0 +1,11 @@
+import asyncio
+import random
+
+
+async def human_delay(
+    base_delay: float = 1.2, *, jitter_ratio: tuple[float, float] = (0.5, 1.5)
+) -> None:
+    jitter = random.uniform(jitter_ratio[0], jitter_ratio[1])
+    total_delay = base_delay * jitter
+    await asyncio.sleep(total_delay)
+    return None

web_queue2-0.1.0/web_queue/utils/page_with_init_script.py

@@ -0,0 +1,15 @@
+import playwright.async_api
+
+
+async def page_with_init_script(
+    page: playwright.async_api.Page,
+) -> playwright.async_api.Page:
+    await page.add_init_script(
+        """
+        Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+        Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
+        Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en', 'ja']});  # noqa: E501
+        window.chrome = {runtime: {}};
+    """
+    )
+    return page

web_queue2-0.1.0/web_queue/utils/simulate_mouse_circling.py

@@ -0,0 +1,49 @@
+import asyncio
+import math
+import random
+import typing
+
+import playwright.async_api
+from playwright._impl._api_structures import ViewportSize
+
+Number: typing.TypeAlias = float | int
+
+
+async def simulate_mouse_circling(
+    page: playwright.async_api.Page,
+    default_viewport_size: ViewportSize | None = None,
+    *,
+    start_position: tuple[Number, Number] | None = None,
+) -> tuple[Number, Number]:
+    _viewport_size = (
+        page.viewport_size
+        or default_viewport_size
+        or ViewportSize(width=1920, height=1080)
+    )
+
+    # Random starting position
+    if start_position:
+        start_x = start_position[0]
+        start_y = start_position[1]
+    else:
+        start_x = random.randint(100, _viewport_size["width"] - 100)
+        start_y = random.randint(100, _viewport_size["height"] - 100)
+    center_x = start_x + 100
+    center_y = start_y + 100
+    radius = 50
+    x = center_x
+    y = center_y
+
+    # Simulate smooth circle: Move to multiple points
+    for angle in range(0, 360, 60):  # Every 30 degrees a point
+        rad = (angle * 3.14159) / 180
+        x = center_x + radius * random.uniform(0.8, 1.2) * random.choice([-1, 1]) * abs(
+            math.cos(rad)
+        )
+        y = center_y + radius * random.uniform(0.8, 1.2) * random.choice([-1, 1]) * abs(
+            math.sin(rad)
+        )
+        await page.mouse.move(x, y, steps=random.randint(10, 20))  # Smooth movement
+        await asyncio.sleep(random.uniform(0.01, 0.05))  # Tiny delay
+
+    return (x, y)

web_queue2-0.1.0/web_queue/utils/simulate_scrolling.py

@@ -0,0 +1,18 @@
+import random
+
+import playwright.async_api
+from typing_extensions import Literal
+
+
+async def simulate_scrolling(
+    page: playwright.async_api.Page,
+    scroll_direction: Literal["down", "up"] | None = None,
+    scroll_distance: int | None = None,
+) -> None:
+    scroll_direction = scroll_direction or random.choice(["down", "up"])
+    scroll_distance = scroll_distance or random.randint(200, 800)
+    if scroll_direction == "down":
+        await page.mouse.wheel(0, scroll_distance)
+    else:
+        await page.mouse.wheel(0, -scroll_distance)
+    return None