webquest 0.7.3__py3-none-any.whl

webquest/browsers/__init__.py

@@ -0,0 +1,4 @@
+from webquest.browsers.browser import Browser
+from webquest.browsers.hyperbrowser import Hyperbrowser
+
+__all__ = ["Browser", "Hyperbrowser"]

webquest/browsers/browser.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from typing import AsyncContextManager
+
+from playwright.async_api import BrowserContext
+
+
+class Browser(ABC):
+    """
+    Abstract base class for browser implementations.
+
+    This class defines the interface for obtaining a browser context, which is used
+    for performing web scraping operations.
+    """
+
+    @abstractmethod
+    def get_context(self) -> AsyncContextManager[BrowserContext]:
+        """
+        Get an asynchronous context manager that yields a Playwright BrowserContext.
+
+        Returns:
+            AsyncContextManager[BrowserContext]: An async context manager that yields a BrowserContext.
+        """
+        ...

webquest/browsers/hyperbrowser.py

@@ -0,0 +1,52 @@
+from contextlib import asynccontextmanager
+from typing import AsyncIterator, override
+
+from hyperbrowser import AsyncHyperbrowser
+from playwright.async_api import BrowserContext, async_playwright
+
+from webquest.browsers.browser import Browser
+
+
+class Hyperbrowser(Browser):
+    """
+    A Browser implementation that uses Hyperbrowser for remote browser sessions.
+
+    This class manages the creation and cleanup of Hyperbrowser sessions and provides
+    a Playwright BrowserContext connected to the remote session.
+    """
+
+    def __init__(
+        self,
+        client: AsyncHyperbrowser | None = None,
+    ):
+        """
+        Initialize the Hyperbrowser instance.
+
+        Args:
+            client (AsyncHyperbrowser | None): An optional AsyncHyperbrowser client.
+                If not provided, a new client will be created.
+        """
+        if client is None:
+            client = AsyncHyperbrowser()
+        self._client = client
+
+    @override
+    @asynccontextmanager
+    async def get_context(self) -> AsyncIterator[BrowserContext]:
+        """
+        Get a browser context from a new Hyperbrowser session.
+
+        This method creates a new session, connects to it using Playwright, yields
+        the context, and ensures the session is stopped afterwards.
+
+        Yields:
+            BrowserContext: The Playwright browser context connected to the Hyperbrowser session.
+        """
+        session = await self._client.sessions.create()
+        async with async_playwright() as p:
+            browser = await p.chromium.connect_over_cdp(session.ws_endpoint)
+            context = browser.contexts[0]
+            try:
+                yield context
+            finally:
+                await self._client.sessions.stop(session.id)

webquest/scrapers/__init__.py

@@ -0,0 +1,47 @@
+from webquest.scrapers.any_article import (
+    AnyArticle,
+    AnyArticleRequest,
+    AnyArticleResponse,
+)
+from webquest.scrapers.duckduckgo_search import (
+    DuckDuckGoSearch,
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+)
+from webquest.scrapers.google_news_search import (
+    GoogleNewsSearch,
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+from webquest.scrapers.openai_parser import OpenAIParser
+from webquest.scrapers.scraper import Scraper
+from webquest.scrapers.youtube_search import (
+    YouTubeSearch,
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+from webquest.scrapers.youtube_transcript import (
+    YouTubeTranscript,
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+
+__all__ = [
+    "AnyArticle",
+    "AnyArticleRequest",
+    "AnyArticleResponse",
+    "DuckDuckGoSearch",
+    "DuckDuckGoSearchRequest",
+    "DuckDuckGoSearchResponse",
+    "GoogleNewsSearch",
+    "GoogleNewsSearchRequest",
+    "GoogleNewsSearchResponse",
+    "OpenAIParser",
+    "Scraper",
+    "YouTubeSearch",
+    "YouTubeSearchRequest",
+    "YouTubeSearchResponse",
+    "YouTubeTranscript",
+    "YouTubeTranscriptRequest",
+    "YouTubeTranscriptResponse",
+]

webquest/scrapers/any_article/__init__.py

@@ -0,0 +1,11 @@
+from webquest.scrapers.any_article.scraper import (
+    AnyArticle,
+    AnyArticleRequest,
+    AnyArticleResponse,
+)
+
+__all__ = [
+    "AnyArticle",
+    "AnyArticleRequest",
+    "AnyArticleResponse",
+]

webquest/scrapers/any_article/schemas.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class AnyArticleRequest(BaseModel):
+    url: str
+
+
+class AnyArticleResponse(BaseModel):
+    publisher: str
+    title: str
+    published_at: str
+    authors: list[str]
+    content: str

webquest/scrapers/any_article/scraper.py

@@ -0,0 +1,43 @@
+from typing import override
+
+from openai import AsyncOpenAI
+from playwright.async_api import BrowserContext
+
+from webquest.browsers.browser import Browser
+from webquest.scrapers.any_article.schemas import AnyArticleRequest, AnyArticleResponse
+from webquest.scrapers.openai_parser import OpenAIParser
+
+
+class AnyArticle(OpenAIParser[AnyArticleRequest, AnyArticleResponse]):
+    """Scraper to extract the main article from any web page using OpenAI."""
+
+    request = AnyArticleRequest
+    response = AnyArticleResponse
+
+    def __init__(
+        self,
+        browser: Browser,
+        client: AsyncOpenAI | None = None,
+        model: str = "gpt-5-mini",
+        character_limit: int = 4000,
+    ) -> None:
+        super().__init__(
+            browser=browser,
+            response_type=AnyArticleResponse,
+            client=client,
+            model=model,
+            input="Parse the following web page and extract the main article:\n\n",
+            character_limit=character_limit,
+        )
+
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: AnyArticleRequest,
+    ) -> str:
+        page = await context.new_page()
+        await page.goto(request.url, wait_until="domcontentloaded")
+        await page.wait_for_timeout(3000)
+        html = await page.content()
+        return html

webquest/scrapers/duckduckgo_search/__init__.py

@@ -0,0 +1,11 @@
+from webquest.scrapers.duckduckgo_search.scraper import (
+    DuckDuckGoSearch,
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+)
+
+__all__ = [
+    "DuckDuckGoSearch",
+    "DuckDuckGoSearchRequest",
+    "DuckDuckGoSearchResponse",
+]

webquest/scrapers/duckduckgo_search/schemas.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+
+class DuckDuckGoSearchRequest(BaseModel):
+    query: str
+
+
+class Page(BaseModel):
+    site: str
+    url: str
+    title: str
+    description: str
+
+
+class DuckDuckGoSearchResponse(BaseModel):
+    pages: list[Page]

webquest/scrapers/duckduckgo_search/scraper.py

@@ -0,0 +1,84 @@
+import asyncio
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.scrapers.duckduckgo_search.schemas import (
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+    Page,
+)
+from webquest.scrapers.scraper import Scraper
+
+
+class DuckDuckGoSearch(Scraper[DuckDuckGoSearchRequest, str, DuckDuckGoSearchResponse]):
+    """Scraper to perform a DuckDuckGo web search and parse the results."""
+
+    request = DuckDuckGoSearchRequest
+    response = DuckDuckGoSearchResponse
+
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: DuckDuckGoSearchRequest,
+    ) -> str:
+        url = f"https://duckduckgo.com/?origin=funnel_home_website&t=h_&q={quote_plus(request.query)}&ia=web"
+        page = await context.new_page()
+
+        await page.goto(url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        await page.wait_for_selector("button#more-results", timeout=15000)
+        await page.click("button#more-results")
+
+        await page.wait_for_selector("li[data-layout='organic']", timeout=15000)
+
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await asyncio.sleep(4)
+
+        html = await page.content()
+
+        return html
+
+    @override
+    async def parse(self, raw: str) -> DuckDuckGoSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        pages: list[Page] = []
+
+        article_tags = soup.find_all("article", {"data-testid": "result"})
+
+        for article_tag in article_tags:
+            site_tag = article_tag.find("p", class_="fOCEb2mA3YZTJXXjpgdS")
+            if not site_tag:
+                continue
+            site = site_tag.get_text(strip=True)
+
+            url_tag = article_tag.find("a", {"data-testid": "result-title-a"})
+            if not url_tag:
+                continue
+            url = url_tag.get("href")
+            if not isinstance(url, str):
+                continue
+
+            title_tag = article_tag.find("span", class_="EKtkFWMYpwzMKOYr0GYm")
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            description_tag = article_tag.find("span", class_="kY2IgmnCmOGjharHErah")
+            if not description_tag:
+                continue
+            description = description_tag.get_text(strip=True)
+
+            page = Page(
+                site=site,
+                url=url,
+                title=title,
+                description=description,
+            )
+            pages.append(page)
+
+        return DuckDuckGoSearchResponse(pages=pages)

webquest/scrapers/google_news_search/__init__.py

@@ -0,0 +1,11 @@
+from webquest.scrapers.google_news_search.scraper import (
+    GoogleNewsSearch,
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+
+__all__ = [
+    "GoogleNewsSearch",
+    "GoogleNewsSearchRequest",
+    "GoogleNewsSearchResponse",
+]

webquest/scrapers/google_news_search/schemas.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+
+class GoogleNewsSearchRequest(BaseModel):
+    query: str
+
+
+class Article(BaseModel):
+    site: str
+    url: str
+    title: str
+    published_at: str
+
+
+class GoogleNewsSearchResponse(BaseModel):
+    articles: list[Article]

webquest/scrapers/google_news_search/scraper.py

@@ -0,0 +1,78 @@
+import asyncio
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.scrapers.google_news_search.schemas import (
+    Article,
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+from webquest.scrapers.scraper import Scraper
+
+
+class GoogleNewsSearch(Scraper[GoogleNewsSearchRequest, str, GoogleNewsSearchResponse]):
+    """Scraper to perform a Google News search and parse the results."""
+
+    request = GoogleNewsSearchRequest
+    response = GoogleNewsSearchResponse
+
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: GoogleNewsSearchRequest,
+    ) -> str:
+        url = f"https://news.google.com/search?q={quote_plus(request.query)}"
+        page = await context.new_page()
+
+        await page.goto(url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        html = await page.content()
+
+        return html
+
+    @override
+    async def parse(self, raw: str) -> GoogleNewsSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        articles: list[Article] = []
+
+        article_tags = soup.find_all("c-wiz")
+        for article_tag in article_tags:
+            title_tag = article_tag.find("a", class_="JtKRv")
+            if not title_tag:
+                continue
+            title = title_tag.get_text().strip()
+
+            url_tag = article_tag.find("a", class_="JtKRv")
+            if not url_tag:
+                continue
+            url = url_tag.get("href")
+            if not isinstance(url, str):
+                continue
+
+            url = f"https://news.google.com{url[1:]}"
+
+            site_tag = article_tag.find("div", class_="vr1PYe")
+            if not site_tag:
+                continue
+            site = site_tag.get_text().strip()
+
+            published_at_tag = article_tag.find("time")
+            if not published_at_tag:
+                continue
+            published_at = published_at_tag.get_text().strip()
+
+            article = Article(
+                site=site,
+                url=url,
+                title=title,
+                published_at=published_at,
+            )
+
+            articles.append(article)
+
+        return GoogleNewsSearchResponse(articles=articles)

webquest/scrapers/openai_parser.py

@@ -0,0 +1,58 @@
+from abc import ABC
+from typing import Generic, Type, TypeVar, override
+
+from bs4 import BeautifulSoup
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+
+from webquest.browsers.browser import Browser
+from webquest.scrapers.scraper import Scraper
+
+TRequest = TypeVar("TRequest", bound=BaseModel)
+TResponse = TypeVar("TResponse", bound=BaseModel)
+
+
+class OpenAIParser(
+    Generic[TRequest, TResponse],
+    Scraper[TRequest, str, TResponse],
+    ABC,
+):
+    """Abstract base class for OpenAI-based parsers."""
+
+    def __init__(
+        self,
+        browser: Browser,
+        response_type: Type[TResponse],
+        client: AsyncOpenAI | None = None,
+        model: str = "gpt-5-mini",
+        input: str = "Parse the following web content:\n",
+        character_limit: int = 20000,
+    ) -> None:
+        self._response_type = response_type
+        if client is None:
+            client = AsyncOpenAI()
+        self._client = client
+        self._model = model
+        self._character_limit = character_limit
+        self._input = input
+        super().__init__(browser=browser)
+
+    @override
+    async def parse(self, raw: str) -> TResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        text = soup.get_text(separator="\n", strip=True)
+
+        if len(text) > self._character_limit:
+            start = (len(text) - self._character_limit) // 2
+            end = start + self._character_limit
+            text = text[start:end]
+
+        response = await self._client.responses.parse(
+            input=f"{self._input}{text}",
+            text_format=self._response_type,
+            model=self._model,
+            reasoning={"effort": "minimal"},
+        )
+        if response.output_parsed is None:
+            raise ValueError("Failed to parse the response into the desired format.")
+        return response.output_parsed

webquest/scrapers/scraper.py

@@ -0,0 +1,116 @@
+import asyncio
+from abc import ABC, abstractmethod
+from typing import ClassVar, Generic, TypeVar, overload
+
+from playwright.async_api import BrowserContext
+from pydantic import BaseModel
+
+from webquest.browsers.browser import Browser
+
+TRequest = TypeVar("TRequest", bound=BaseModel)
+TRaw = TypeVar("TRaw")
+TResponse = TypeVar("TResponse", bound=BaseModel)
+
+
+class Scraper(ABC, Generic[TRequest, TRaw, TResponse]):
+    """
+    Abstract base class for web scrapers.
+
+    This class defines the structure for a scraper, including fetching raw data
+    and parsing it into a structured response. It handles the execution flow
+    using a provided Browser instance.
+
+    Type Parameters:
+        TRequest: The type of the request object.
+        TRaw: The type of the raw data fetched from the browser.
+        TResponse: The type of the parsed response object.
+    """
+
+    request: ClassVar[type[TRequest]]
+    response: ClassVar[type[TResponse]]
+
+    def __init__(self, browser: Browser) -> None:
+        """
+        Initialize the Scraper.
+
+        Args:
+            browser (Browser): The browser instance to use for scraping.
+        """
+        self._browser = browser
+
+    @abstractmethod
+    async def fetch(self, context: BrowserContext, request: TRequest) -> TRaw:
+        """
+        Fetch raw data from the target website.
+
+        Args:
+            context (BrowserContext): The browser context to use.
+            request (TRequest): The request object containing parameters for the fetch operation.
+
+        Returns:
+            TRaw: The raw data fetched from the website.
+        """
+        ...
+
+    @abstractmethod
+    async def parse(self, raw: TRaw) -> TResponse:
+        """
+        Parse the raw data into a structured response.
+
+        Args:
+            raw (TRaw): The raw data returned by the fetch method.
+
+        Returns:
+            TResponse: The structured response object.
+        """
+        ...
+
+    @overload
+    async def run(self, request: TRequest, /) -> TResponse: ...
+
+    @overload
+    async def run(self, requests: list[TRequest], /) -> list[TResponse]: ...
+
+    @overload
+    async def run(self, *requests: TRequest) -> list[TResponse]: ...
+
+    async def run(
+        self,
+        *requests: TRequest | list[TRequest],
+    ) -> list[TResponse] | TResponse:
+        """
+        Run the scraper for one or more requests.
+
+        This method handles the browser context creation, concurrent fetching,
+        and parsing of results.
+
+        Args:
+            *requests: One or more request objects, or a list of request objects.
+
+        Returns:
+            list[TResponse] | TResponse: A single response if a single request was passed,
+            or a list of responses corresponding to the input requests.
+        """
+        normalized_requests: list[TRequest]
+        return_single = False
+
+        if len(requests) == 1 and isinstance(requests[0], list):
+            normalized_requests = requests[0]
+        else:
+            normalized_requests = []
+            for req in requests:
+                if isinstance(req, list):
+                    raise TypeError("Expected request object, got list")
+                normalized_requests.append(req)
+
+            if len(normalized_requests) == 1:
+                return_single = True
+
+        async with self._browser.get_context() as context:
+            raw_items = await asyncio.gather(
+                *[self.fetch(context, request) for request in normalized_requests]
+            )
+        responses = await asyncio.gather(*[self.parse(raw) for raw in raw_items])
+        if return_single:
+            return responses[0]
+        return responses

webquest/scrapers/youtube_search/__init__.py

@@ -0,0 +1,11 @@
+from webquest.scrapers.youtube_search.scraper import (
+    YouTubeSearch,
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+
+__all__ = [
+    "YouTubeSearch",
+    "YouTubeSearchRequest",
+    "YouTubeSearchResponse",
+]

webquest/scrapers/youtube_search/schemas.py

@@ -0,0 +1,51 @@
+from pydantic import BaseModel
+
+
+class Video(BaseModel):
+    id: str
+    url: str
+    title: str
+    description: str
+    published_at: str
+    views: str
+    channel_id: str
+    channel_url: str
+    channel_name: str
+
+
+class Channel(BaseModel):
+    id: str
+    url: str
+    name: str
+    description: str | None
+    subscribers: str
+
+
+class Post(BaseModel):
+    id: str
+    url: str
+    content: str
+    published_at: str
+    channel_id: str
+    channel_url: str
+    channel_name: str
+    comments: str
+    likes: str
+
+
+class Short(BaseModel):
+    id: str
+    url: str
+    title: str
+    views: str
+
+
+class YouTubeSearchRequest(BaseModel):
+    query: str
+
+
+class YouTubeSearchResponse(BaseModel):
+    videos: list[Video]
+    channels: list[Channel]
+    posts: list[Post]
+    shorts: list[Short]

webquest/scrapers/youtube_search/scraper.py

@@ -0,0 +1,303 @@
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.scrapers.scraper import Scraper
+from webquest.scrapers.youtube_search.schemas import (
+    Channel,
+    Post,
+    Short,
+    Video,
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+
+
+class YouTubeSearch(Scraper[YouTubeSearchRequest, str, YouTubeSearchResponse]):
+    """Scraper to perform a YouTube search and parse the results."""
+
+    request = YouTubeSearchRequest
+    response = YouTubeSearchResponse
+
+    def _parse_videos(self, soup: BeautifulSoup) -> list[Video]:
+        videos: list[Video] = []
+        video_tags = soup.find_all("ytd-video-renderer")
+
+        for video_tag in video_tags:
+            title_tag = video_tag.find(
+                "h3",
+                class_="title-and-badge style-scope ytd-video-renderer",
+            )
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            views_tag, published_at_tag = video_tag.find_all(
+                "span",
+                class_="inline-metadata-item style-scope ytd-video-meta-block",
+            )
+            views = views_tag.get_text(strip=True)
+            published_at = published_at_tag.get_text(strip=True)
+
+            description_tag = video_tag.find(
+                "yt-formatted-string",
+                class_="metadata-snippet-text style-scope ytd-video-renderer",
+            )
+            if not description_tag:
+                continue
+            description = description_tag.get_text(strip=True)
+
+            channel_name_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            channel_id_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get("href")
+            if not isinstance(channel_id, str):
+                continue
+            channel_id = channel_id[1:]
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            video_id_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope ytd-video-renderer",
+            )
+            if not video_id_tag:
+                continue
+            video_id = video_id_tag.get("href")
+            if not isinstance(video_id, str):
+                continue
+            video_id = video_id.split("v=")[-1].split("&")[0]
+
+            video_url = f"https://www.youtube.com/watch?v={video_id}"
+
+            video = Video(
+                id=video_id,
+                url=video_url,
+                title=title,
+                description=description,
+                published_at=published_at,
+                views=views,
+                channel_id=channel_id,
+                channel_url=channel_url,
+                channel_name=channel_name,
+            )
+            videos.append(video)
+
+        videos = [video for video in videos if len(video.id) == 11]
+
+        unique_videos = {video.id: video for video in videos}
+        videos = list(unique_videos.values())
+
+        return videos
+
+    def _parse_channels(self, soup: BeautifulSoup) -> list[Channel]:
+        channels: list[Channel] = []
+        channel_tags = soup.find_all("ytd-channel-renderer")
+        for channel_tag in channel_tags:
+            channel_name_tag = channel_tag.find(
+                "yt-formatted-string",
+                class_="style-scope ytd-channel-name",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            description_tag = channel_tag.find("yt-formatted-string", id="description")
+            if not description_tag:
+                continue
+            description: str | None = description_tag.get_text(strip=True)
+            if description == "":
+                description = None
+
+            channel_id_tag = channel_tag.find("yt-formatted-string", id="subscribers")
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get_text(strip=True)
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            subscribers_tag = channel_tag.find("span", id="video-count")
+            if not subscribers_tag:
+                continue
+            subscribers = subscribers_tag.get_text(strip=True)
+
+            channel = Channel(
+                id=channel_id,
+                url=channel_url,
+                name=channel_name,
+                description=description,
+                subscribers=subscribers,
+            )
+            channels.append(channel)
+        return channels
+
+    def _parse_posts(self, soup: BeautifulSoup) -> list[Post]:
+        posts: list[Post] = []
+        post_tags = soup.find_all("ytd-post-renderer")
+        for post_tag in post_tags:
+            content_tag = post_tag.find(
+                "div",
+                id="content",
+            )
+            if not content_tag:
+                continue
+            content = content_tag.get_text(strip=True)
+
+            channel_name_tag = post_tag.find(
+                "div",
+                id="author",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            published_at_tag = post_tag.find(
+                "yt-formatted-string",
+                id="published-time-text",
+            )
+            if not published_at_tag:
+                continue
+            published_at = published_at_tag.get_text(strip=True)
+
+            channel_id_tag = post_tag.find(
+                "a",
+                id="author-text",
+            )
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get("href")
+            if not isinstance(channel_id, str):
+                continue
+            channel_id = channel_id[1:]
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            post_id_tag = post_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not post_id_tag:
+                continue
+            post_id = post_id_tag.get("href")
+            if not isinstance(post_id, str):
+                continue
+            post_id = post_id.split("/post/")[-1]
+
+            post_url = f"https://www.youtube.com/post/{post_id}"
+
+            likes_tag = post_tag.find(
+                "span",
+                id="vote-count-middle",
+            )
+            if not likes_tag:
+                continue
+            likes = likes_tag.get_text(strip=True)
+
+            comments_tag = post_tag.find(
+                "div",
+                class_="yt-spec-button-shape-next__button-text-content",
+            )
+            if not comments_tag:
+                continue
+            comments = comments_tag.get_text(strip=True)
+
+            post = Post(
+                id=post_id,
+                url=post_url,
+                content=content,
+                published_at=published_at,
+                channel_id=channel_id,
+                channel_url=channel_url,
+                channel_name=channel_name,
+                comments=comments,
+                likes=likes,
+            )
+            posts.append(post)
+
+        return posts
+
+    def _parse_shorts(self, soup: BeautifulSoup) -> list[Short]:
+        shorts: list[Short] = []
+        short_tags = soup.find_all("ytm-shorts-lockup-view-model-v2")
+        for short_tag in short_tags:
+            title_tag = short_tag.find(
+                "h3",
+                role="presentation",
+            )
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            views_tag = short_tag.find(
+                "div",
+                class_="shortsLockupViewModelHostOutsideMetadataSubhead shortsLockupViewModelHostMetadataSubhead",
+            )
+            if not views_tag:
+                continue
+            views = views_tag.get_text(strip=True)
+
+            short_id_tag = short_tag.find(
+                "a",
+                class_="shortsLockupViewModelHostEndpoint shortsLockupViewModelHostOutsideMetadataEndpoint",
+            )
+            if not short_id_tag:
+                continue
+            short_id = short_id_tag.get("href")
+            if not isinstance(short_id, str):
+                continue
+            short_id = short_id.split("shorts/")[-1]
+
+            short_url = f"https://www.youtube.com/shorts/{short_id}"
+
+            short = Short(
+                id=short_id,
+                url=short_url,
+                title=title,
+                views=views,
+            )
+            shorts.append(short)
+        return shorts
+
+    def _parse_search_results(self, soup: BeautifulSoup) -> YouTubeSearchResponse:
+        videos = self._parse_videos(soup)
+        channels = self._parse_channels(soup)
+        posts = self._parse_posts(soup)
+        shorts = self._parse_shorts(soup)
+        return YouTubeSearchResponse(
+            videos=videos,
+            channels=channels,
+            posts=posts,
+            shorts=shorts,
+        )
+
+    @override
+    async def parse(self, raw: str) -> YouTubeSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        result = self._parse_search_results(soup)
+        return result
+
+    @override
+    async def fetch(
+        self, context: BrowserContext, request: YouTubeSearchRequest
+    ) -> str:
+        url = (
+            f"https://www.youtube.com/results?search_query={quote_plus(request.query)}"
+        )
+        page = await context.new_page()
+        await page.goto(url)
+        await page.wait_for_selector("ytd-video-renderer", timeout=10000)
+        html = await page.content()
+        return html

webquest/scrapers/youtube_transcript/__init__.py

@@ -0,0 +1,11 @@
+from webquest.scrapers.youtube_transcript.scraper import (
+    YouTubeTranscript,
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+
+__all__ = [
+    "YouTubeTranscript",
+    "YouTubeTranscriptRequest",
+    "YouTubeTranscriptResponse",
+]

webquest/scrapers/youtube_transcript/schemas.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+
+
+class YouTubeTranscriptRequest(BaseModel):
+    video_id: str
+
+
+class YouTubeTranscriptResponse(BaseModel):
+    transcript: str

webquest/scrapers/youtube_transcript/scraper.py

@@ -0,0 +1,84 @@
+import asyncio
+from typing import override
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.scrapers.scraper import Scraper
+from webquest.scrapers.youtube_transcript.schemas import (
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+
+
+class YouTubeTranscript(
+    Scraper[YouTubeTranscriptRequest, str, YouTubeTranscriptResponse]
+):
+    """Scraper to extract the transcript of a YouTube video."""
+
+    request = YouTubeTranscriptRequest
+    response = YouTubeTranscriptResponse
+
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: YouTubeTranscriptRequest,
+    ) -> str:
+        video_url = f"https://www.youtube.com/watch?v={request.video_id}"
+
+        page = await context.new_page()
+
+        await page.goto(video_url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        await page.wait_for_selector("div#description", timeout=10000)
+        await page.click("div#description")
+
+        await asyncio.sleep(0.5)
+
+        transcript_button = await page.wait_for_selector(
+            'button[aria-label="Show transcript"]', timeout=10000
+        )
+        if not transcript_button:
+            raise Exception("Transcript button not found")
+
+        await transcript_button.click()
+
+        await page.wait_for_selector(
+            "ytd-transcript-segment-list-renderer", timeout=10000
+        )
+
+        html = await page.content()
+        return html
+
+    @override
+    async def parse(self, raw: str) -> YouTubeTranscriptResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+
+        # Find the transcript segment list renderer
+        segment_renderer = soup.select_one("ytd-transcript-segment-list-renderer")
+        if not segment_renderer:
+            raise Exception("No transcript segments found")
+
+        # Find the segments container
+        segments_container = segment_renderer.select_one("div#segments-container")
+        if not segments_container:
+            raise Exception("No transcript segments found")
+
+        # Find all transcript segment renderers
+        segments = segments_container.select("ytd-transcript-segment-renderer")
+        if not segments:
+            raise Exception("No transcript segments found")
+
+        # Extract text from each segment
+        transcript_segments = []
+        for segment in segments:
+            text_element = segment.select_one("yt-formatted-string")
+            if text_element:
+                transcript_segments.append(text_element.get_text())
+
+        formatted_transcript = " ".join(transcript_segments).strip()
+        result = YouTubeTranscriptResponse(transcript=formatted_transcript)
+
+        return result

webquest-0.7.3.dist-info/METADATA

@@ -0,0 +1,97 @@
+Metadata-Version: 2.3
+Name: webquest
+Version: 0.7.3
+Summary: WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers.
+Requires-Dist: beautifulsoup4>=4.14.2
+Requires-Dist: hyperbrowser>=0.68.0
+Requires-Dist: openai>=2.6.0
+Requires-Dist: playwright>=1.55.0
+Requires-Dist: pydantic>=2.12.3
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# WebQuest
+
+WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers.
+
+**Scrapers**
+
+- **Any Article:** Extracts readable content from arbitrary web articles.
+- **DuckDuckGo Search:** General web search using DuckDuckGo.
+- **Google News Search:** News-focused search via Google News.
+- **YouTube Search:** Search YouTube videos, channels, posts, and shorts.
+- **YouTube Transcript:** Fetch transcripts for YouTube videos.
+
+**Browsers**
+
+- **Hyperbrowser:** A cloud-based browser service for running Playwright scrapers without managing infrastructure.
+
+## Installation
+
+Installing using pip:
+
+```bash
+pip install webquest
+```
+
+Installing using uv:
+
+```bash
+uv add webquest
+```
+
+## Usage
+
+Example usage of the DuckDuckGo Search scraper:
+
+```python
+import asyncio
+
+from webquest.browsers import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch
+
+
+async def main() -> None:
+    scraper = DuckDuckGoSearch(browser=Hyperbrowser())
+
+    response = await scraper.run(
+        scraper.request(query="Pizza Toppings"),
+    )
+    print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+You can also run multiple requests at the same time:
+
+```python
+import asyncio
+
+from webquest.browsers import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch
+
+
+async def main() -> None:
+    scraper = DuckDuckGoSearch(browser=Hyperbrowser())
+
+    responses = await scraper.run(
+        scraper.request(query="Pizza Toppings"),
+        scraper.request(query="AI News"),
+    )
+    for response in responses:
+        print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+> To use the Hyperbrowser browser, you need to set the `HYPERBROWSER_API_KEY` environment variable.
+
+> To use the Any Article scraper, you need to set the `OPENAI_API_KEY` environment variable.
+
+## Disclaimer
+
+This tool is for educational and research purposes only. The developers of WebQuest are not responsible for any misuse of this tool. Scraping websites may violate their Terms of Service. Users are solely responsible for ensuring their activities comply with all applicable laws and website policies.

webquest-0.7.3.dist-info/RECORD

@@ -0,0 +1,26 @@
+webquest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+webquest/browsers/__init__.py,sha256=zx78XH7Ys0xKrMaeV5Pkwx5q37GDagAOBIxted_K2Gk,141
+webquest/browsers/browser.py,sha256=XSojMYWfsUuxZypOi0g9ylhHlN7Fct9RTwrNyBCJIgs,674
+webquest/browsers/hyperbrowser.py,sha256=FRV3tAd6OcMlfnsGNt3vW-j4CEJPgqLi-6U_tUQVMCA,1750
+webquest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+webquest/scrapers/__init__.py,sha256=t7f7oy1R66mIfdw9fWDcpHJ8njHoCgOFUn_N2I145PI,1198
+webquest/scrapers/any_article/__init__.py,sha256=vrAqkwWHrRnnm5jcvSbOZDOzkw_HAJNmcbKbFF93tyg,201
+webquest/scrapers/any_article/schemas.py,sha256=qogFp13pIQy0u3aNHhDSqRJVZpApqCvbCi6tGls5tHw,217
+webquest/scrapers/any_article/scraper.py,sha256=hQBN5SWd1iucZ7-O_Nh1WnzTNUAPC62odwckaF3bKWA,1338
+webquest/scrapers/duckduckgo_search/__init__.py,sha256=9egcEqtLtxQSiyy8lg6e8_2uU4psuxp0nZf7lqhQ1IU,243
+webquest/scrapers/duckduckgo_search/schemas.py,sha256=YT1jJXs2j0_Lc1HHdgfX4N_88-XtrasqTeiqhfl9UpA,245
+webquest/scrapers/duckduckgo_search/scraper.py,sha256=DfNgvgRoCU0-ZbhRSxtBeEbUvoxC1DHADFmeMptqPic,2684
+webquest/scrapers/google_news_search/__init__.py,sha256=-NhQBRay_flLTBtHGwe-KQtY6AdWjw5QnMBOZlzJvBs,244
+webquest/scrapers/google_news_search/schemas.py,sha256=qY0U217bHZwtYvQ_f6FDKzNI_0wT9M-s6z0MtJm3qF4,255
+webquest/scrapers/google_news_search/scraper.py,sha256=2aRvoYszJMWN8CZ3TP-g1GWPsaQJuCaCjX4_oHNOoQc,2307
+webquest/scrapers/openai_parser.py,sha256=meK1nBFkDvF04QUEQC5NCTukH2_Mhs8swyL5EATAzK0,1849
+webquest/scrapers/scraper.py,sha256=jLzHQnvx9EsR-MohBEc1rtBmv9PqxliSHbqtu4vrmsE,3649
+webquest/scrapers/youtube_search/__init__.py,sha256=BvuHhLSZQ9N9pRW6ciKZjIcScecaFqS6BGzgaytwH3o,222
+webquest/scrapers/youtube_search/schemas.py,sha256=Lf4QOObvrYC3sHETS5KKlunci4yTS1GKD177-N9evKY,796
+webquest/scrapers/youtube_search/scraper.py,sha256=bGWlD2BWV9UKFRYptisKW-IP_4AZ5OSoJfbEN8Vq-Dk,10108
+webquest/scrapers/youtube_transcript/__init__.py,sha256=m1LyZcr-PGQqNRNDtWOVsuhIMIV-lG5YIivCyRENTiY,250
+webquest/scrapers/youtube_transcript/schemas.py,sha256=pIrq3XPbXERTTmaJwnUrgyu1LBfD8T-Vn7be088-Ss4,160
+webquest/scrapers/youtube_transcript/scraper.py,sha256=A9JjQhdOQZXAPw5NCzQaRc9yFpgUpRpfR2OdfEvdYYI,2723
+webquest-0.7.3.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+webquest-0.7.3.dist-info/METADATA,sha256=eYRZeyvC0hOKEaCwsS2-U6BXTki2tfoUiZ5ZPPinY_Q,2778
+webquest-0.7.3.dist-info/RECORD,,

webquest-0.7.3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.8.24
+Root-Is-Purelib: true
+Tag: py3-none-any