webquest 0.2.0__tar.gz

webquest-0.2.0/PKG-INFO

@@ -0,0 +1,96 @@
+Metadata-Version: 2.3
+Name: webquest
+Version: 0.2.0
+Summary: WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers.
+Requires-Dist: beautifulsoup4>=4.14.2
+Requires-Dist: hyperbrowser>=0.68.0
+Requires-Dist: openai>=2.6.0
+Requires-Dist: playwright>=1.55.0
+Requires-Dist: pydantic>=2.12.3
+Requires-Dist: pydantic-settings>=2.11.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# WebQuest
+
+WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers.
+
+Scrapers:
+
+- Any Article
+- DuckDuckGo Search
+- Google News Search
+- YouTube Search
+- YouTube Transcript
+
+Runners:
+
+- Hyperbrowser
+
+## Installation
+
+Installing using pip:
+
+```bash
+pip install webquest
+```
+
+Installing using uv:
+
+```bash
+uv add webquest
+```
+
+## Usage
+
+Example usage of the DuckDuckGo Search scraper:
+
+```python
+import asyncio
+
+from webquest.runners import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch, DuckDuckGoSearchRequest
+
+
+async def main() -> None:
+    runner = Hyperbrowser()
+    scraper = DuckDuckGoSearch()
+    response = await runner.run(
+        scraper,
+        DuckDuckGoSearchRequest(query="Pizza Toppings"),
+    )
+    print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+> To use the Hyperbrowser runner, you need to set the `HYPERBROWSER_API_KEY` environment variable.
+
+You can also run multiple requests at the same time:
+
+```python
+import asyncio
+
+from webquest.runners import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch, DuckDuckGoSearchRequest
+
+
+async def main() -> None:
+    runner = Hyperbrowser()
+    scraper = DuckDuckGoSearch()
+    responses = await runner.run_multiple(
+        scraper,
+        [
+            DuckDuckGoSearchRequest(query="Pizza Toppings"),
+            DuckDuckGoSearchRequest(query="AI News"),
+        ],
+    )
+    for response in responses:
+        print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```

webquest-0.2.0/README.md

@@ -0,0 +1,83 @@
+# WebQuest
+
+WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers.
+
+Scrapers:
+
+- Any Article
+- DuckDuckGo Search
+- Google News Search
+- YouTube Search
+- YouTube Transcript
+
+Runners:
+
+- Hyperbrowser
+
+## Installation
+
+Installing using pip:
+
+```bash
+pip install webquest
+```
+
+Installing using uv:
+
+```bash
+uv add webquest
+```
+
+## Usage
+
+Example usage of the DuckDuckGo Search scraper:
+
+```python
+import asyncio
+
+from webquest.runners import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch, DuckDuckGoSearchRequest
+
+
+async def main() -> None:
+    runner = Hyperbrowser()
+    scraper = DuckDuckGoSearch()
+    response = await runner.run(
+        scraper,
+        DuckDuckGoSearchRequest(query="Pizza Toppings"),
+    )
+    print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+> To use the Hyperbrowser runner, you need to set the `HYPERBROWSER_API_KEY` environment variable.
+
+You can also run multiple requests at the same time:
+
+```python
+import asyncio
+
+from webquest.runners import Hyperbrowser
+from webquest.scrapers import DuckDuckGoSearch, DuckDuckGoSearchRequest
+
+
+async def main() -> None:
+    runner = Hyperbrowser()
+    scraper = DuckDuckGoSearch()
+    responses = await runner.run_multiple(
+        scraper,
+        [
+            DuckDuckGoSearchRequest(query="Pizza Toppings"),
+            DuckDuckGoSearchRequest(query="AI News"),
+        ],
+    )
+    for response in responses:
+        print(response.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```

webquest-0.2.0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "webquest"
+version = "0.2.0"
+description = "WebQuest is an extensible Python toolkit for high-level web scraping, built around a generic Playwright-based scraper interface for quickly building, running, and reusing custom scrapers."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "beautifulsoup4>=4.14.2",
+    "hyperbrowser>=0.68.0",
+    "openai>=2.6.0",
+    "playwright>=1.55.0",
+    "pydantic>=2.12.3",
+    "pydantic-settings>=2.11.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.18,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.2",
+    "pytest-asyncio>=1.2.0",
+    "pytest-mypy>=1.0.1",
+    "ruff>=0.14.1",
+]
+
+[tool.pytest.ini_options]
+addopts = "--mypy"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+
+[[tool.mypy.overrides]]
+module = ["hyperbrowser.*"]
+ignore_missing_imports = true

webquest-0.2.0/src/webquest/base/__init__.py

@@ -0,0 +1,8 @@
+from webquest.base.base_scraper import BaseScraper
+from webquest.base.openai_parser import OpenAIParser, OpenAIParserSettings
+
+__all__ = [
+    "BaseScraper",
+    "OpenAIParser",
+    "OpenAIParserSettings",
+]

webquest-0.2.0/src/webquest/base/base_scraper.py

@@ -0,0 +1,22 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from playwright.async_api import BrowserContext
+from pydantic import BaseModel
+
+TRequest = TypeVar("TRequest", bound=BaseModel)
+TRaw = TypeVar("TRaw")
+TResponse = TypeVar("TResponse", bound=BaseModel)
+
+
+class BaseScraper(ABC, Generic[TRequest, TRaw, TResponse]):
+    @abstractmethod
+    async def fetch(self, context: BrowserContext, request: TRequest) -> TRaw: ...
+
+    @abstractmethod
+    async def parse(self, raw: TRaw) -> TResponse: ...
+
+    async def scrape(self, context: BrowserContext, request: TRequest) -> TResponse:
+        raw = await self.fetch(context, request)
+        response = await self.parse(raw)
+        return response

webquest-0.2.0/src/webquest/base/openai_parser.py

@@ -0,0 +1,66 @@
+from abc import ABC
+from typing import Generic, Type, TypeVar, override
+
+from bs4 import BeautifulSoup
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from webquest.base.base_scraper import BaseScraper
+
+TRequest = TypeVar("TRequest", bound=BaseModel)
+TResponse = TypeVar("TResponse", bound=BaseModel)
+
+
+class OpenAIParserSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        extra="ignore",
+    )
+    openai_api_key: str | None = None
+
+
+class OpenAIParser(
+    Generic[TRequest, TResponse],
+    BaseScraper[TRequest, str, TResponse],
+    ABC,
+):
+    def __init__(
+        self,
+        response_type: Type[TResponse],
+        openai: AsyncOpenAI | None = None,
+        settings: OpenAIParserSettings | None = None,
+        model: str = "gpt-5-mini",
+        input: str | None = None,
+        character_limit: int = 20000,
+    ) -> None:
+        self._response_type = response_type
+        if settings is None:
+            settings = OpenAIParserSettings()
+        self._settings = settings
+        if openai is None:
+            openai = AsyncOpenAI(api_key=self._settings.openai_api_key)
+        self._openai = openai
+        self._model = model
+        self._character_limit = character_limit
+        self._input = input or ""
+
+    @override
+    async def parse(self, raw: str) -> TResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        text = soup.get_text(separator="\n", strip=True)
+
+        if len(text) > self._character_limit:
+            start = (len(text) - self._character_limit) // 2
+            end = start + self._character_limit
+            text = text[start:end]
+
+        response = await self._openai.responses.parse(
+            input=f"{self._input}{text}",
+            text_format=self._response_type,
+            model=self._model,
+            reasoning={"effort": "minimal"},
+        )
+        if response.output_parsed is None:
+            raise ValueError("Failed to parse the response into the desired format.")
+        return response.output_parsed

webquest-0.2.0/src/webquest/runners/__init__.py

@@ -0,0 +1,3 @@
+from webquest.runners.hyperbrowser import Hyperbrowser, HyperbrowserSettings
+
+__all__ = ["Hyperbrowser", "HyperbrowserSettings"]

webquest-0.2.0/src/webquest/runners/hyperbrowser.py

@@ -0,0 +1,60 @@
+import asyncio
+from typing import TypeVar
+
+from hyperbrowser import AsyncHyperbrowser
+from playwright.async_api import async_playwright
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from webquest.base.base_scraper import BaseScraper
+
+TRequest = TypeVar("TRequest", bound=BaseModel)
+TRaw = TypeVar("TRaw")
+TResponse = TypeVar("TResponse", bound=BaseModel)
+
+
+class HyperbrowserSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        extra="ignore",
+    )
+    hyperbrowser_api_key: str | None = None
+
+
+class Hyperbrowser:
+    def __init__(
+        self,
+        settings: HyperbrowserSettings | None = None,
+        hyperbrowser_client: AsyncHyperbrowser | None = None,
+    ):
+        self._settings = settings or HyperbrowserSettings()
+        self._hyperbrowser_client = hyperbrowser_client or AsyncHyperbrowser(
+            api_key=self._settings.hyperbrowser_api_key,
+        )
+
+    async def run_multiple(
+        self,
+        scraper: BaseScraper[TRequest, TRaw, TResponse],
+        requests: list[TRequest],
+    ) -> list[TResponse]:
+        session = await self._hyperbrowser_client.sessions.create()
+        async with async_playwright() as p:
+            browser = await p.chromium.connect_over_cdp(session.ws_endpoint)
+            context = browser.contexts[0]
+            raw_items = await asyncio.gather(
+                *[scraper.fetch(context, request) for request in requests]
+            )
+        await self._hyperbrowser_client.sessions.stop(session.id)
+
+        responses = await asyncio.gather(
+            *[scraper.parse(raw_item) for raw_item in raw_items]
+        )
+        return responses
+
+    async def run(
+        self,
+        scraper: BaseScraper[TRequest, TRaw, TResponse],
+        request: TRequest,
+    ) -> TResponse:
+        responses = await self.run_multiple(scraper, [request])
+        return responses[0]

webquest-0.2.0/src/webquest/scrapers/__init__.py

@@ -0,0 +1,43 @@
+from webquest.scrapers.any_article import (
+    AnyArticle,
+    AnyArticleRequest,
+    AnyArticleResponse,
+)
+from webquest.scrapers.duckduckgo_search import (
+    DuckDuckGoSearch,
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+)
+from webquest.scrapers.google_news_search import (
+    GoogleNewsSearch,
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+from webquest.scrapers.youtube_search import (
+    YouTubeSearch,
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+from webquest.scrapers.youtube_transcript import (
+    YouTubeTranscript,
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+
+__all__ = [
+    "AnyArticle",
+    "AnyArticleRequest",
+    "AnyArticleResponse",
+    "DuckDuckGoSearch",
+    "DuckDuckGoSearchRequest",
+    "DuckDuckGoSearchResponse",
+    "GoogleNewsSearch",
+    "GoogleNewsSearchRequest",
+    "GoogleNewsSearchResponse",
+    "YouTubeSearch",
+    "YouTubeSearchRequest",
+    "YouTubeSearchResponse",
+    "YouTubeTranscript",
+    "YouTubeTranscriptRequest",
+    "YouTubeTranscriptResponse",
+]

webquest-0.2.0/src/webquest/scrapers/any_article/__init__.py

@@ -0,0 +1,4 @@
+from webquest.scrapers.any_article.schemas import AnyArticleRequest, AnyArticleResponse
+from webquest.scrapers.any_article.scraper import AnyArticle
+
+__all__ = ["AnyArticleRequest", "AnyArticleResponse", "AnyArticle"]

webquest-0.2.0/src/webquest/scrapers/any_article/schemas.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class AnyArticleRequest(BaseModel):
+    url: str
+
+
+class AnyArticleResponse(BaseModel):
+    publisher: str
+    title: str
+    published_at: str
+    authors: list[str]
+    content: str

webquest-0.2.0/src/webquest/scrapers/any_article/scraper.py

@@ -0,0 +1,35 @@
+from typing import override
+
+from openai import AsyncOpenAI
+from playwright.async_api import BrowserContext
+
+from webquest.base.openai_parser import OpenAIParser, OpenAIParserSettings
+from webquest.scrapers.any_article.schemas import AnyArticleRequest, AnyArticleResponse
+
+
+class AnyArticle(OpenAIParser[AnyArticleRequest, AnyArticleResponse]):
+    def __init__(
+        self,
+        openai: AsyncOpenAI | None = None,
+        settings: OpenAIParserSettings | None = None,
+        model: str = "gpt-5-mini",
+    ) -> None:
+        super().__init__(
+            response_type=AnyArticleResponse,
+            openai=openai,
+            settings=settings,
+            model=model,
+            input="Parse the following web page and extract the main article:\n\n",
+        )
+
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: AnyArticleRequest,
+    ) -> str:
+        page = await context.new_page()
+        await page.goto(request.url, wait_until="domcontentloaded")
+        await page.wait_for_timeout(3000)
+        html = await page.content()
+        return html

webquest-0.2.0/src/webquest/scrapers/duckduckgo_search/__init__.py

@@ -0,0 +1,7 @@
+from webquest.scrapers.duckduckgo_search.schemas import (
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+)
+from webquest.scrapers.duckduckgo_search.scraper import DuckDuckGoSearch
+
+__all__ = ["DuckDuckGoSearchRequest", "DuckDuckGoSearchResponse", "DuckDuckGoSearch"]

webquest-0.2.0/src/webquest/scrapers/duckduckgo_search/schemas.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+
+class DuckDuckGoSearchRequest(BaseModel):
+    query: str
+
+
+class Page(BaseModel):
+    site: str
+    url: str
+    title: str
+    description: str
+
+
+class DuckDuckGoSearchResponse(BaseModel):
+    pages: list[Page]

webquest-0.2.0/src/webquest/scrapers/duckduckgo_search/scraper.py

@@ -0,0 +1,81 @@
+import asyncio
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.base.base_scraper import BaseScraper
+from webquest.scrapers.duckduckgo_search.schemas import (
+    DuckDuckGoSearchRequest,
+    DuckDuckGoSearchResponse,
+    Page,
+)
+
+
+class DuckDuckGoSearch(
+    BaseScraper[DuckDuckGoSearchRequest, str, DuckDuckGoSearchResponse]
+):
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: DuckDuckGoSearchRequest,
+    ) -> str:
+        url = f"https://duckduckgo.com/?origin=funnel_home_website&t=h_&q={quote_plus(request.query)}&ia=web"
+        page = await context.new_page()
+
+        await page.goto(url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        await page.wait_for_selector("button#more-results", timeout=15000)
+        await page.click("button#more-results")
+
+        await page.wait_for_selector("li[data-layout='organic']", timeout=15000)
+
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await asyncio.sleep(4)
+
+        html = await page.content()
+
+        return html
+
+    @override
+    async def parse(self, raw: str) -> DuckDuckGoSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        pages: list[Page] = []
+
+        article_tags = soup.find_all("article", {"data-testid": "result"})
+
+        for article_tag in article_tags:
+            site_tag = article_tag.find("p", class_="fOCEb2mA3YZTJXXjpgdS")
+            if not site_tag:
+                continue
+            site = site_tag.get_text(strip=True)
+
+            url_tag = article_tag.find("a", {"data-testid": "result-title-a"})
+            if not url_tag:
+                continue
+            url = url_tag.get("href")
+            if not isinstance(url, str):
+                continue
+
+            title_tag = article_tag.find("span", class_="EKtkFWMYpwzMKOYr0GYm")
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            description_tag = article_tag.find("span", class_="kY2IgmnCmOGjharHErah")
+            if not description_tag:
+                continue
+            description = description_tag.get_text(strip=True)
+
+            page = Page(
+                site=site,
+                url=url,
+                title=title,
+                description=description,
+            )
+            pages.append(page)
+
+        return DuckDuckGoSearchResponse(pages=pages)

webquest-0.2.0/src/webquest/scrapers/google_news_search/__init__.py

@@ -0,0 +1,7 @@
+from webquest.scrapers.google_news_search.schemas import (
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+from webquest.scrapers.google_news_search.scraper import GoogleNewsSearch
+
+__all__ = ["GoogleNewsSearch", "GoogleNewsSearchRequest", "GoogleNewsSearchResponse"]

webquest-0.2.0/src/webquest/scrapers/google_news_search/schemas.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+
+
+class GoogleNewsSearchRequest(BaseModel):
+    query: str
+    locale: str | None = None
+
+
+class Article(BaseModel):
+    site: str
+    url: str
+    title: str
+    published_at: str
+
+
+class GoogleNewsSearchResponse(BaseModel):
+    articles: list[Article]

webquest-0.2.0/src/webquest/scrapers/google_news_search/scraper.py

@@ -0,0 +1,75 @@
+import asyncio
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.base.base_scraper import BaseScraper
+from webquest.scrapers.google_news_search.schemas import (
+    Article,
+    GoogleNewsSearchRequest,
+    GoogleNewsSearchResponse,
+)
+
+
+class GoogleNewsSearch(
+    BaseScraper[GoogleNewsSearchRequest, str, GoogleNewsSearchResponse]
+):
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: GoogleNewsSearchRequest,
+    ) -> str:
+        url = f"https://news.google.com/search?q={quote_plus(request.query)}"
+        page = await context.new_page()
+
+        await page.goto(url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        html = await page.content()
+
+        return html
+
+    @override
+    async def parse(self, raw: str) -> GoogleNewsSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        articles: list[Article] = []
+
+        article_tags = soup.find_all("article")
+        for article_tag in article_tags:
+            title_tag = article_tag.find("a", class_="JtKRv")
+            if not title_tag:
+                continue
+            title = title_tag.get_text().strip()
+
+            url_tag = article_tag.find("a", class_="JtKRv")
+            if not url_tag:
+                continue
+            url = url_tag.get("href")
+            if not isinstance(url, str):
+                continue
+
+            url = f"https://news.google.com{url[1:]}"
+
+            site_tag = article_tag.find("div", class_="vr1PYe")
+            if not site_tag:
+                continue
+            site = site_tag.get_text().strip()
+
+            published_at_tag = article_tag.find("time")
+            if not published_at_tag:
+                continue
+            published_at = published_at_tag.get_text().strip()
+
+            article = Article(
+                site=site,
+                url=url,
+                title=title,
+                published_at=published_at,
+            )
+
+            articles.append(article)
+
+        return GoogleNewsSearchResponse(articles=articles)

webquest-0.2.0/src/webquest/scrapers/youtube_search/__init__.py

@@ -0,0 +1,7 @@
+from webquest.scrapers.youtube_search.schemas import (
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+from webquest.scrapers.youtube_search.scraper import YouTubeSearch
+
+__all__ = ["YouTubeSearchRequest", "YouTubeSearchResponse", "YouTubeSearch"]

webquest-0.2.0/src/webquest/scrapers/youtube_search/schemas.py

@@ -0,0 +1,51 @@
+from pydantic import BaseModel
+
+
+class Video(BaseModel):
+    id: str
+    url: str
+    title: str
+    description: str
+    published_at: str
+    views: str
+    channel_id: str
+    channel_url: str
+    channel_name: str
+
+
+class Channel(BaseModel):
+    id: str
+    url: str
+    name: str
+    description: str | None
+    subscribers: str
+
+
+class Post(BaseModel):
+    id: str
+    url: str
+    content: str
+    published_at: str
+    channel_id: str
+    channel_url: str
+    channel_name: str
+    comments: str
+    likes: str
+
+
+class Short(BaseModel):
+    id: str
+    url: str
+    title: str
+    views: str
+
+
+class YouTubeSearchRequest(BaseModel):
+    query: str
+
+
+class YouTubeSearchResponse(BaseModel):
+    videos: list[Video]
+    channels: list[Channel]
+    posts: list[Post]
+    shorts: list[Short]

webquest-0.2.0/src/webquest/scrapers/youtube_search/scraper.py

@@ -0,0 +1,298 @@
+from typing import override
+from urllib.parse import quote_plus
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.base.base_scraper import BaseScraper
+from webquest.scrapers.youtube_search.schemas import (
+    Channel,
+    Post,
+    Short,
+    Video,
+    YouTubeSearchRequest,
+    YouTubeSearchResponse,
+)
+
+
+class YouTubeSearch(BaseScraper[YouTubeSearchRequest, str, YouTubeSearchResponse]):
+    def _parse_videos(self, soup: BeautifulSoup) -> list[Video]:
+        videos: list[Video] = []
+        video_tags = soup.find_all("ytd-video-renderer")
+
+        for video_tag in video_tags:
+            title_tag = video_tag.find(
+                "h3",
+                class_="title-and-badge style-scope ytd-video-renderer",
+            )
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            views_tag, published_at_tag = video_tag.find_all(
+                "span",
+                class_="inline-metadata-item style-scope ytd-video-meta-block",
+            )
+            views = views_tag.get_text(strip=True)
+            published_at = published_at_tag.get_text(strip=True)
+
+            description_tag = video_tag.find(
+                "yt-formatted-string",
+                class_="metadata-snippet-text style-scope ytd-video-renderer",
+            )
+            if not description_tag:
+                continue
+            description = description_tag.get_text(strip=True)
+
+            channel_name_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            channel_id_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get("href")
+            if not isinstance(channel_id, str):
+                continue
+            channel_id = channel_id[1:]
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            video_id_tag = video_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope ytd-video-renderer",
+            )
+            if not video_id_tag:
+                continue
+            video_id = video_id_tag.get("href")
+            if not isinstance(video_id, str):
+                continue
+            video_id = video_id.split("v=")[-1].split("&")[0]
+
+            video_url = f"https://www.youtube.com/watch?v={video_id}"
+
+            video = Video(
+                id=video_id,
+                url=video_url,
+                title=title,
+                description=description,
+                published_at=published_at,
+                views=views,
+                channel_id=channel_id,
+                channel_url=channel_url,
+                channel_name=channel_name,
+            )
+            videos.append(video)
+
+        videos = [video for video in videos if len(video.id) == 11]
+
+        unique_videos = {video.id: video for video in videos}
+        videos = list(unique_videos.values())
+
+        return videos
+
+    def _parse_channels(self, soup: BeautifulSoup) -> list[Channel]:
+        channels: list[Channel] = []
+        channel_tags = soup.find_all("ytd-channel-renderer")
+        for channel_tag in channel_tags:
+            channel_name_tag = channel_tag.find(
+                "yt-formatted-string",
+                class_="style-scope ytd-channel-name",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            description_tag = channel_tag.find("yt-formatted-string", id="description")
+            if not description_tag:
+                continue
+            description: str | None = description_tag.get_text(strip=True)
+            if description == "":
+                description = None
+
+            channel_id_tag = channel_tag.find("yt-formatted-string", id="subscribers")
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get_text(strip=True)
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            subscribers_tag = channel_tag.find("span", id="video-count")
+            if not subscribers_tag:
+                continue
+            subscribers = subscribers_tag.get_text(strip=True)
+
+            channel = Channel(
+                id=channel_id,
+                url=channel_url,
+                name=channel_name,
+                description=description,
+                subscribers=subscribers,
+            )
+            channels.append(channel)
+        return channels
+
+    def _parse_posts(self, soup: BeautifulSoup) -> list[Post]:
+        posts: list[Post] = []
+        post_tags = soup.find_all("ytd-post-renderer")
+        for post_tag in post_tags:
+            content_tag = post_tag.find(
+                "div",
+                id="content",
+            )
+            if not content_tag:
+                continue
+            content = content_tag.get_text(strip=True)
+
+            channel_name_tag = post_tag.find(
+                "div",
+                id="author",
+            )
+            if not channel_name_tag:
+                continue
+            channel_name = channel_name_tag.get_text(strip=True)
+
+            published_at_tag = post_tag.find(
+                "yt-formatted-string",
+                id="published-time-text",
+            )
+            if not published_at_tag:
+                continue
+            published_at = published_at_tag.get_text(strip=True)
+
+            channel_id_tag = post_tag.find(
+                "a",
+                id="author-text",
+            )
+            if not channel_id_tag:
+                continue
+            channel_id = channel_id_tag.get("href")
+            if not isinstance(channel_id, str):
+                continue
+            channel_id = channel_id[1:]
+
+            channel_url = f"https://www.youtube.com/{channel_id}"
+
+            post_id_tag = post_tag.find(
+                "a",
+                class_="yt-simple-endpoint style-scope yt-formatted-string",
+            )
+            if not post_id_tag:
+                continue
+            post_id = post_id_tag.get("href")
+            if not isinstance(post_id, str):
+                continue
+            post_id = post_id.split("/post/")[-1]
+
+            post_url = f"https://www.youtube.com/post/{post_id}"
+
+            likes_tag = post_tag.find(
+                "span",
+                id="vote-count-middle",
+            )
+            if not likes_tag:
+                continue
+            likes = likes_tag.get_text(strip=True)
+
+            comments_tag = post_tag.find(
+                "div",
+                class_="yt-spec-button-shape-next__button-text-content",
+            )
+            if not comments_tag:
+                continue
+            comments = comments_tag.get_text(strip=True)
+
+            post = Post(
+                id=post_id,
+                url=post_url,
+                content=content,
+                published_at=published_at,
+                channel_id=channel_id,
+                channel_url=channel_url,
+                channel_name=channel_name,
+                comments=comments,
+                likes=likes,
+            )
+            posts.append(post)
+
+        return posts
+
+    def _parse_shorts(self, soup: BeautifulSoup) -> list[Short]:
+        shorts: list[Short] = []
+        short_tags = soup.find_all("ytm-shorts-lockup-view-model-v2")
+        for short_tag in short_tags:
+            title_tag = short_tag.find(
+                "h3",
+                role="presentation",
+            )
+            if not title_tag:
+                continue
+            title = title_tag.get_text(strip=True)
+
+            views_tag = short_tag.find(
+                "div",
+                class_="shortsLockupViewModelHostOutsideMetadataSubhead shortsLockupViewModelHostMetadataSubhead",
+            )
+            if not views_tag:
+                continue
+            views = views_tag.get_text(strip=True)
+
+            short_id_tag = short_tag.find(
+                "a",
+                class_="shortsLockupViewModelHostEndpoint shortsLockupViewModelHostOutsideMetadataEndpoint",
+            )
+            if not short_id_tag:
+                continue
+            short_id = short_id_tag.get("href")
+            if not isinstance(short_id, str):
+                continue
+            short_id = short_id.split("shorts/")[-1]
+
+            short_url = f"https://www.youtube.com/shorts/{short_id}"
+
+            short = Short(
+                id=short_id,
+                url=short_url,
+                title=title,
+                views=views,
+            )
+            shorts.append(short)
+        return shorts
+
+    def _parse_search_results(self, soup: BeautifulSoup) -> YouTubeSearchResponse:
+        videos = self._parse_videos(soup)
+        channels = self._parse_channels(soup)
+        posts = self._parse_posts(soup)
+        shorts = self._parse_shorts(soup)
+        return YouTubeSearchResponse(
+            videos=videos,
+            channels=channels,
+            posts=posts,
+            shorts=shorts,
+        )
+
+    @override
+    async def parse(self, raw: str) -> YouTubeSearchResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+        result = self._parse_search_results(soup)
+        return result
+
+    @override
+    async def fetch(
+        self, context: BrowserContext, request: YouTubeSearchRequest
+    ) -> str:
+        url = (
+            f"https://www.youtube.com/results?search_query={quote_plus(request.query)}"
+        )
+        page = await context.new_page()
+        await page.goto(url)
+        await page.wait_for_selector("ytd-video-renderer", timeout=10000)
+        html = await page.content()
+        return html

webquest-0.2.0/src/webquest/scrapers/youtube_transcript/__init__.py

@@ -0,0 +1,7 @@
+from webquest.scrapers.youtube_transcript.schemas import (
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+from webquest.scrapers.youtube_transcript.scraper import YouTubeTranscript
+
+__all__ = ["YouTubeTranscriptRequest", "YouTubeTranscriptResponse", "YouTubeTranscript"]

webquest-0.2.0/src/webquest/scrapers/youtube_transcript/schemas.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+
+
+class YouTubeTranscriptRequest(BaseModel):
+    video_id: str
+
+
+class YouTubeTranscriptResponse(BaseModel):
+    transcript: str

webquest-0.2.0/src/webquest/scrapers/youtube_transcript/scraper.py

@@ -0,0 +1,79 @@
+import asyncio
+from typing import override
+
+from bs4 import BeautifulSoup
+from playwright.async_api import BrowserContext
+
+from webquest.base.base_scraper import BaseScraper
+from webquest.scrapers.youtube_transcript.schemas import (
+    YouTubeTranscriptRequest,
+    YouTubeTranscriptResponse,
+)
+
+
+class YouTubeTranscript(
+    BaseScraper[YouTubeTranscriptRequest, str, YouTubeTranscriptResponse]
+):
+    @override
+    async def fetch(
+        self,
+        context: BrowserContext,
+        request: YouTubeTranscriptRequest,
+    ) -> str:
+        video_url = f"https://www.youtube.com/watch?v={request.video_id}"
+
+        page = await context.new_page()
+
+        await page.goto(video_url, wait_until="networkidle", timeout=30000)
+        await asyncio.sleep(1)
+
+        await page.wait_for_selector("div#description", timeout=10000)
+        await page.click("div#description")
+
+        await asyncio.sleep(0.5)
+
+        transcript_button = await page.wait_for_selector(
+            'button[aria-label="Show transcript"]', timeout=10000
+        )
+        if not transcript_button:
+            raise Exception("Transcript button not found")
+
+        await transcript_button.click()
+
+        await page.wait_for_selector(
+            "ytd-transcript-segment-list-renderer", timeout=10000
+        )
+
+        html = await page.content()
+        return html
+
+    @override
+    async def parse(self, raw: str) -> YouTubeTranscriptResponse:
+        soup = BeautifulSoup(raw, "html.parser")
+
+        # Find the transcript segment list renderer
+        segment_renderer = soup.select_one("ytd-transcript-segment-list-renderer")
+        if not segment_renderer:
+            raise Exception("No transcript segments found")
+
+        # Find the segments container
+        segments_container = segment_renderer.select_one("div#segments-container")
+        if not segments_container:
+            raise Exception("No transcript segments found")
+
+        # Find all transcript segment renderers
+        segments = segments_container.select("ytd-transcript-segment-renderer")
+        if not segments:
+            raise Exception("No transcript segments found")
+
+        # Extract text from each segment
+        transcript_segments = []
+        for segment in segments:
+            text_element = segment.select_one("yt-formatted-string")
+            if text_element:
+                transcript_segments.append(text_element.get_text())
+
+        formatted_transcript = " ".join(transcript_segments).strip()
+        result = YouTubeTranscriptResponse(transcript=formatted_transcript)
+
+        return result