rusticai-playwright 0.0.1__tar.gz

rusticai_playwright-0.0.1/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.1
+Name: rusticai-playwright
+Version: 0.0.1
+Summary: Rustic AI module for scraping using playwright
+Home-page: https://www.rustic.ai/
+License: Apache-2.0
+Author: Dragonscale Industries Inc.
+Author-email: dev@dragonscale.ai
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: test
+Requires-Dist: install-playwright (>=0.1.1,<0.2.0)
+Requires-Dist: markdownify (>=1.1.0,<2.0.0)
+Requires-Dist: playwright (>=1.51.0,<2.0.0)
+Requires-Dist: rusticai-core (>=0.0.5,<0.1.0)
+Project-URL: Repository, https://github.com/rustic-ai/python-framework
+Project-URL: Rustic AI Core, https://pypi.org/project/rusticai-core/
+Description-Content-Type: text/markdown
+
+# Rustic AI Playwright
+
+This module employs [Playwright](https://playwright.dev/python/docs/intro) to facilitate web scraping tasks.
+
+## Installing
+
+```shell
+pip install rusticai-playwright
+```
+**Note:** It depends on [rusticai-core](https://pypi.org/project/rusticai-core/)
+
+## Building from Source
+
+```shell
+poetry install --with dev
+poetry build
+```
+

rusticai_playwright-0.0.1/README.md

@@ -0,0 +1,17 @@
+# Rustic AI Playwright
+
+This module employs [Playwright](https://playwright.dev/python/docs/intro) to facilitate web scraping tasks.
+
+## Installing
+
+```shell
+pip install rusticai-playwright
+```
+**Note:** It depends on [rusticai-core](https://pypi.org/project/rusticai-core/)
+
+## Building from Source
+
+```shell
+poetry install --with dev
+poetry build
+```

rusticai_playwright-0.0.1/pyproject.toml

@@ -0,0 +1,79 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "rusticai-playwright"
+version = "0.0.1"
+description = "Rustic AI module for scraping using playwright"
+authors = ["Dragonscale Industries Inc. <dev@dragonscale.ai>"]
+license = "Apache-2.0"
+readme = "README.md"
+homepage = "https://www.rustic.ai/"
+repository = "https://github.com/rustic-ai/python-framework"
+packages = [{ include = "rustic_ai", from = "src" }]
+
+[tool.poetry.urls]
+"Rustic AI Core" = "https://pypi.org/project/rusticai-core/"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+rusticai-core = {   version = "0.0.5"}
+playwright = "^1.51.0"
+install-playwright = "^0.1.1"
+markdownify = "^1.1.0"
+
+[tool.poetry-monorepo.deps]
+
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.4"
+pytest-asyncio = "^0.26.0"
+black = { version = "^24.2.0", extras = ["jupyter"] }
+flake8 = "^7.1.2"
+tox = "^4.24.1"
+isort = "^6.0.0"
+mypy = "^1.15.0"
+flaky = "^3.8.1"
+rusticai-testing = { version = "0.0.1"}
+
+[tool.poetry.extras]
+test = ["pytest", "pytest-asyncio", "flaky", "rusticai-testing"]
+
+[tool.black]
+line-length = 120
+target-version = ['py312']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.mypy]
+python_version = "3.12"
+ignore_missing_imports = true
+check_untyped_defs = true
+plugins = "pydantic.mypy"
+explicit_package_bases = true
+
+
+[tool.isort]
+profile = "black"
+known_third_party = ["pydantic"]
+known_first_party = ["rustic_ai.core","rustic_ai.playwright"]
+known_local_folder = ["rustic_ai.testing"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope="function"

rusticai_playwright-0.0.1/src/rustic_ai/playwright/agent.py

@@ -0,0 +1,135 @@
+import hashlib
+import logging
+import mimetypes
+import os
+from enum import StrEnum
+from typing import List
+from urllib.parse import urlsplit
+
+import shortuuid
+from install_playwright import install
+from markdownify import markdownify as md
+from pydantic import BaseModel, Field
+
+from playwright.async_api import async_playwright
+from rustic_ai.core.agents.commons.media import MediaLink
+from rustic_ai.core.agents.commons.message_formats import ErrorMessage
+from rustic_ai.core.guild import Agent, AgentSpec, agent
+from rustic_ai.core.guild.agent_ext.depends.filesystem.filesystem import FileSystem
+from rustic_ai.core.utils.json_utils import JsonDict
+
+
+class ScrapingOutputFormat(StrEnum):
+    TEXT_HTML = "text/html"
+    MARKDOWN = "text/markdown"
+
+
+class WebScrapingRequest(BaseModel):
+    id: str = Field(default_factory=shortuuid.uuid, title="ID of the request")
+    links: List[MediaLink] = Field(..., title="URL to scrape")
+    output_format: ScrapingOutputFormat = Field(
+        default=ScrapingOutputFormat.TEXT_HTML,
+        title="Output format of the scraped content",
+        description="The format in which the scraped content will be saved. Default is text/html.",
+    )
+
+    transformer_options: JsonDict = Field(
+        default={},
+        title="Options for the transformer",
+        description="Options for the transformer to be applied to the scraped content. Default is an empty dictionary.",
+    )
+
+
+class WebScrapingCompleted(BaseModel):
+    id: str = Field(..., title="ID of the request")
+    links: List[MediaLink] = Field(..., title="URL to scrape")
+
+
+class PlaywrightScraperAgent(Agent):
+    def __init__(self, agent_spec: AgentSpec):
+        super().__init__(agent_spec=agent_spec)
+
+    @agent.processor(
+        WebScrapingRequest, depends_on=[agent.AgentDependency(dependency_key="filesystem", guild_level=True)]
+    )
+    async def scrape(self, ctx: agent.ProcessContext[WebScrapingRequest], filesystem: FileSystem) -> None:
+        async with async_playwright() as p:
+            install(p.chromium)
+            browser = await p.chromium.launch()
+            page = await browser.new_page()
+
+            scraping_request = ctx.payload
+
+            scraped_docs: List[MediaLink] = []
+
+            for link in scraping_request.links:
+                url = link.url
+
+                response = await page.goto(url)
+
+                if response and response.status != 200:
+                    ctx.send_error(
+                        ErrorMessage(
+                            agent_type=self.get_qualified_class_name(),
+                            error_type="HTTP_ERROR_{response.status}",
+                            error_message=f"HTTP error: {response.status} for URL: {url}. Error message: {response.status_text}",
+                        )
+                    )
+                    continue
+
+                title = await page.title()
+                content = await page.content()
+                urls = urlsplit(url)
+                upath = urls.path
+                basename = os.path.basename(upath)
+                _, filetype = os.path.splitext(basename)
+
+                if not filetype and scraping_request.output_format == ScrapingOutputFormat.TEXT_HTML:
+                    filetype = ".html"
+                elif scraping_request.output_format == ScrapingOutputFormat.MARKDOWN:
+                    filetype = ".md"
+                elif not filetype:
+                    filetype = ".txt"
+
+                filename = hashlib.md5(content.encode("utf-8")).hexdigest() + filetype
+
+                mimetype = mimetypes.guess_type(filename)[0]
+
+                try:
+                    if scraping_request.output_format == ScrapingOutputFormat.MARKDOWN:
+                        content = md(
+                            content,
+                            **scraping_request.transformer_options,
+                        )
+                    with filesystem.open(f"scraped_data/{filename}", "w") as f:
+                        f.write(content)
+                except Exception as e:
+                    logging.error(f"Error writing file: {e}")
+                    ctx.send_error(
+                        ErrorMessage(
+                            agent_type=self.get_qualified_class_name(),
+                            error_type="FileWriteError",
+                            error_message=str(e),
+                        )
+                    )
+
+                meta = link.metadata or {}
+                metadata = meta | {"scraped_url": url, "title": title, "request_id": scraping_request.id}
+
+                output = MediaLink(
+                    url=f"scraped_data/{filename}",
+                    name=filename,
+                    metadata=metadata,
+                    on_filesystem=True,
+                    mimetype=mimetype,
+                    encoding="utf-8",
+                )
+
+                scraped_docs.append(output)
+                ctx.send(output)
+
+            await browser.close()
+
+            unique_scraped_docs = list({doc.name: doc for doc in scraped_docs}.values())
+
+            ctx.send(WebScrapingCompleted(id=scraping_request.id, links=unique_scraped_docs))