webweavex 0.1.0__tar.gz

webweavex-0.1.0/.gitignore

@@ -0,0 +1,84 @@
+# OS
+.DS_Store
+Thumbs.db
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.egg-info/
+.venv/
+.venv-*/
+venv/
+.env
+
+# Node
+node_modules/
+.npm/
+.pnpm-store/
+.yarn/
+.yarn-cache/
+.yarnrc.yml
+pnpm-debug.log*
+npm-debug.log*
+
+# Java/Kotlin/Gradle
+.gradle/
+build/
+*.class
+*.jar
+*.war
+*.ear
+
+# Maven
+target/
+
+# Dart/Flutter
+.dart_tool/
+.flutter-plugins
+.flutter-plugins-dependencies
+.packages
+.pub-cache/
+build/
+
+# IDE
+.idea/
+.vscode/
+*.iml
+
+# Logs
+*.log
+
+# Coverage
+coverage/
+htmlcov/
+
+# Test artifacts
+.pytest_cache/
+
+# Terraform or infra
+*.tfstate
+*.tfstate.backup
+
+# Misc
+*.tmp
+*.swp
+.m2/
+.pkg-verify*/
+.tmp_dart_publish_check/
+
+# Runtime artifacts
+rag_dataset.jsonl
+knowledge_graph.graphml
+repo_dataset.jsonl
+repo_graph.graphml
+repo_summary.md
+.tmp_site/
+website/.docusaurus/
+website/.npm-cache/
+sdk/node/.npm-cache/
+sdk/node/*.tgz
+
+!sdk/dart/bin/
+!sdk/dart/lib/

webweavex-0.1.0/PKG-INFO

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: webweavex
+Version: 0.1.0
+Summary: AI-native web crawling platform
+Author: Piyush Mishra
+License: Apache-2.0
+Keywords: ai,crawler,data extraction,knowledge graph,rag,web scraping
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.8
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: certifi>=2024.9.0
+Requires-Dist: fastapi>=0.110.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: markdownify>=0.13.1
+Requires-Dist: networkx>=3.0
+Requires-Dist: playwright>=1.42.0
+Requires-Dist: pydantic>=2.6.0
+Requires-Dist: redis>=5.0
+Requires-Dist: spacy>=3.7.0
+Requires-Dist: uvicorn>=0.29.0
+Description-Content-Type: text/markdown
+
+# WebWeaveX 🚀
+
+> The Next-Gen AI-Powered Web Crawling Engine
+> Multi-language SDKs • Structured Data • Knowledge Graphs • RAG-ready
+
+---
+
+## 🌍 Why WebWeaveX?
+
+WebWeaveX is not just a crawler.
+
+It is a **data intelligence engine** designed for:
+
+* 🤖 AI / LLM pipelines (RAG datasets)
+* 📊 Structured web extraction
+* 🧠 Knowledge graph generation
+* ⚡ High-performance crawling
+* 🌐 Multi-language developer ecosystem
+
+---
+
+## ⚡ Features
+
+* 🔍 Smart crawling (HTML, metadata, links)
+* 🧠 AI-ready outputs (JSON, text, markdown)
+* 🌐 Multi-language SDKs:
+
+  * Python (pip)
+  * Node.js (npm)
+  * Dart (pub.dev)
+  * Java (Maven)
+  * Kotlin
+* ⚡ Async + high-performance engine
+* 🔐 SSL + secure crawling
+* 📦 CLI + API server
+
+---
+
+## 📦 Installation
+
+```bash
+pip install webweavex
+```
+
+---
+
+## 🚀 Quick Start
+
+```python
+from webweavex import AsyncWebWeaveX
+
+crawler = AsyncWebWeaveX()
+result = crawler.crawl("https://example.com")
+
+print(result["metadata"]["title"])
+```
+
+---
+
+## 🧪 CLI Usage
+
+```bash
+webweavex crawl https://example.com
+```
+
+---
+
+## 🧠 Output Example
+
+```json
+{
+  "url": "https://example.com",
+  "status": 200,
+  "metadata": {
+    "title": "Example Domain"
+  },
+  "links": [
+    {
+      "url": "https://iana.org/domains/example",
+      "text": "Learn more"
+    }
+  ]
+}
+```
+
+---
+
+## 🌐 Multi-Language SDKs
+
+WebWeaveX is built for **global developer adoption**:
+
+| Language | Package               |
+| -------- | --------------------- |
+| Python   | pip install webweavex |
+| Node     | npm install webweavex |
+| Dart     | pub.dev               |
+| Java     | Maven                 |
+| Kotlin   | Gradle                |
+
+---
+
+## ⚡ Benchmarks
+
+* ⚡ Fast async crawling
+* 📉 Low memory usage
+* 🔁 Concurrent processing
+* 🔍 Optimized parsing
+
+---
+
+## 🔐 Security
+
+* SSL verification enabled by default
+* Safe crawling practices
+* Configurable policies
+
+---
+
+## 🤝 Contributing
+
+We welcome contributions 🚀
+
+```bash
+git clone https://github.com/PIYUSH-MISHRA-00/WebWeaveX.git
+```
+
+---
+
+## 📜 License
+
+Apache License 2.0
+
+---
+
+## ⭐ Support
+
+If you like this project:
+
+👉 Star the repo
+👉 Share with developers
+👉 Use in production
+
+---
+
+## 🚀 Vision
+
+WebWeaveX is built for the future of:
+
+* AI Agents
+* Autonomous systems
+* Data intelligence platforms
+
+---

webweavex-0.1.0/README.md

@@ -0,0 +1,153 @@
+# WebWeaveX 🚀
+
+> The Next-Gen AI-Powered Web Crawling Engine
+> Multi-language SDKs • Structured Data • Knowledge Graphs • RAG-ready
+
+---
+
+## 🌍 Why WebWeaveX?
+
+WebWeaveX is not just a crawler.
+
+It is a **data intelligence engine** designed for:
+
+* 🤖 AI / LLM pipelines (RAG datasets)
+* 📊 Structured web extraction
+* 🧠 Knowledge graph generation
+* ⚡ High-performance crawling
+* 🌐 Multi-language developer ecosystem
+
+---
+
+## ⚡ Features
+
+* 🔍 Smart crawling (HTML, metadata, links)
+* 🧠 AI-ready outputs (JSON, text, markdown)
+* 🌐 Multi-language SDKs:
+
+  * Python (pip)
+  * Node.js (npm)
+  * Dart (pub.dev)
+  * Java (Maven)
+  * Kotlin
+* ⚡ Async + high-performance engine
+* 🔐 SSL + secure crawling
+* 📦 CLI + API server
+
+---
+
+## 📦 Installation
+
+```bash
+pip install webweavex
+```
+
+---
+
+## 🚀 Quick Start
+
+```python
+from webweavex import AsyncWebWeaveX
+
+crawler = AsyncWebWeaveX()
+result = crawler.crawl("https://example.com")
+
+print(result["metadata"]["title"])
+```
+
+---
+
+## 🧪 CLI Usage
+
+```bash
+webweavex crawl https://example.com
+```
+
+---
+
+## 🧠 Output Example
+
+```json
+{
+  "url": "https://example.com",
+  "status": 200,
+  "metadata": {
+    "title": "Example Domain"
+  },
+  "links": [
+    {
+      "url": "https://iana.org/domains/example",
+      "text": "Learn more"
+    }
+  ]
+}
+```
+
+---
+
+## 🌐 Multi-Language SDKs
+
+WebWeaveX is built for **global developer adoption**:
+
+| Language | Package               |
+| -------- | --------------------- |
+| Python   | pip install webweavex |
+| Node     | npm install webweavex |
+| Dart     | pub.dev               |
+| Java     | Maven                 |
+| Kotlin   | Gradle                |
+
+---
+
+## ⚡ Benchmarks
+
+* ⚡ Fast async crawling
+* 📉 Low memory usage
+* 🔁 Concurrent processing
+* 🔍 Optimized parsing
+
+---
+
+## 🔐 Security
+
+* SSL verification enabled by default
+* Safe crawling practices
+* Configurable policies
+
+---
+
+## 🤝 Contributing
+
+We welcome contributions 🚀
+
+```bash
+git clone https://github.com/PIYUSH-MISHRA-00/WebWeaveX.git
+```
+
+---
+
+## 📜 License
+
+Apache License 2.0
+
+---
+
+## ⭐ Support
+
+If you like this project:
+
+👉 Star the repo
+👉 Share with developers
+👉 Use in production
+
+---
+
+## 🚀 Vision
+
+WebWeaveX is built for the future of:
+
+* AI Agents
+* Autonomous systems
+* Data intelligence platforms
+
+---

webweavex-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["hatchling>=1.21.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "webweavex"
+version = "0.1.0"
+description = "AI-native web crawling platform"
+readme = { file = "README.md", content-type = "text/markdown" }
+requires-python = ">=3.8"
+license = { text = "Apache-2.0" }
+
+authors = [
+  { name = "Piyush Mishra" }
+]
+
+keywords = [
+  "crawler",
+  "web scraping",
+  "rag",
+  "knowledge graph",
+  "ai",
+  "data extraction"
+]
+
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+  "Intended Audience :: Developers",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries"
+]
+
+dependencies = [
+  "httpx>=0.25.0",
+  "beautifulsoup4>=4.12.0",
+  "markdownify>=0.13.1",
+  "playwright>=1.42.0",
+  "pydantic>=2.6.0",
+  "fastapi>=0.110.0",
+  "uvicorn>=0.29.0",
+  "spacy>=3.7.0",
+  "redis>=5.0",
+  "networkx>=3.0",
+  "certifi>=2024.9.0"
+]
+
+[project.scripts]
+webweavex = "webweavex.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["webweavex"]

webweavex-0.1.0/webweavex/__init__.py

@@ -0,0 +1,33 @@
+"""WebWeaveX core package."""
+
+import os
+import ssl
+import sys
+from pathlib import Path
+
+# Ensure CA certificates are available for HTTPS requests.
+# Some environments (including certain CI containers) may lack a default CA bundle.
+try:
+  import certifi
+  cafile = ssl.get_default_verify_paths().cafile
+  if not cafile or not os.path.exists(cafile):
+    os.environ.setdefault("SSL_CERT_FILE", certifi.where())
+except ImportError:  # pragma: no cover
+  pass
+
+ROOT = Path(__file__).resolve().parents[1]  # core
+sys.path.insert(0, str(ROOT.parent))  # root
+
+from .async_engine import AsyncWebWeaveX
+from .config import CrawlConfig
+from .engine import WebWeaveX
+from .models import Link, Metadata, PageResult
+
+__all__ = [
+  "AsyncWebWeaveX",
+  "CrawlConfig",
+  "WebWeaveX",
+  "Link",
+  "Metadata",
+  "PageResult",
+]

webweavex-0.1.0/webweavex/api_server.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+from .async_engine import AsyncWebWeaveX
+from .models import PageResult
+
+
+class UrlRequest(BaseModel):
+  url: str
+
+
+app = FastAPI(title="WebWeaveX API", version="0.1.0")
+_engine = AsyncWebWeaveX()
+
+
+@app.on_event("shutdown")
+async def _shutdown_engine() -> None:
+  await _engine.aclose()
+
+
+@app.post("/crawl", response_model=PageResult)
+async def crawl(request: UrlRequest) -> PageResult:
+  return await _engine.crawl(request.url)
+
+
+@app.post("/crawl_site", response_model=list[PageResult])
+async def crawl_site(request: UrlRequest) -> list[PageResult]:
+  return await _engine.crawl_site(request.url)
+
+
+@app.post("/rag_dataset")
+async def rag_dataset(request: UrlRequest) -> list[dict[str, object]]:
+  return await _engine.build_rag_dataset(request.url)
+
+
+@app.post("/knowledge_graph")
+async def knowledge_graph(request: UrlRequest) -> dict[str, list[dict[str, str]]]:
+  return await _engine.build_knowledge_graph(request.url)

webweavex-0.1.0/webweavex/async_crawler.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import asyncio
+
+from .async_fetcher import AsyncFetcher
+from .config import CrawlConfig
+from .crawler import parse_html
+from .js_renderer import JSRenderer
+from .logging import get_logger
+from .models import Metadata, PageResult
+from .robots import RobotsHandler
+
+logger = get_logger(__name__)
+
+
+class AsyncCrawler:
+  """Concurrent async crawler for multiple URLs."""
+
+  def __init__(
+    self,
+    fetcher: AsyncFetcher,
+    config: CrawlConfig,
+    robots: RobotsHandler | None = None,
+    renderer: JSRenderer | None = None,
+  ) -> None:
+    self._fetcher = fetcher
+    self._config = config
+    self._robots = robots
+    self._renderer = renderer
+    self._semaphore = asyncio.Semaphore(max(1, config.max_concurrency))
+
+  async def crawl(self, url: str) -> PageResult:
+    logger.info("Async crawling %s", url)
+    if self._robots:
+      allowed = await self._robots.allowed(url)
+      if not allowed:
+        logger.info("Robots blocked %s", url)
+        return PageResult(url=url, status=403, html=None, links=[], metadata=Metadata())
+
+    async with self._semaphore:
+      if self._config.enable_js:
+        if self._renderer is None:
+          logger.warning("JS rendering enabled but no renderer configured; falling back to fetcher")
+          status, html = await self._fetcher.fetch(url)
+        else:
+          logger.info("JS rendering used for %s", url)
+          html = await self._renderer.render(url)
+          status = 200
+      else:
+        status, html = await self._fetcher.fetch(url)
+    links, metadata = parse_html(html)
+    return PageResult(url=url, status=status, html=html, links=links, metadata=metadata)
+
+  async def crawl_many(self, urls: list[str]) -> list[PageResult]:
+    tasks = [asyncio.create_task(self.crawl(url)) for url in urls]
+    return await asyncio.gather(*tasks)

webweavex-0.1.0/webweavex/async_engine.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from .async_crawler import AsyncCrawler
+from .async_fetcher import AsyncFetcher
+from .config import CrawlConfig
+from .js_renderer import JSRenderer
+from .logging import get_logger
+from .models import PageResult
+from .robots import RobotsHandler
+
+logger = get_logger(__name__)
+
+
+class AsyncWebWeaveX:
+  """Async entry point for concurrent crawling."""
+
+  def __init__(
+    self,
+    config: CrawlConfig | None = None,
+    fetcher: AsyncFetcher | None = None,
+    renderer: JSRenderer | None = None,
+    robots: RobotsHandler | None = None,
+  ) -> None:
+    self.config = config or CrawlConfig()
+    self.fetcher = fetcher or AsyncFetcher(self.config)
+    self._owns_fetcher = fetcher is None
+    self.renderer = renderer or (JSRenderer(self.config) if self.config.enable_js else None)
+    self.robots = robots or RobotsHandler(self.config)
+    self._owns_robots = robots is None
+    self._crawler = AsyncCrawler(
+      self.fetcher,
+      self.config,
+      robots=self.robots,
+      renderer=self.renderer,
+    )
+    logger.debug("AsyncWebWeaveX engine initialized")
+
+  async def crawl(self, url: str) -> PageResult:
+    logger.info("Async engine crawl requested for %s", url)
+    return await self._crawler.crawl(url)
+
+  async def crawl_many(self, urls: list[str]) -> list[PageResult]:
+    logger.info("Async engine crawl_many requested for %s urls", len(urls))
+    return await self._crawler.crawl_many(urls)
+
+  async def crawl_site(self, url: str) -> list[PageResult]:
+    logger.info("Async engine site crawl requested for %s", url)
+    from crawler_engine.site_crawler import SiteCrawler
+    site_crawler = SiteCrawler(
+      self.config,
+      fetcher=self.fetcher,
+      robots=self.robots,
+      renderer=self.renderer,
+    )
+    try:
+      return await site_crawler.crawl_site(url)
+    finally:
+      await site_crawler.aclose()
+
+  async def build_rag_dataset(self, url: str) -> list[dict[str, object]]:
+    logger.info("Dataset generation started for %s", url)
+    from rag.rag_pipeline import build_dataset
+    pages = await self.crawl_site(url)
+    dataset = build_dataset(pages)
+    logger.info("Dataset size %s", len(dataset))
+    return dataset
+
+  async def build_knowledge_graph(self, url: str) -> dict[str, list[dict[str, str]]]:
+    logger.info("Knowledge graph generation started for %s", url)
+    from knowledge_graph.graph_pipeline import build_graph
+    pages = await self.crawl_site(url)
+    graph = build_graph(pages)
+    logger.info("Knowledge graph nodes %s edges %s", len(graph.get("nodes", [])), len(graph.get("edges", [])))
+    return graph
+
+  async def aclose(self) -> None:
+    if self._owns_fetcher:
+      await self.fetcher.close()
+    if self._owns_robots:
+      await self.robots.close()
+
+  async def __aenter__(self) -> "AsyncWebWeaveX":
+    return self
+
+  async def __aexit__(self, exc_type, exc, tb) -> None:
+    await self.aclose()