xdive 1.5.0__tar.gz

xdive-1.5.0/.gitignore

@@ -0,0 +1,76 @@
+# Binaries
+/crawler
+/server
+/benchmark
+/bin/
+*.exe
+
+# Crawl output
+/out/
+/benchmark-results/
+/backups/
+
+# SQLite database files
+*.db
+*.db-shm
+*.db-wal
+
+# Go
+*.test
+*.out
+vendor/
+
+# Local env files (secrets)
+.env
+web/.env
+
+# Editor / tooling output
+.idea/
+.vscode/
+graphify-out/
+.DS_Store
+
+# Frontend build output
+web/dist/
+web/node_modules/
+web/test-results/
+web/playwright-report/
+
+# Local Claude Code tooling (preview launch config)
+.claude/
+loadtest/results/
+# SDK build artifacts
+sdk/js/node_modules/
+sdk/js/dist/
+sdk/vite/node_modules/
+sdk/vite/dist/
+sdk/ruby/.bundle/
+sdk/ruby/Gemfile.lock
+sdk/ruby/vendor/
+sdk/ruby/*.gem
+sdk/java/target/
+sdk/java/.idea/
+sdk/java/*.iml
+sdk/java/.gradle/
+sdk/java/build/
+sdk/php/vendor/
+sdk/php/composer.lock
+sdk/php/.phpunit.cache/
+sdk/php/.phpstan.cache/
+sdk/php/.php-cs-fixer.cache
+sdk/rust/target/
+sdk/rust/Cargo.lock
+sdk/dotnet/**/bin/
+sdk/dotnet/**/obj/
+sdk/dotnet/out/
+sdk/dotnet/**/*.user
+sdk/dotnet/.vs/
+sdk/dotnet/**/TestResults/
+sdk/python/build/
+sdk/python/dist/
+sdk/python/*.egg-info/
+sdk/python/.pytest_cache/
+__pycache__/
+# Head-to-head comparison output and credentials
+arena-results/
+.env.arena

xdive-1.5.0/CHANGELOG.md

@@ -0,0 +1,75 @@
+# Changelog
+
+All notable changes to the `xdive` Python package. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the package uses
+[Semantic Versioning](https://semver.org/).
+
+## Unreleased
+
+### Changed
+
+- **The package is now `xdive`** (it was `crawler-sdk`, a name that belongs to
+  an unrelated PyPI project). Import it as `xdive`: the clients are `XDive` and
+  `AsyncXDive`, errors derive from `XDiveError`, and the integrations are
+  `XDiveSearchTool`, `XDiveScrapeTool`, `XDiveResearchTool`, `XDiveLoader` and
+  `XDiveReader`. The tools are named `xdive_search`, `xdive_scrape` and
+  `xdive_research`.
+- The environment variables are `XDIVE_API_KEY` and `XDIVE_BASE_URL`. The
+  older `CRAWLER_API_KEY` and `CRAWLER_BASE_URL` still work as fallbacks.
+
+### Added
+
+- Every `POST` carries an `Idempotency-Key`, reused by all retries of that
+  request, so a retry never runs or bills the work twice.
+- Timeouts and network failures are retried (up to the retry limit) for `GET`
+  and `POST`, as is `409 IDEMPOTENCY_KEY_IN_PROGRESS`.
+- `XDiveError.retryable` exposes the server's `retryable` flag.
+- `AsyncXDive.batch_scrape`, `AsyncXDive.llms_txt` and
+  `AsyncXDive.crawl_errors`, with the same signatures as the synchronous
+  `XDive` methods. `AsyncXDive` now has every method `XDive` has.
+- `page_markdown(page)`, exported from `xdive`: a page's Markdown from
+  `artifacts.markdown.content` or `markdown`, whichever carries it. The
+  LangChain and LlamaIndex integrations use it.
+- A `py.typed` marker, so type checkers use the package's annotations, and
+  the `Typing :: Typed` classifier. `scrape(formats=...)` is annotated to
+  accept format dicts as well as names.
+- The `research` docstring describes `usage.plan` (`single`, `multi_hop` or
+  `unavailable`).
+- `LICENSE` (MIT) in the package, and a `Documentation` project URL.
+
+### Fixed
+
+- Only overload is retried: a `429`, or a `503` whose body has the code
+  `SERVER_BUSY`. Any other `503` (for example "search is not configured" or
+  "could not enqueue crawl") now raises `XDiveError` at once with the API's
+  status, message and code, instead of being retried and raised as
+  `RateLimited`.
+- The `search` docstrings no longer say that `search_depth="advanced"`
+  re-ranks results. It reads the top results and attaches each one's
+  query-relevant `passages` and a `relevance_score`; results keep the search
+  engine's order.
+
+## 1.5.0
+
+First packaged release.
+
+- `XDive` (synchronous, `httpx.Client`) and `AsyncXDive` (asynchronous,
+  `httpx.AsyncClient`) clients, both usable as context managers.
+- Configuration from arguments or the environment: `api_key` /
+  `CRAWLER_API_KEY`, `base_url` / `CRAWLER_BASE_URL`, plus `org_id`,
+  `timeout` and `max_retries`. The organization id is discovered once through
+  `GET /whoami`.
+- `XDive` methods: `whoami`, `scrape`, `start_crawl`, `get_crawl`,
+  `cancel_crawl`, `crawl_errors`, `wait_for_crawl`, `crawl`, `batch_scrape`,
+  `map`, `search`, `research`, `extract` and `llms_txt`. `AsyncXDive` had
+  all of them as coroutines except `batch_scrape`, `llms_txt` and
+  `crawl_errors`.
+- Automatic retries of `429` and `503` responses after the server's
+  `Retry-After`, up to `max_retries` (default 3).
+- Crawl polling with `If-None-Match`, so unchanged status answers are bodiless
+  `304`s, at the interval the server suggests.
+- Typed errors: `XDiveError` (`status`, `message`, `code`), `RateLimited`
+  (`retry_after`), `QuotaExceeded` (HTTP 402) and `CrawlTimeout`.
+- LangChain integration (`xdive[langchain]`): `XDiveSearchTool`,
+  `XDiveScrapeTool`, `XDiveResearchTool` and `XDiveLoader`.
+- LlamaIndex integration (`xdive[llama-index]`): `XDiveReader`.

xdive-1.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mehdi Sadeghian
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

xdive-1.5.0/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.5
+Name: xdive
+Version: 1.5.0
+Summary: Python SDK for XDive: scrape, crawl, map, search and research the web with citations, for AI agents.
+Project-URL: Homepage, https://crawler.mehdisadeghian.com
+Project-URL: Documentation, https://crawler.mehdisadeghian.com/docs/sdk-python
+Author: XDive
+License: MIT
+License-File: LICENSE
+Keywords: agents,crawler,langchain,llm,markdown,mcp,rag,research,search,web scraping,xdive
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: httpx<1,>=0.25
+Provides-Extra: dev
+Requires-Dist: langchain-core>=0.2; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == 'langchain'
+Provides-Extra: llama-index
+Requires-Dist: llama-index-core>=0.10; extra == 'llama-index'
+Description-Content-Type: text/markdown
+
+# xdive
+
+Python client for [XDive](https://crawler.mehdisadeghian.com): scrape pages
+to Markdown, crawl and map sites, search the web, and answer questions with
+verified citations. Synchronous and asynchronous clients, typed errors,
+automatic retries, and LangChain and LlamaIndex integrations.
+
+Full documentation: https://crawler.mehdisadeghian.com/docs/sdk-python
+
+## Install
+
+```bash
+pip install xdive
+pip install "xdive[langchain]"     # LangChain tools and document loader
+pip install "xdive[llama-index]"   # LlamaIndex reader
+```
+
+Requires Python 3.9 or later. The only runtime dependency is `httpx`.
+
+## Quickstart
+
+```python
+from xdive import XDive, page_markdown
+
+with XDive() as xdive:  # reads XDIVE_API_KEY
+    page = xdive.scrape("https://example.com", formats=["markdown", "links"])
+    print(page_markdown(page))
+    site = xdive.crawl("https://docs.example.com", max_pages=50)  # waits for the crawl
+    answer = xdive.research("What does PEP 639 change about license metadata?")
+    print(answer["answer"])
+    for citation in answer["citations"]:
+        print(citation["number"], citation["url"])
+```
+
+Create an API key under **API Keys** in the dashboard. A key belongs to one
+organization; the client looks the organization up once through
+`GET /whoami`, so the key is all you configure.
+
+## Configuration
+
+| Argument | Environment variable | Default | Description |
+|---|---|---|---|
+| `api_key` | `XDIVE_API_KEY` (or `CRAWLER_API_KEY`) | (required) | Organization API key (`ck_live_…`). |
+| `base_url` | `XDIVE_BASE_URL` (or `CRAWLER_BASE_URL`) | `https://crawler.mehdisadeghian.com/api` | API base URL, including `/api`. |
+| `org_id` | | discovered | Skip the `/whoami` lookup when you know the organization id. |
+| `timeout` | | `120.0` | Per-request timeout in seconds. |
+| `max_retries` | | `3` | Retries of overload responses: `429`, and `503` with code `SERVER_BUSY`. |
+
+A missing key raises `ValueError` when the client is created.
+
+## Methods
+
+`XDive` and `AsyncXDive` have the same methods; on `AsyncXDive` each is
+a coroutine. Keyword options are passed through as the API's JSON fields.
+
+| Method | API call | Returns |
+|---|---|---|
+| `whoami()` | `GET /whoami` | The key's organization, scopes and plan. |
+| `scrape(url, formats=None, **options)` | `POST /orgs/{id}/scrape` | The page object (`data`). |
+| `crawl(url, *, timeout=600, poll_interval=2, **options)` | `POST /orgs/{id}/crawl`, then polls | The finished crawl record with `results`. |
+| `start_crawl(url, **options)` | `POST /orgs/{id}/crawl` | The crawl id. |
+| `get_crawl(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}` | The crawl record. |
+| `wait_for_crawl(crawl_id, *, timeout=600, poll_interval=2)` | polls `GET /orgs/{id}/crawls/{crawlId}` | The finished crawl record. |
+| `cancel_crawl(crawl_id)` | `DELETE /orgs/{id}/crawls/{crawlId}` | `None`. |
+| `crawl_errors(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}/errors` | Failed pages (`errors`) and `robots_blocked` URLs. |
+| `batch_scrape(urls, *, wait=True, timeout=600, **options)` | `POST /orgs/{id}/batch/scrape` | The finished record, or the job id with `wait=False`. |
+| `map(url, *, search=None, limit=None, max_depth=None)` | `POST /orgs/{id}/map` | A list of URLs. |
+| `search(query, **options)` | `POST /orgs/{id}/search` | The search response with `results`. |
+| `research(query, *, depth="quick", output=None, schema=None, **options)` | `POST /orgs/{id}/research` | `answer` or `data`, `citations`, `sources`, `confidence`, `usage`. |
+| `extract(urls, *, prompt=None, schema=None, **options)` | `POST /orgs/{id}/extract` | The extracted data. |
+| `llms_txt(url, *, max_urls=None, full_text=False)` | `POST /orgs/{id}/llmstxt` | `llms_txt` (and `llms_full_txt`). |
+
+`XDive.org_id` is a property; on `AsyncXDive` it is a coroutine,
+`await client.org_id()`.
+
+`formats` takes the API's format names (`markdown`, `text`, `html`,
+`rawHtml`, `links`, `images`, `json`, `summary`, `question`, `highlights`,
+`branding`, `audio`, `screenshot`, `product`, `menu`) or dicts such as
+`{"type": "json", "schema": {...}}`. `page_markdown(page)` returns a page's
+Markdown whichever field carries it (`artifacts.markdown.content` or
+`markdown`).
+
+## Async
+
+```python
+import asyncio
+from xdive import AsyncXDive
+
+async def main():
+    async with AsyncXDive() as client:
+        pages = await asyncio.gather(*(client.scrape(u) for u in ["https://a.example", "https://b.example"]))
+
+asyncio.run(main())
+```
+
+Close clients you do not use as context managers: `xdive.close()` or
+`await client.aclose()`.
+
+## Errors and retries
+
+| Exception | When | Attributes |
+|---|---|---|
+| `XDiveError` | Any API error (status 400 and above). Base class of the others. | `status`, `message`, `code` |
+| `RateLimited` | Still overloaded (`429`, or `503` with code `SERVER_BUSY`) after `max_retries` retries. | `retry_after` (seconds) |
+| `QuotaExceeded` | `402`: the organization's monthly quota is used up. | |
+| `CrawlTimeout` | `crawl`, `wait_for_crawl` or `batch_scrape` waited longer than `timeout`. The job keeps running on the server. | `crawl_id` |
+
+Only overload is retried: a `429` (organization rate or concurrency limit)
+and a `503` whose JSON body has the code `SERVER_BUSY` (server at capacity).
+The client waits the number of seconds in the response's `Retry-After`
+header (1 second when absent) before each retry. Every other error, including
+any other `503` (for example a feature the server has not configured), raises
+`XDiveError` at once with the API's `status`, `message` and `code`. Network
+errors and timeouts surface as `httpx` exceptions.
+
+Crawl polling sends `If-None-Match`, so an unchanged status comes back as an
+empty `304`, and waits the interval the server sets in `Retry-After`,
+falling back to `poll_interval`.
+
+## Integrations
+
+```python
+from xdive.langchain import XDiveLoader, XDiveResearchTool, XDiveScrapeTool, XDiveSearchTool
+
+tools = [XDiveSearchTool(), XDiveScrapeTool(), XDiveResearchTool()]
+docs = XDiveLoader("https://docs.example.com", mode="crawl", max_pages=100).load()
+```
+
+```python
+from xdive.llama_index import XDiveReader
+
+documents = XDiveReader().load_data("https://docs.example.com", mode="crawl", max_pages=100)
+```
+
+`XDiveSearchTool` returns each result's query-relevant passages rather than
+whole pages, which keeps an agent's context small. See the
+[LangChain](https://crawler.mehdisadeghian.com/docs/langchain) and
+[LlamaIndex](https://crawler.mehdisadeghian.com/docs/llamaindex) guides.
+
+## Self-hosted
+
+```python
+xdive = XDive(api_key="ck_live_…", base_url="https://your-host/api")
+```
+
+or set `XDIVE_BASE_URL=https://your-host/api`.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT. The license text is in the `LICENSE` file included in the package.

xdive-1.5.0/README.md

@@ -0,0 +1,157 @@
+# xdive
+
+Python client for [XDive](https://crawler.mehdisadeghian.com): scrape pages
+to Markdown, crawl and map sites, search the web, and answer questions with
+verified citations. Synchronous and asynchronous clients, typed errors,
+automatic retries, and LangChain and LlamaIndex integrations.
+
+Full documentation: https://crawler.mehdisadeghian.com/docs/sdk-python
+
+## Install
+
+```bash
+pip install xdive
+pip install "xdive[langchain]"     # LangChain tools and document loader
+pip install "xdive[llama-index]"   # LlamaIndex reader
+```
+
+Requires Python 3.9 or later. The only runtime dependency is `httpx`.
+
+## Quickstart
+
+```python
+from xdive import XDive, page_markdown
+
+with XDive() as xdive:  # reads XDIVE_API_KEY
+    page = xdive.scrape("https://example.com", formats=["markdown", "links"])
+    print(page_markdown(page))
+    site = xdive.crawl("https://docs.example.com", max_pages=50)  # waits for the crawl
+    answer = xdive.research("What does PEP 639 change about license metadata?")
+    print(answer["answer"])
+    for citation in answer["citations"]:
+        print(citation["number"], citation["url"])
+```
+
+Create an API key under **API Keys** in the dashboard. A key belongs to one
+organization; the client looks the organization up once through
+`GET /whoami`, so the key is all you configure.
+
+## Configuration
+
+| Argument | Environment variable | Default | Description |
+|---|---|---|---|
+| `api_key` | `XDIVE_API_KEY` (or `CRAWLER_API_KEY`) | (required) | Organization API key (`ck_live_…`). |
+| `base_url` | `XDIVE_BASE_URL` (or `CRAWLER_BASE_URL`) | `https://crawler.mehdisadeghian.com/api` | API base URL, including `/api`. |
+| `org_id` | | discovered | Skip the `/whoami` lookup when you know the organization id. |
+| `timeout` | | `120.0` | Per-request timeout in seconds. |
+| `max_retries` | | `3` | Retries of overload responses: `429`, and `503` with code `SERVER_BUSY`. |
+
+A missing key raises `ValueError` when the client is created.
+
+## Methods
+
+`XDive` and `AsyncXDive` have the same methods; on `AsyncXDive` each is
+a coroutine. Keyword options are passed through as the API's JSON fields.
+
+| Method | API call | Returns |
+|---|---|---|
+| `whoami()` | `GET /whoami` | The key's organization, scopes and plan. |
+| `scrape(url, formats=None, **options)` | `POST /orgs/{id}/scrape` | The page object (`data`). |
+| `crawl(url, *, timeout=600, poll_interval=2, **options)` | `POST /orgs/{id}/crawl`, then polls | The finished crawl record with `results`. |
+| `start_crawl(url, **options)` | `POST /orgs/{id}/crawl` | The crawl id. |
+| `get_crawl(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}` | The crawl record. |
+| `wait_for_crawl(crawl_id, *, timeout=600, poll_interval=2)` | polls `GET /orgs/{id}/crawls/{crawlId}` | The finished crawl record. |
+| `cancel_crawl(crawl_id)` | `DELETE /orgs/{id}/crawls/{crawlId}` | `None`. |
+| `crawl_errors(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}/errors` | Failed pages (`errors`) and `robots_blocked` URLs. |
+| `batch_scrape(urls, *, wait=True, timeout=600, **options)` | `POST /orgs/{id}/batch/scrape` | The finished record, or the job id with `wait=False`. |
+| `map(url, *, search=None, limit=None, max_depth=None)` | `POST /orgs/{id}/map` | A list of URLs. |
+| `search(query, **options)` | `POST /orgs/{id}/search` | The search response with `results`. |
+| `research(query, *, depth="quick", output=None, schema=None, **options)` | `POST /orgs/{id}/research` | `answer` or `data`, `citations`, `sources`, `confidence`, `usage`. |
+| `extract(urls, *, prompt=None, schema=None, **options)` | `POST /orgs/{id}/extract` | The extracted data. |
+| `llms_txt(url, *, max_urls=None, full_text=False)` | `POST /orgs/{id}/llmstxt` | `llms_txt` (and `llms_full_txt`). |
+
+`XDive.org_id` is a property; on `AsyncXDive` it is a coroutine,
+`await client.org_id()`.
+
+`formats` takes the API's format names (`markdown`, `text`, `html`,
+`rawHtml`, `links`, `images`, `json`, `summary`, `question`, `highlights`,
+`branding`, `audio`, `screenshot`, `product`, `menu`) or dicts such as
+`{"type": "json", "schema": {...}}`. `page_markdown(page)` returns a page's
+Markdown whichever field carries it (`artifacts.markdown.content` or
+`markdown`).
+
+## Async
+
+```python
+import asyncio
+from xdive import AsyncXDive
+
+async def main():
+    async with AsyncXDive() as client:
+        pages = await asyncio.gather(*(client.scrape(u) for u in ["https://a.example", "https://b.example"]))
+
+asyncio.run(main())
+```
+
+Close clients you do not use as context managers: `xdive.close()` or
+`await client.aclose()`.
+
+## Errors and retries
+
+| Exception | When | Attributes |
+|---|---|---|
+| `XDiveError` | Any API error (status 400 and above). Base class of the others. | `status`, `message`, `code` |
+| `RateLimited` | Still overloaded (`429`, or `503` with code `SERVER_BUSY`) after `max_retries` retries. | `retry_after` (seconds) |
+| `QuotaExceeded` | `402`: the organization's monthly quota is used up. | |
+| `CrawlTimeout` | `crawl`, `wait_for_crawl` or `batch_scrape` waited longer than `timeout`. The job keeps running on the server. | `crawl_id` |
+
+Only overload is retried: a `429` (organization rate or concurrency limit)
+and a `503` whose JSON body has the code `SERVER_BUSY` (server at capacity).
+The client waits the number of seconds in the response's `Retry-After`
+header (1 second when absent) before each retry. Every other error, including
+any other `503` (for example a feature the server has not configured), raises
+`XDiveError` at once with the API's `status`, `message` and `code`. Network
+errors and timeouts surface as `httpx` exceptions.
+
+Crawl polling sends `If-None-Match`, so an unchanged status comes back as an
+empty `304`, and waits the interval the server sets in `Retry-After`,
+falling back to `poll_interval`.
+
+## Integrations
+
+```python
+from xdive.langchain import XDiveLoader, XDiveResearchTool, XDiveScrapeTool, XDiveSearchTool
+
+tools = [XDiveSearchTool(), XDiveScrapeTool(), XDiveResearchTool()]
+docs = XDiveLoader("https://docs.example.com", mode="crawl", max_pages=100).load()
+```
+
+```python
+from xdive.llama_index import XDiveReader
+
+documents = XDiveReader().load_data("https://docs.example.com", mode="crawl", max_pages=100)
+```
+
+`XDiveSearchTool` returns each result's query-relevant passages rather than
+whole pages, which keeps an agent's context small. See the
+[LangChain](https://crawler.mehdisadeghian.com/docs/langchain) and
+[LlamaIndex](https://crawler.mehdisadeghian.com/docs/llamaindex) guides.
+
+## Self-hosted
+
+```python
+xdive = XDive(api_key="ck_live_…", base_url="https://your-host/api")
+```
+
+or set `XDIVE_BASE_URL=https://your-host/api`.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT. The license text is in the `LICENSE` file included in the package.

xdive-1.5.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "xdive"
+version = "1.5.0"
+description = "Python SDK for XDive: scrape, crawl, map, search and research the web with citations, for AI agents."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "XDive" }]
+keywords = ["xdive", "web scraping", "crawler", "markdown", "rag", "search", "research", "llm", "agents", "langchain", "mcp"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: Internet :: WWW/HTTP",
+  "Typing :: Typed",
+]
+dependencies = ["httpx>=0.25,<1"]
+
+[project.optional-dependencies]
+langchain = ["langchain-core>=0.2"]
+llama-index = ["llama-index-core>=0.10"]
+dev = ["pytest>=8", "langchain-core>=0.2"]
+
+[project.urls]
+Homepage = "https://crawler.mehdisadeghian.com"
+Documentation = "https://crawler.mehdisadeghian.com/docs/sdk-python"
+
+[tool.hatch.build.targets.wheel]
+packages = ["xdive"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]