llama-index-readers-contextfetch 0.1.0__tar.gz

llama_index_readers_contextfetch-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+.venv/
+node_modules/
+.next/
+dist/
+engine/.model_cache/
+engine/.data/
+.env
+.env.npm
+.env.webshare
+.venv
+engine/.model_cache
+growth-sniper/leads.csv
+__pycache__/
+.pytest_cache/
+.env.hetzner
+.next-*/
+.vercel/
+benchmarks/data/
+*.tsbuildinfo
+next-env.d.ts

llama_index_readers_contextfetch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2026 ContextFetch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

llama_index_readers_contextfetch-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.5
+Name: llama-index-readers-contextfetch
+Version: 0.1.0
+Summary: llama-index readers contextfetch integration
+Project-URL: Homepage, https://contextfetch.dev
+Project-URL: Documentation, https://contextfetch.dev/docs/reference
+Author: ContextFetch
+Maintainer: lachy3433
+License-Expression: MIT
+License-File: LICENSE
+Keywords: contextfetch,markdown,web,web search
+Requires-Python: <4.0,>=3.10
+Requires-Dist: contextfetch>=0.1.0
+Requires-Dist: llama-index-core<0.15,>=0.13.0
+Description-Content-Type: text/markdown
+
+# LlamaIndex Readers Integration: ContextFetch
+
+```bash
+pip install llama-index-readers-contextfetch
+```
+
+The ContextFetch reader loads web pages as LlamaIndex Documents with the page's
+Markdown as text, using the [ContextFetch](https://contextfetch.dev) API. Give it
+URLs, a web search query, or both.
+
+Pass `api_key`, or set the `CONTEXTFETCH_API_KEY` environment variable. With
+neither, requests use ContextFetch's keyless tier, a small daily allowance per IP.
+The `contextfetch` package can request a free starter key:
+`from contextfetch import ContextFetch; ContextFetch().free_key()`.
+
+## Usage
+
+```python
+from llama_index.core import VectorStoreIndex
+from llama_index.readers.contextfetch import ContextFetchReader
+
+reader = ContextFetchReader()  # or ContextFetchReader(api_key="...")
+
+# Pages by URL
+documents = reader.load_data(
+    urls=["https://docs.python.org/3/library/asyncio-task.html"]
+)
+
+# The pages a web search finds
+documents = reader.load_data(query="asyncio task groups", num_results=5)
+
+index = VectorStoreIndex.from_documents(documents)
+```
+
+Each Document has `id_` set to its URL and metadata `url`, `title` and, when the
+page provides them, `published_date` and `author`. A URL that fails to load is
+skipped and logged as a warning.
+
+This loader is designed to be used as a way to load data into [LlamaIndex](https://github.com/run-llama/llama_index/).
+For an agent tool, see `llama-index-tools-contextfetch`.

llama_index_readers_contextfetch-0.1.0/README.md

@@ -0,0 +1,40 @@
+# LlamaIndex Readers Integration: ContextFetch
+
+```bash
+pip install llama-index-readers-contextfetch
+```
+
+The ContextFetch reader loads web pages as LlamaIndex Documents with the page's
+Markdown as text, using the [ContextFetch](https://contextfetch.dev) API. Give it
+URLs, a web search query, or both.
+
+Pass `api_key`, or set the `CONTEXTFETCH_API_KEY` environment variable. With
+neither, requests use ContextFetch's keyless tier, a small daily allowance per IP.
+The `contextfetch` package can request a free starter key:
+`from contextfetch import ContextFetch; ContextFetch().free_key()`.
+
+## Usage
+
+```python
+from llama_index.core import VectorStoreIndex
+from llama_index.readers.contextfetch import ContextFetchReader
+
+reader = ContextFetchReader()  # or ContextFetchReader(api_key="...")
+
+# Pages by URL
+documents = reader.load_data(
+    urls=["https://docs.python.org/3/library/asyncio-task.html"]
+)
+
+# The pages a web search finds
+documents = reader.load_data(query="asyncio task groups", num_results=5)
+
+index = VectorStoreIndex.from_documents(documents)
+```
+
+Each Document has `id_` set to its URL and metadata `url`, `title` and, when the
+page provides them, `published_date` and `author`. A URL that fails to load is
+skipped and logged as a warning.
+
+This loader is designed to be used as a way to load data into [LlamaIndex](https://github.com/run-llama/llama_index/).
+For an agent tool, see `llama-index-tools-contextfetch`.

llama_index_readers_contextfetch-0.1.0/llama_index/readers/contextfetch/__init__.py

@@ -0,0 +1,3 @@
+from llama_index.readers.contextfetch.base import ContextFetchReader
+
+__all__ = ["ContextFetchReader"]

llama_index_readers_contextfetch-0.1.0/llama_index/readers/contextfetch/base.py

@@ -0,0 +1,91 @@
+"""ContextFetch reader."""
+
+import logging
+from typing import List, Optional
+
+from contextfetch import ContextFetch, Result
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+logger = logging.getLogger(__name__)
+
+# POST /v1/contents accepts at most this many URLs per call.
+MAX_URLS_PER_CALL = 10
+
+
+class ContextFetchReader(BaseReader):
+    """
+    Load web pages, or the pages a web search finds, as Markdown Documents.
+
+    Args:
+        api_key: ContextFetch API key. Defaults to the CONTEXTFETCH_API_KEY
+            environment variable; with neither, requests use the keyless tier.
+
+    Example:
+        .. code-block:: python
+
+            from llama_index.readers.contextfetch import ContextFetchReader
+
+            reader = ContextFetchReader()
+            documents = reader.load_data(
+                urls=["https://docs.python.org/3/library/asyncio-task.html"]
+            )
+            documents = reader.load_data(query="asyncio task groups", num_results=5)
+
+    """
+
+    def __init__(self, api_key: Optional[str] = None) -> None:
+        """Initialize with parameters."""
+        self.client = ContextFetch(api_key=api_key)
+
+    def load_data(
+        self,
+        urls: Optional[List[str]] = None,
+        query: Optional[str] = None,
+        num_results: Optional[int] = None,
+    ) -> List[Document]:
+        """
+        Load pages as Documents, one per page, with the page's Markdown as text.
+
+        Args:
+            urls: Pages to load. A URL that fails to load is skipped with a warning.
+            query: A web search; the pages it finds are loaded.
+            num_results: How many search results to load (the API default is 10).
+
+        Returns:
+            Documents with id_ set to the URL and metadata url, title and, when
+            known, published_date and author.
+
+        """
+        if not urls and not query:
+            raise ValueError("Pass urls, query, or both.")
+        urls = urls or []
+        results: List[Result] = []
+        for i in range(0, len(urls), MAX_URLS_PER_CALL):
+            contents = self.client.contents(urls[i : i + MAX_URLS_PER_CALL])
+            results += contents.get("results", [])
+            for status in contents.get("statuses", []):
+                if status.get("status") == "error":
+                    logger.warning(
+                        "ContextFetch could not load %s: %s",
+                        status.get("id"),
+                        status.get("error"),
+                    )
+        if query:
+            search = self.client.search(query, num_results=num_results, text=True)
+            results += search.get("results", [])
+        return [_to_document(result) for result in results]
+
+
+def _to_document(result: Result) -> Document:
+    metadata = {
+        "url": result.get("url"),
+        "title": result.get("title"),
+        "published_date": result.get("publishedDate"),
+        "author": result.get("author"),
+    }
+    return Document(
+        id_=result.get("url"),
+        text=result.get("text") or "",
+        metadata={k: v for k, v in metadata.items() if v is not None},
+    )

llama_index_readers_contextfetch-0.1.0/pyproject.toml

@@ -0,0 +1,73 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "black[jupyter]>=23.7.0,<=26.5.1",
+    "codespell[toml]>=v2.2.6",
+    "diff-cover>=9.2.0",
+    "ipython==8.10.0",
+    "jupyter>=1.0.0,<2",
+    "mypy==0.991",
+    "pre-commit==3.2.0",
+    "pylint==2.15.10",
+    "pytest",
+    "pytest-cov>=6.1.1",
+    "pytest-mock==3.11.1",
+    "ruff==0.11.11",
+    "types-Deprecated>=0.1.0",
+    "types-PyYAML>=6.0.12.12,<7",
+    "types-protobuf>=4.24.0.4,<5",
+    "types-redis==4.5.5.0",
+    "types-requests==2.28.11.8",
+    "types-setuptools==67.1.0.0",
+]
+
+[project]
+name = "llama-index-readers-contextfetch"
+version = "0.1.0"
+description = "llama-index readers contextfetch integration"
+authors = [{name = "ContextFetch"}]
+requires-python = ">=3.10,<4.0"
+readme = "README.md"
+license = "MIT"
+maintainers = [{name = "lachy3433"}]
+keywords = [
+    "contextfetch",
+    "markdown",
+    "web",
+    "web search",
+]
+dependencies = [
+    "contextfetch>=0.1.0",
+    "llama-index-core>=0.13.0,<0.15",
+]
+
+[project.urls]
+Homepage = "https://contextfetch.dev"
+Documentation = "https://contextfetch.dev/docs/reference"
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.hatch.build.targets.sdist]
+include = ["llama_index/"]
+
+[tool.hatch.build.targets.wheel]
+include = ["llama_index/"]
+
+[tool.llamahub]
+contains_example = false
+import_path = "llama_index.readers.contextfetch"
+
+[tool.llamahub.class_authors]
+ContextFetchReader = "lachy3433"
+
+[tool.mypy]
+disallow_untyped_defs = true
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+python_version = "3.8"