langchain-wtmapi 0.1.0__tar.gz

langchain_wtmapi-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Filippo Tedeschi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_wtmapi-0.1.0/PKG-INFO

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: langchain-wtmapi
+Version: 0.1.0
+Summary: LangChain Document Loader for WTM API — convert any webpage to Markdown
+Author-email: Filippo Tedeschi <filippotedeschi98@gmail.com>
+License: MIT
+Project-URL: Homepage, https://wtmapi.com
+Project-URL: Documentation, https://wtmapi.com/docs
+Project-URL: Repository, https://github.com/filtede98/langchain-wtmapi
+Keywords: langchain,document-loader,markdown,web-scraping,rag,llm,wtm-api
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Text Processing :: Markup
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain-core>=0.2.0
+Dynamic: license-file
+
+# langchain-wtmapi
+
+LangChain Document Loader for [WTM API](https://wtmapi.com) — convert any webpage to clean, structured Markdown.
+
+## Installation
+
+```bash
+pip install langchain-wtmapi
+```
+
+## Quick Start
+
+```python
+from langchain_wtmapi import WTMApiLoader
+
+# Get your free API key at https://wtmapi.com
+loader = WTMApiLoader(
+    urls=[
+        "https://en.wikipedia.org/wiki/Artificial_intelligence",
+        "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+    ],
+    api_key="wtm_your_api_key",
+)
+
+# Load all documents
+docs = loader.load()
+
+for doc in docs:
+    print(f"Source: {doc.metadata['source']}")
+    print(f"Length: {doc.metadata['length']} chars")
+    print(f"Response time: {doc.metadata['response_time_ms']}ms")
+    print(doc.page_content[:500])
+    print("---")
+```
+
+## Lazy Loading
+
+For large batches, use lazy loading to process documents one at a time:
+
+```python
+for doc in loader.lazy_load():
+    # Process each document as it's loaded
+    process_document(doc)
+```
+
+## Use with RAG Pipeline
+
+```python
+from langchain_wtmapi import WTMApiLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import FAISS
+
+# Load web pages as Markdown
+loader = WTMApiLoader(
+    urls=["https://docs.python.org/3/tutorial/index.html"],
+    api_key="wtm_your_api_key",
+)
+docs = loader.load()
+
+# Split into chunks
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = splitter.split_documents(docs)
+
+# Create vector store
+embeddings = OpenAIEmbeddings()
+vectorstore = FAISS.from_documents(chunks, embeddings)
+
+# Query
+results = vectorstore.similarity_search("How do I use lists in Python?")
+```
+
+## Options
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `urls` | `list[str]` | required | List of URLs to convert |
+| `api_key` | `str` | required | Your WTM API key |
+| `include_links` | `bool` | `True` | Include links in output |
+| `include_images` | `bool` | `True` | Include images in output |
+| `api_url` | `str` | `https://wtmapi.com/api/v1/convert` | API endpoint |
+
+## Document Metadata
+
+Each loaded document includes metadata:
+
+```python
+{
+    "source": "https://example.com",        # Original URL
+    "length": 15234,                         # Markdown length in chars
+    "response_time_ms": 523,                 # API response time
+    "usage_used": 5,                         # Monthly calls used
+    "usage_limit": 50,                       # Monthly call limit
+    "plan": "free",                          # Current plan
+}
+```
+
+## Get Your API Key
+
+Sign up at [wtmapi.com](https://wtmapi.com) — free tier includes 50 calls/month, no credit card required.
+
+## License
+
+MIT

langchain_wtmapi-0.1.0/README.md

@@ -0,0 +1,104 @@
+# langchain-wtmapi
+
+LangChain Document Loader for [WTM API](https://wtmapi.com) — convert any webpage to clean, structured Markdown.
+
+## Installation
+
+```bash
+pip install langchain-wtmapi
+```
+
+## Quick Start
+
+```python
+from langchain_wtmapi import WTMApiLoader
+
+# Get your free API key at https://wtmapi.com
+loader = WTMApiLoader(
+    urls=[
+        "https://en.wikipedia.org/wiki/Artificial_intelligence",
+        "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+    ],
+    api_key="wtm_your_api_key",
+)
+
+# Load all documents
+docs = loader.load()
+
+for doc in docs:
+    print(f"Source: {doc.metadata['source']}")
+    print(f"Length: {doc.metadata['length']} chars")
+    print(f"Response time: {doc.metadata['response_time_ms']}ms")
+    print(doc.page_content[:500])
+    print("---")
+```
+
+## Lazy Loading
+
+For large batches, use lazy loading to process documents one at a time:
+
+```python
+for doc in loader.lazy_load():
+    # Process each document as it's loaded
+    process_document(doc)
+```
+
+## Use with RAG Pipeline
+
+```python
+from langchain_wtmapi import WTMApiLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import FAISS
+
+# Load web pages as Markdown
+loader = WTMApiLoader(
+    urls=["https://docs.python.org/3/tutorial/index.html"],
+    api_key="wtm_your_api_key",
+)
+docs = loader.load()
+
+# Split into chunks
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = splitter.split_documents(docs)
+
+# Create vector store
+embeddings = OpenAIEmbeddings()
+vectorstore = FAISS.from_documents(chunks, embeddings)
+
+# Query
+results = vectorstore.similarity_search("How do I use lists in Python?")
+```
+
+## Options
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `urls` | `list[str]` | required | List of URLs to convert |
+| `api_key` | `str` | required | Your WTM API key |
+| `include_links` | `bool` | `True` | Include links in output |
+| `include_images` | `bool` | `True` | Include images in output |
+| `api_url` | `str` | `https://wtmapi.com/api/v1/convert` | API endpoint |
+
+## Document Metadata
+
+Each loaded document includes metadata:
+
+```python
+{
+    "source": "https://example.com",        # Original URL
+    "length": 15234,                         # Markdown length in chars
+    "response_time_ms": 523,                 # API response time
+    "usage_used": 5,                         # Monthly calls used
+    "usage_limit": 50,                       # Monthly call limit
+    "plan": "free",                          # Current plan
+}
+```
+
+## Get Your API Key
+
+Sign up at [wtmapi.com](https://wtmapi.com) — free tier includes 50 calls/month, no credit card required.
+
+## License
+
+MIT

langchain_wtmapi-0.1.0/langchain_wtmapi/__init__.py

@@ -0,0 +1,3 @@
+from langchain_wtmapi.document_loader import WTMApiLoader
+
+__all__ = ["WTMApiLoader"]

langchain_wtmapi-0.1.0/langchain_wtmapi/document_loader.py

@@ -0,0 +1,138 @@
+"""WTM API Document Loader for LangChain.
+
+Convert any webpage to clean Markdown using the WTM API (https://wtmapi.com).
+"""
+
+from __future__ import annotations
+
+import urllib.request
+import json
+from typing import Iterator, Optional
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+
+class WTMApiLoader(BaseLoader):
+    """Load web pages as Markdown documents using the WTM API.
+
+    WTM API converts any webpage to structured Markdown, preserving
+    headings, tables, code blocks, links, and images.
+
+    Setup:
+        Sign up at https://wtmapi.com to get a free API key (50 calls/month).
+
+        .. code-block:: bash
+
+            pip install langchain-wtmapi
+
+    Instantiate:
+        .. code-block:: python
+
+            from langchain_wtmapi import WTMApiLoader
+
+            loader = WTMApiLoader(
+                urls=["https://en.wikipedia.org/wiki/Mars"],
+                api_key="wtm_your_api_key",
+            )
+
+    Load:
+        .. code-block:: python
+
+            docs = loader.load()
+            print(docs[0].page_content[:200])
+            print(docs[0].metadata)
+
+    Lazy load:
+        .. code-block:: python
+
+            for doc in loader.lazy_load():
+                print(doc.metadata["source"])
+    """
+
+    urls: list[str]
+    """List of URLs to convert."""
+
+    api_key: str
+    """WTM API key. Get one free at https://wtmapi.com"""
+
+    api_url: str = "https://wtmapi.com/api/v1/convert"
+    """WTM API endpoint URL."""
+
+    include_links: bool = True
+    """Whether to include links in the Markdown output."""
+
+    include_images: bool = True
+    """Whether to include images in the Markdown output."""
+
+    def __init__(
+        self,
+        urls: list[str],
+        api_key: str,
+        api_url: str = "https://wtmapi.com/api/v1/convert",
+        include_links: bool = True,
+        include_images: bool = True,
+    ) -> None:
+        self.urls = urls
+        self.api_key = api_key
+        self.api_url = api_url
+        self.include_links = include_links
+        self.include_images = include_images
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Lazy load documents from URLs via WTM API.
+
+        Yields:
+            Document: A document with Markdown content and metadata.
+        """
+        for url in self.urls:
+            try:
+                doc = self._convert_url(url)
+                if doc:
+                    yield doc
+            except Exception as e:
+                yield Document(
+                    page_content="",
+                    metadata={
+                        "source": url,
+                        "error": str(e),
+                    },
+                )
+
+    def _convert_url(self, url: str) -> Optional[Document]:
+        """Convert a single URL to a Document via the WTM API."""
+        payload = json.dumps({
+            "url": url,
+            "include_links": self.include_links,
+            "include_images": self.include_images,
+        }).encode("utf-8")
+
+        req = urllib.request.Request(
+            self.api_url,
+            data=payload,
+            headers={
+                "Content-Type": "application/json",
+                "x-api-key": self.api_key,
+            },
+            method="POST",
+        )
+
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            data = json.loads(resp.read().decode("utf-8"))
+
+        if not data.get("success"):
+            error = data.get("error", "Unknown error")
+            raise ValueError(f"WTM API error: {error}")
+
+        result = data["data"]
+        return Document(
+            page_content=result["markdown"],
+            metadata={
+                "source": url,
+                "length": result.get("length", 0),
+                "response_time_ms": data.get("meta", {}).get("response_time_ms"),
+                "usage_used": result.get("usage", {}).get("used"),
+                "usage_limit": result.get("usage", {}).get("limit"),
+                "plan": result.get("usage", {}).get("plan"),
+            },
+        )

langchain_wtmapi-0.1.0/langchain_wtmapi.egg-info/PKG-INFO

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: langchain-wtmapi
+Version: 0.1.0
+Summary: LangChain Document Loader for WTM API — convert any webpage to Markdown
+Author-email: Filippo Tedeschi <filippotedeschi98@gmail.com>
+License: MIT
+Project-URL: Homepage, https://wtmapi.com
+Project-URL: Documentation, https://wtmapi.com/docs
+Project-URL: Repository, https://github.com/filtede98/langchain-wtmapi
+Keywords: langchain,document-loader,markdown,web-scraping,rag,llm,wtm-api
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Text Processing :: Markup
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain-core>=0.2.0
+Dynamic: license-file
+
+# langchain-wtmapi
+
+LangChain Document Loader for [WTM API](https://wtmapi.com) — convert any webpage to clean, structured Markdown.
+
+## Installation
+
+```bash
+pip install langchain-wtmapi
+```
+
+## Quick Start
+
+```python
+from langchain_wtmapi import WTMApiLoader
+
+# Get your free API key at https://wtmapi.com
+loader = WTMApiLoader(
+    urls=[
+        "https://en.wikipedia.org/wiki/Artificial_intelligence",
+        "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+    ],
+    api_key="wtm_your_api_key",
+)
+
+# Load all documents
+docs = loader.load()
+
+for doc in docs:
+    print(f"Source: {doc.metadata['source']}")
+    print(f"Length: {doc.metadata['length']} chars")
+    print(f"Response time: {doc.metadata['response_time_ms']}ms")
+    print(doc.page_content[:500])
+    print("---")
+```
+
+## Lazy Loading
+
+For large batches, use lazy loading to process documents one at a time:
+
+```python
+for doc in loader.lazy_load():
+    # Process each document as it's loaded
+    process_document(doc)
+```
+
+## Use with RAG Pipeline
+
+```python
+from langchain_wtmapi import WTMApiLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import FAISS
+
+# Load web pages as Markdown
+loader = WTMApiLoader(
+    urls=["https://docs.python.org/3/tutorial/index.html"],
+    api_key="wtm_your_api_key",
+)
+docs = loader.load()
+
+# Split into chunks
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = splitter.split_documents(docs)
+
+# Create vector store
+embeddings = OpenAIEmbeddings()
+vectorstore = FAISS.from_documents(chunks, embeddings)
+
+# Query
+results = vectorstore.similarity_search("How do I use lists in Python?")
+```
+
+## Options
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `urls` | `list[str]` | required | List of URLs to convert |
+| `api_key` | `str` | required | Your WTM API key |
+| `include_links` | `bool` | `True` | Include links in output |
+| `include_images` | `bool` | `True` | Include images in output |
+| `api_url` | `str` | `https://wtmapi.com/api/v1/convert` | API endpoint |
+
+## Document Metadata
+
+Each loaded document includes metadata:
+
+```python
+{
+    "source": "https://example.com",        # Original URL
+    "length": 15234,                         # Markdown length in chars
+    "response_time_ms": 523,                 # API response time
+    "usage_used": 5,                         # Monthly calls used
+    "usage_limit": 50,                       # Monthly call limit
+    "plan": "free",                          # Current plan
+}
+```
+
+## Get Your API Key
+
+Sign up at [wtmapi.com](https://wtmapi.com) — free tier includes 50 calls/month, no credit card required.
+
+## License
+
+MIT

langchain_wtmapi-0.1.0/langchain_wtmapi.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+LICENSE
+README.md
+pyproject.toml
+langchain_wtmapi/__init__.py
+langchain_wtmapi/document_loader.py
+langchain_wtmapi.egg-info/PKG-INFO
+langchain_wtmapi.egg-info/SOURCES.txt
+langchain_wtmapi.egg-info/dependency_links.txt
+langchain_wtmapi.egg-info/requires.txt
+langchain_wtmapi.egg-info/top_level.txt

langchain_wtmapi-0.1.0/langchain_wtmapi.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

langchain_wtmapi-0.1.0/langchain_wtmapi.egg-info/requires.txt

@@ -0,0 +1 @@
+langchain-core>=0.2.0

langchain_wtmapi-0.1.0/langchain_wtmapi.egg-info/top_level.txt

@@ -0,0 +1 @@
+langchain_wtmapi

langchain_wtmapi-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "langchain-wtmapi"
+version = "0.1.0"
+description = "LangChain Document Loader for WTM API — convert any webpage to Markdown"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Filippo Tedeschi", email = "filippotedeschi98@gmail.com"},
+]
+keywords = ["langchain", "document-loader", "markdown", "web-scraping", "rag", "llm", "wtm-api"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Text Processing :: Markup",
+]
+dependencies = [
+    "langchain-core>=0.2.0",
+]
+
+[project.urls]
+Homepage = "https://wtmapi.com"
+Documentation = "https://wtmapi.com/docs"
+Repository = "https://github.com/filtede98/langchain-wtmapi"

langchain_wtmapi-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+