factag 0.1.0__tar.gz

factag-0.1.0/.gitignore

@@ -0,0 +1,14 @@
+node_modules/
+dist/
+*.tsbuildinfo
+.env
+.env.*
+__pycache__/
+*.pyc
+*.egg-info/
+build/
+.pytest_cache/
+facts.jsonl
+facts/
+*.log
+.DS_Store

factag-0.1.0/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.5
+Name: factag
+Version: 0.1.0
+Summary: Fact-Augmented Generation (FAG) — RAG-shaped load → split → extract → normalize → store → assemble pipeline for verified facts. Pure standard library.
+Project-URL: Homepage, https://github.com/factag-generator/fact-augmented-generation
+Project-URL: Repository, https://github.com/factag-generator/fact-augmented-generation
+Project-URL: Issues, https://github.com/factag-generator/fact-augmented-generation/issues
+Author: FAG contributors
+License: Apache-2.0
+Keywords: fact-augmented-generation,fact-extraction,fag,grounding,llm,rag-alternative,retrieval
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# factag (Python)
+
+Python twin of the TypeScript [`factag`](https://www.npmjs.com/package/factag) package — same stage names, same JSONL fact-store format, same `=== VERIFIED FACTS ===` context block. Zero dependencies, standard library only.
+
+```python
+import asyncio
+from factag import DirectoryLoader, HeuristicExtractor, RecursiveHeadingSplitter, MemoryFactStore, IngestionPipeline, BudgetAssembler
+
+async def main():
+    store = MemoryFactStore()
+    result = await IngestionPipeline(
+        loader=DirectoryLoader("./docs"),
+        splitter=RecursiveHeadingSplitter(chunk_size=800, chunk_overlap=120),
+        extractor=HeuristicExtractor(),
+        store=store,
+    ).run()
+    print(result)  # IngestResult(documents=..., chunks=..., raw=..., kept=...)
+
+    block = await BudgetAssembler(store, budget=4000).to_context("what does the business plan cost?")
+    print(block)
+    # === VERIFIED FACTS ===
+    # - (file:///.../pricing.md) importance=5 category=pricing Business plan costs $99 per month...
+    # === END VERIFIED FACTS ===
+
+asyncio.run(main())
+```
+
+Install: `pip install factag`
+
+Full documentation, design principles and the FAG spec: see the [repository README](https://github.com/factag-generator/fact-augmented-generation).

factag-0.1.0/README.md

@@ -0,0 +1,30 @@
+# factag (Python)
+
+Python twin of the TypeScript [`factag`](https://www.npmjs.com/package/factag) package — same stage names, same JSONL fact-store format, same `=== VERIFIED FACTS ===` context block. Zero dependencies, standard library only.
+
+```python
+import asyncio
+from factag import DirectoryLoader, HeuristicExtractor, RecursiveHeadingSplitter, MemoryFactStore, IngestionPipeline, BudgetAssembler
+
+async def main():
+    store = MemoryFactStore()
+    result = await IngestionPipeline(
+        loader=DirectoryLoader("./docs"),
+        splitter=RecursiveHeadingSplitter(chunk_size=800, chunk_overlap=120),
+        extractor=HeuristicExtractor(),
+        store=store,
+    ).run()
+    print(result)  # IngestResult(documents=..., chunks=..., raw=..., kept=...)
+
+    block = await BudgetAssembler(store, budget=4000).to_context("what does the business plan cost?")
+    print(block)
+    # === VERIFIED FACTS ===
+    # - (file:///.../pricing.md) importance=5 category=pricing Business plan costs $99 per month...
+    # === END VERIFIED FACTS ===
+
+asyncio.run(main())
+```
+
+Install: `pip install factag`
+
+Full documentation, design principles and the FAG spec: see the [repository README](https://github.com/factag-generator/fact-augmented-generation).

factag-0.1.0/factag/__init__.py

@@ -0,0 +1,64 @@
+"""factag — Fact-Augmented Generation.
+
+RAG-shaped pipeline: load → split → extract → normalize → store → assemble.
+Pure standard library. Zero keys required (HeuristicExtractor default).
+"""
+
+from .documents import Document, Fact, Metadata, source_url_of
+from .extractors import Extractor, HeuristicExtractor, LLMExtractor, parse_fact_json, DEFAULT_LLM_SYSTEM_PROMPT
+from .loaders import (
+    BaseLoader,
+    DirectoryLoader,
+    TextLoader,
+    URLLoader,
+    SitemapLoader,
+    html_to_text,
+    looks_like_html,
+)
+from .normalize import normalize_facts
+from .pipeline import IngestionPipeline, IngestResult
+from .retrievers import (
+    FACTS_FOOTER,
+    FACTS_HEADER,
+    BudgetAssembler,
+    KeywordRetriever,
+    fact_line,
+    to_context_block,
+)
+from .splitter import RecursiveHeadingSplitter, DEFAULT_SEPARATORS
+from .stores import FileFactStore, MemoryFactStore
+
+VERSION = "0.1.0"
+
+__all__ = [
+    "Document",
+    "Fact",
+    "Metadata",
+    "source_url_of",
+    "Extractor",
+    "HeuristicExtractor",
+    "LLMExtractor",
+    "parse_fact_json",
+    "DEFAULT_LLM_SYSTEM_PROMPT",
+    "BaseLoader",
+    "DirectoryLoader",
+    "TextLoader",
+    "URLLoader",
+    "SitemapLoader",
+    "html_to_text",
+    "looks_like_html",
+    "normalize_facts",
+    "IngestionPipeline",
+    "IngestResult",
+    "FACTS_HEADER",
+    "FACTS_FOOTER",
+    "BudgetAssembler",
+    "KeywordRetriever",
+    "fact_line",
+    "to_context_block",
+    "RecursiveHeadingSplitter",
+    "DEFAULT_SEPARATORS",
+    "FileFactStore",
+    "MemoryFactStore",
+    "VERSION",
+]

factag-0.1.0/factag/documents.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+Metadata = dict[str, Any]
+
+
+@dataclass
+class Document:
+    page_content: str
+    metadata: Metadata = field(default_factory=dict)
+
+
+@dataclass
+class Fact:
+    text: str
+    importance: int
+    category: str
+    source_url: str
+    chunk_id: str | None = None
+    metadata: Metadata = field(default_factory=dict)
+
+    def to_wire(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "text": self.text,
+            "importance": self.importance,
+            "category": self.category,
+            "sourceUrl": self.source_url,
+        }
+        if self.chunk_id:
+            out["chunkId"] = self.chunk_id
+        if self.metadata:
+            out["metadata"] = self.metadata
+        return out
+
+    @staticmethod
+    def from_wire(data: dict[str, Any]) -> "Fact":
+        return Fact(
+            text=str(data.get("text", "")),
+            importance=int(data.get("importance", 3)),
+            category=str(data.get("category", "fact")),
+            source_url=str(data.get("sourceUrl", "unknown")),
+            chunk_id=data.get("chunkId"),
+            metadata=data.get("metadata") or {},
+        )
+
+
+def source_url_of(doc: Document) -> str:
+    raw = doc.metadata.get("url") or doc.metadata.get("source")
+    if isinstance(raw, str) and raw:
+        return raw
+    return "unknown"

factag-0.1.0/factag/extractors.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import json
+import re
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+from .documents import Document, Fact, Metadata, source_url_of
+
+DEFAULT_LLM_SYSTEM_PROMPT = (
+    "You are a fact extraction engine. Extract atomic, verifiable, self-contained facts from the text. "
+    "Return ONLY a JSON array. Each element: "
+    '{"text": "...", "importance": 1-5, "category": "pricing|feature|limit|contact|policy|api|fact|date|entity|other"}. '
+    "Never invent facts. Never merge facts from different sentences. "
+    "Importance: 5 = must-have core fact, 3 = useful detail, 1 = trivia."
+)
+
+_SENT_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'\(])|\n+")
+
+
+class Extractor(Protocol):
+    def extract(self, doc: Document) -> list[Fact]: ...
+
+
+def coerce_importance(value: Any) -> int:
+    try:
+        v = int(value)
+    except (TypeError, ValueError):
+        return 3
+    return max(1, min(5, v))
+
+
+def parse_fact_json(raw: str, doc: Document, chunk_index: int = 0) -> list[Fact]:
+    text = raw.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
+        text = re.sub(r"\n?```$", "", text)
+    m = re.search(r"\[[\s\S]*\]", text)
+    if m:
+        text = m.group(0)
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(data, list):
+        return []
+    url = source_url_of(doc)
+    out: list[Fact] = []
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+        ftext = str(item.get("text", "")).strip()
+        if not ftext:
+            continue
+        out.append(
+            Fact(
+                text=ftext,
+                importance=coerce_importance(item.get("importance")),
+                category=str(item.get("category", "fact")).strip().lower() or "fact",
+                source_url=url,
+                chunk_id=f"{chunk_index}:::{ftext[:24]}",
+                metadata={"title": doc.metadata["title"]} if doc.metadata.get("title") else {},
+            )
+        )
+    return out
+
+
+@dataclass
+class HeuristicExtractor:
+    """Zero-key extractor: rule-based importance + category guess from the raw text."""
+
+    min_importance: int = 1
+    rules: list[tuple[int, re.Pattern[str]]] = field(
+        default_factory=lambda: [
+            (4, re.compile(r"\$\s?\d|\b\d+(\.\d+)?\s?(usd|eur|gbp|mb|gb|ms|s|rpm|tpm)\b|per month|per year|/mo|/yr", re.I)),
+            (3, re.compile(r"\b\d+(\.\d+)?%|\bfree\b|\bunlimited\b|\b\d+\s?(api|key|seat|user|request|token)", re.I)),
+            (2, re.compile(r"\bmust|required|requires|always|never|only|guarantee|warranty|support(ed)?\b", re.I)),
+            (2, re.compile(r"\b(https?://|www\.)\S+|[\w.+-]+@[\w-]+\.[a-z]{2,}", re.I)),
+            (1, re.compile(r"\b(available|launch(ed)?|founded|released|version)\b", re.I)),
+            (1, re.compile(r"\b\d+(\.\d+)?\b")),
+        ]
+    )
+
+    def _guess_category(self, text: str) -> str:
+        t = text.lower()
+        if re.search(r"\$|price|cost|plan|billing|invoice|[¥€£]", t):
+            return "pricing"
+        if re.search(r"@[\w.-]+|email|phone|address|contact", t):
+            return "contact"
+        if re.search(r"\b\d+\s?(rpm|tpm|qps|limit|quota|max|requests? per)\b", t):
+            return "limit"
+        if re.search(r"https?://|api|endpoint|sdk|library", t):
+            return "api"
+        if re.search(r"feature|dashboard|supports?|includes?", t):
+            return "feature"
+        if re.search(r"privacy|gdpr|terms|policy|retention", t):
+            return "policy"
+        if re.search(r"\d{4}-\d{2}-\d{2}|\b(19|20)\d{2}\b", t):
+            return "date"
+        return "fact"
+
+    def extract(self, doc: Document) -> list[Fact]:
+        url = source_url_of(doc)
+        out: list[Fact] = []
+        seen: set[str] = set()
+        for sentence in _SENT_SPLIT.split(doc.page_content):
+            s = sentence.strip()
+            if len(s) < 12:
+                continue
+            score = 0
+            for weight, pattern in self.rules:
+                if pattern.search(s):
+                    score += weight
+            if score == 0:
+                continue
+            importance = max(self.min_importance, (score + 3) // 2)
+            key = re.sub(r"[^a-z0-9]+", " ", s.lower()).strip()[:120]
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(
+                Fact(
+                    text=s,
+                    importance=min(5, importance),
+                    category=self._guess_category(s),
+                    source_url=url,
+                    chunk_id=f"0:::{key[:24]}",
+                    metadata={"title": doc.metadata["title"]} if doc.metadata.get("title") else {},
+                )
+            )
+        return out
+
+
+@dataclass
+class LLMExtractor:
+    """OpenAI-compatible /chat/completions extractor. Requires api_key + base_url."""
+
+    api_key: str
+    base_url: str
+    model: str = "gpt-4o-mini"
+    system_prompt: str = DEFAULT_LLM_SYSTEM_PROMPT
+    batch_size: int = 6
+    concurrency: int = 3
+    timeout: float = 60.0
+    max_retries: int = 3
+    temperature: float = 0.0
+    chunk_char_budget: int = 6000
+
+    def _call(self, texts: list[str]) -> list[Fact]:
+        joined = "\n\n".join(f"[chunk {i}]\n{t}" for i, t in enumerate(texts))
+        payload = json.dumps(
+            {
+                "model": self.model,
+                "temperature": self.temperature,
+                "messages": [
+                    {"role": "system", "content": self.system_prompt},
+                    {"role": "user", "content": f"Extract facts:\n\n{joined[: self.chunk_char_budget]}"},
+                ],
+            }
+        ).encode("utf-8")
+        url = self.base_url.rstrip("/") + "/chat/completions"
+        last_err: Exception | None = None
+        for attempt in range(self.max_retries):
+            req = urllib.request.Request(
+                url,
+                data=payload,
+                headers={"content-type": "application/json", "authorization": f"Bearer {self.api_key}"},
+                method="POST",
+            )
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:  # noqa: S310
+                    data = json.loads(resp.read().decode("utf-8"))
+                content = data["choices"][0]["message"]["content"]
+                return parse_fact_json(content, Document(page_content=joined), 0)
+            except Exception as exc:  # noqa: BLE001
+                last_err = exc
+                if attempt < self.max_retries - 1:
+                    import time
+
+                    time.sleep(2**attempt)
+                    continue
+                break
+        raise RuntimeError(f"LLM extraction failed: {last_err}")
+
+    def extract(self, doc: Document) -> list[Fact]:
+        text = doc.page_content
+        slices: list[str] = []
+        start = 0
+        while start < len(text):
+            end = min(len(text), start + self.chunk_char_budget)
+            if end < len(text):
+                nl = text.rfind("\n", start, end)
+                if nl > start + self.chunk_char_budget // 2:
+                    end = nl
+            slices.append(text[start:end])
+            start = end
+        all_facts: list[Fact] = []
+        with ThreadPoolExecutor(max_workers=self.concurrency) as pool:
+            for batch_facts in pool.map(self._call_slices, [slices[i : i + self.batch_size] for i in range(0, len(slices), self.batch_size)]):
+                all_facts.extend(batch_facts)
+        url = source_url_of(doc)
+        for i, fact in enumerate(all_facts):
+            fact.source_url = url
+            fact.chunk_id = fact.chunk_id or f"{i}:::{fact.text[:24]}"
+        return all_facts
+
+    def _call_slices(self, batch: list[str]) -> list[Fact]:
+        return self._call(batch)
+
+
+def chunks_of(text: str, size: int = 2000) -> list[str]:
+    return [text[i : i + size] for i in range(0, len(text), size)] or [text]

factag-0.1.0/factag/loaders.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+
+from .documents import Document, Metadata, source_url_of
+
+FACTAG_USER_AGENT = "FactAugmentedGeneration/0.1 (+https://github.com/factag-generator/factag)"
+
+_NAMED = {
+    "amp": "&", "lt": "<", "gt": ">", "quot": '"', "apos": "'", "nbsp": " ",
+    "ndash": "–", "mdash": "—", "hellip": "…", "copy": "©", "reg": "®",
+    "trade": "™", "euro": "€", "pound": "£", "yen": "¥", "middot": "·", "bull": "•",
+}
+
+_DROP_ALWAYS = re.compile(
+    r"<(script|style|head|title|noscript|template|svg|iframe|canvas|form|button|select|textarea)[^>]*>[\s\S]*?</\1\s*>",
+    re.I,
+)
+_DROP_ALWAYS_SELF = re.compile(r"<(hr|br|img|input|meta|link|source|track|wbr|col)\b[^>]*\/?>", re.I)
+_DROP_CHROME = re.compile(r"<(nav|footer|aside|header|dialog|menu)[^>]*>[\s\S]*?</\1\s*>", re.I)
+_COMMENTS = re.compile(r"<!--[\s\S]*?-->")
+_HEADING_OPEN = re.compile(r"<h([1-6])[^>]*>", re.I)
+_HEADING_CLOSE = re.compile(r"</h[1-6]\s*>", re.I)
+_LI_OPEN = re.compile(r"<li[^>]*>", re.I)
+_BLOCK_CLOSE = re.compile(r"</(p|div|section|article|ul|ol|table|tbody|thead|tr|blockquote|pre|figure|figcaption|details|summary|dl|dt|dd)\s*>", re.I)
+_BLOCK_OPEN = re.compile(
+    r"<(p|div|section|article|ul|ol|table|tbody|thead|tr|blockquote|pre|figure|figcaption|details|summary|dl|dt|dd|td|th|caption)\b[^>]*>",
+    re.I,
+)
+_ANY_TAG = re.compile(r"<[^>]+>")
+_TITLE = re.compile(r"<title[^>]*>([\s\S]*?)</title>", re.I)
+
+
+def decode_entities(text: str) -> str:
+    def hex_sub(m: re.Match[str]) -> str:
+        return chr(int(m.group(1), 16))
+
+    def dec_sub(m: re.Match[str]) -> str:
+        return chr(int(m.group(1)))
+
+    def name_sub(m: re.Match[str]) -> str:
+        return _NAMED.get(m.group(1).lower(), m.group(0))
+
+    text = re.sub(r"&#x([0-9a-f]+);", hex_sub, text, flags=re.I)
+    text = re.sub(r"&#(\d+);", dec_sub, text)
+    return re.sub(r"&([a-z]+);", name_sub, text, flags=re.I)
+
+
+def extract_title(html: str) -> str:
+    m = _TITLE.search(html)
+    return re.sub(r"\s+", " ", decode_entities(m.group(1))).strip() if m else ""
+
+
+def html_to_text(html: str, remove_chrome: bool = True) -> str:
+    out = _COMMENTS.sub("", html)
+    out = _DROP_ALWAYS.sub("", out)
+    out = _DROP_ALWAYS_SELF.sub(" ", out)
+    if remove_chrome:
+        out = _DROP_CHROME.sub("", out)
+    out = _HEADING_OPEN.sub(lambda m: "\n" + "#" * int(m.group(1)) + " ", out)
+    out = _HEADING_CLOSE.sub("\n", out)
+    out = _LI_OPEN.sub("\n- ", out)
+    out = _BLOCK_CLOSE.sub("\n", out)
+    out = _BLOCK_OPEN.sub("\n", out)
+    out = re.sub(r"<br\s*/?>", "\n", out, flags=re.I)
+    out = _ANY_TAG.sub("", out)
+    out = out.replace("&nbsp;", " ")
+    out = decode_entities(out)
+    lines = [re.sub(r"[ \t]+", " ", line).strip() for line in out.split("\n")]
+    out = "\n".join(lines)
+    out = re.sub(r"\n{3,}", "\n\n", out)
+    return out.strip()
+
+
+def looks_like_html(content_type: str | None, sample: str) -> bool:
+    ct = (content_type or "").lower()
+    if "html" in ct or "xhtml" in ct:
+        return True
+    if "markdown" in ct or "text/plain" in ct:
+        return False
+    return bool(re.match(r"\s*(<!doctype html|<html|<head|<body|<h1|<div|<p>)", sample, re.I))
+
+
+class BaseLoader(ABC):
+    @abstractmethod
+    def load(self) -> list[Document]:
+        raise NotImplementedError
+
+
+def _fetch(url: str, timeout: float = 15.0) -> tuple[bytes, str | None]:
+    import urllib.request
+
+    req = urllib.request.Request(url, headers={"user-agent": FACTAG_USER_AGENT})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:  # noqa: S310
+        return resp.read(), resp.headers.get("content-type")
+
+
+class TextLoader(BaseLoader):
+    def __init__(self, content: str, metadata: Metadata | None = None, fmt: str = "auto") -> None:
+        self.content = content
+        self.metadata = metadata or {}
+        self.fmt = fmt
+
+    def load(self) -> list[Document]:
+        is_html = self.fmt == "html" or (
+            self.fmt == "auto" and looks_like_html(None, self.content[:1000])
+        )
+        text = html_to_text(self.content) if is_html else self.content.strip()
+        meta: Metadata = {"source": "inline", "url": "inline", **self.metadata}
+        return [Document(page_content=text, metadata=meta)]
+
+
+class URLLoader(BaseLoader):
+    def __init__(self, url: str, metadata: Metadata | None = None, timeout: float = 15.0) -> None:
+        self.url = url
+        self.metadata = metadata or {}
+        self.timeout = timeout
+
+    def load(self) -> list[Document]:
+        body_bytes, content_type = _fetch(self.url, self.timeout)
+        body = body_bytes.decode("utf-8", errors="replace")
+        meta: Metadata = {"source": self.url, "url": self.url, **self.metadata}
+        if looks_like_html(content_type, body[:500]):
+            title = extract_title(body)
+            if title:
+                meta["title"] = title
+            return [Document(page_content=html_to_text(body), metadata=meta)]
+        return [Document(page_content=body.strip(), metadata=meta)]
+
+
+class SitemapLoader(BaseLoader):
+    def __init__(self, sitemap_url: str, max_pages: int = 50, timeout: float = 15.0) -> None:
+        self.sitemap_url = sitemap_url
+        self.max_pages = max_pages
+        self.timeout = timeout
+
+    def _discover(self, url: str, depth: int = 0) -> list[str]:
+        body_bytes, _ = _fetch(url, self.timeout)
+        xml = body_bytes.decode("utf-8", errors="replace")
+        locs = [m.strip() for m in re.findall(r"<loc>\s*([\s\S]*?)\s*</loc>", xml, re.I)]
+        if re.search(r"<sitemapindex[\s>]", xml, re.I) and depth < 2:
+            out: list[str] = []
+            for child in locs[:20]:
+                try:
+                    out.extend(self._discover(child, depth + 1))
+                except Exception:
+                    continue
+            return out
+        return locs
+
+    def load(self) -> list[Document]:
+        found = self._discover(self.sitemap_url)
+        urls = list(dict.fromkeys(u for u in found if re.match(r"^https?://", u, re.I)))[: self.max_pages]
+        docs: list[Document] = []
+        for url in urls:
+            try:
+                docs.extend(URLLoader(url, timeout=self.timeout).load())
+            except Exception:
+                continue
+        return docs
+
+
+class DirectoryLoader(BaseLoader):
+    def __init__(
+        self,
+        dir_path: str,
+        extensions: list[str] | None = None,
+        recursive: bool = True,
+        max_files: int = 500,
+        metadata: Metadata | None = None,
+    ) -> None:
+        self.dir_path = dir_path
+        self.extensions = [e.lower() for e in (extensions or [".md", ".markdown", ".txt", ".html", ".htm"])]
+        self.recursive = recursive
+        self.max_files = max_files
+        self.metadata = metadata or {}
+
+    def _walk(self, root: str) -> list[str]:
+        import os
+
+        out: list[str] = []
+        for dirpath, dirnames, filenames in os.walk(root):
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+            for name in filenames:
+                out.append(os.path.join(dirpath, name))
+                if len(out) >= self.max_files:
+                    return out
+            if not self.recursive:
+                break
+        return out
+
+    def load(self) -> list[Document]:
+        import os
+        import pathlib
+
+        target = os.path.abspath(self.dir_path)
+        if os.path.isfile(target):
+            files = [target]
+        else:
+            files = self._walk(target)
+        docs: list[Document] = []
+        for path in files:
+            ext = os.path.splitext(path)[1].lower()
+            if ext not in self.extensions:
+                continue
+            raw = pathlib.Path(path).read_text(encoding="utf-8", errors="replace")
+            content = html_to_text(raw) if ext in (".html", ".htm") else raw.strip()
+            if not content:
+                continue
+            file_url = pathlib.Path(path).as_uri()
+            meta: Metadata = {
+                "source": file_url,
+                "url": file_url,
+                "title": os.path.basename(path),
+                **self.metadata,
+            }
+            docs.append(Document(page_content=content, metadata=meta))
+        return docs
+
+
+def source_url_of_doc(doc: Document) -> str:
+    return source_url_of(doc)