llm-api-scope 0.4.0__tar.gz → 0.5.1__tar.gz

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-api-scope
-Version: 0.4.0
+Version: 0.5.1
 Summary: read and cache structured documents from remote for LLM agents
 Author-email: D7x7z49 <85430783+D7x7z49@users.noreply.github.com>
 License: MIT License
@@ -73,9 +73,14 @@ browse OpenAPI specifications with subcommands for discovering, listing, and des
 
 aliases let you register frequently used specs once and reference them by short name. fetching is transparent — local copies are cached for fast repeat access, and a proxy can be configured for restricted networks.
 
+### rfc
+
+read, search, and navigate RFC documents from the IETF.
+
+the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
+
 ## future
 
-- read RFC documents by number
 - read academic papers from arxiv
 - more formal document formats as the need arises
 

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/README.md

@@ -24,9 +24,14 @@ browse OpenAPI specifications with subcommands for discovering, listing, and des
 
 aliases let you register frequently used specs once and reference them by short name. fetching is transparent — local copies are cached for fast repeat access, and a proxy can be configured for restricted networks.
 
+### rfc
+
+read, search, and navigate RFC documents from the IETF.
+
+the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
+
 ## future
 
-- read RFC documents by number
 - read academic papers from arxiv
 - more formal document formats as the need arises
 

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/apiscope/config.py

@@ -1,6 +1,7 @@
 # apiscope/config.py
 
 import json
+import time
 from contextlib import contextmanager
 from os import environ
 from pathlib import Path
@@ -27,13 +28,36 @@ CACHE_ROOT = DEFAULT_ROOT / "cache"
 # ==============================================================================
 
 
-class OpenapiConfig(BaseModel):
+class BaseConfig(BaseModel):
+    cache_ttl: int  # seconds
+
+    def is_stale_since(self, timestamp: float) -> bool:
+        return time.time() - timestamp > self.cache_ttl
+
+    def is_stale_path(self, path: Path) -> bool:
+        if not path.exists():
+            return True
+        stat = path.stat()
+        latest = max(stat.st_mtime, stat.st_ctime)
+        return self.is_stale_since(latest)
+
+
+class OpenapiConfig(BaseConfig):
     proxy: str | None = Field(default=None)
     alias: dict[str, str] = Field(default_factory=dict)
 
+    # override
+    cache_ttl: int = Field(default=60 * 60 * 24)  # 1 day
+
+
+class RfcConfig(BaseConfig):
+    # override
+    cache_ttl: int = Field(default=60 * 60 * 24 * 7)  # 1 week
+
 
 class Config(BaseModel):
     openapi: OpenapiConfig = Field(default_factory=OpenapiConfig)
+    rfc: RfcConfig = Field(default_factory=RfcConfig)
 
     @classmethod
     def read(cls, path: Path) -> "Config":

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/apiscope/main.py

@@ -12,10 +12,20 @@
 #   subcommand callbacks extend ctx.obj.extras with group-specific keys.
 #   schema.py exists solely to document the context shape — no runtime logic.
 
+import json
+
 import typer
 
-from apiscope.config import APP_NAME, DEFAULT_ROOT, get_config
+from apiscope.config import (
+    APP_NAME,
+    CACHE_ROOT,
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_ROOT,
+    get_config,
+    get_project_config_path,
+)
 from apiscope.openapi import openapi_app
+from apiscope.rfc import check_rfc_deps, rfc_app
 from apiscope.schema import CommandContext
 
 # ==============================================================================
@@ -46,6 +56,7 @@ def callback(ctx: typer.Context) -> None:
 # ==============================================================================
 
 app.add_typer(openapi_app, name="openapi")
+app.add_typer(rfc_app, name="rfc")
 
 # ==============================================================================
 # commands
@@ -53,11 +64,45 @@ app.add_typer(openapi_app, name="openapi")
 
 
 @app.command(help="check that apiscope is installed and working")
-def health(ctx: typer.Context) -> None:
-    cache_dir = DEFAULT_ROOT / "cache"
+def health(
+    ctx: typer.Context,
+    json_output: bool = typer.Option(False, "--json", help="output as JSON"),
+) -> None:
+    # ensure directories exist
     DEFAULT_ROOT.mkdir(parents=True, exist_ok=True)
-    cache_dir.mkdir(parents=True, exist_ok=True)
-    typer.echo(f"{APP_NAME} is healthy")
+    CACHE_ROOT.mkdir(parents=True, exist_ok=True)
+
+    # gather issues from all modules
+    issues: list[str] = []
+    for label, check in [("rfc", check_rfc_deps)]:
+        err = check()
+        if err is not None:
+            issues.append(f"[{label}] {err}")
+
+    # config paths
+    global_cfg = str(DEFAULT_CONFIG_PATH) if DEFAULT_CONFIG_PATH.exists() else None
+    project_path = get_project_config_path()
+    project_cfg = str(project_path) if project_path and project_path.exists() else None
+
+    if json_output:
+        result: dict = {
+            "home": str(DEFAULT_ROOT),
+            "config": {"global": global_cfg, "project": project_cfg},
+            "cache": str(CACHE_ROOT),
+            "issues": issues,
+        }
+        typer.echo(json.dumps(result, ensure_ascii=False))
+    else:
+        typer.echo(f"[home] <{DEFAULT_ROOT}>")
+        typer.echo(f"[config] global <{global_cfg or 'none'}>")
+        typer.echo(f"[config] project <{project_cfg or 'none'}>")
+        typer.echo(f"[cache] <{CACHE_ROOT}>")
+        typer.echo("---")
+        if issues:
+            for msg in issues:
+                typer.echo(f"[!] {msg}", err=True)
+            raise typer.Exit(code=1)
+        typer.echo(f"[+] {APP_NAME} is healthy")
 
 
 # ==============================================================================

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/apiscope/openapi/app.py

@@ -7,7 +7,7 @@ from typing import Any
 import typer
 
 from apiscope.config import CACHE_ROOT, Config
-from apiscope.openapi.fetch import fetch_openapi_spec
+from apiscope.openapi.fetch import fetch_openapi_spec, openapi_cache_path
 from apiscope.openapi.reader import HttpMethod, OpenapiReader
 from apiscope.openapi.schema import OpenapiCommandContext
 from apiscope.openapi.spec import spec_app
@@ -26,16 +26,20 @@ def _resolve_source(alias_or_source: str, config: Config) -> str:
     return alias_or_source
 
 
-def _load_reader(source: str, cache_dir: Path, proxy: str | None = None) -> OpenapiReader:
-    cached = fetch_openapi_spec(source, cache_dir, proxy)
+def _load_reader(
+    source: str, cache_dir: Path, proxy: str | None = None, refresh: bool = False
+) -> OpenapiReader:
+    cached = fetch_openapi_spec(source, cache_dir, proxy, refresh=refresh)
     return OpenapiReader.load(cached)
 
 
-def _get_reader(source: str, ctx: typer.Context) -> OpenapiReader:
+def _get_reader(source: str, ctx: typer.Context, force: bool = False) -> OpenapiReader:
     resolved = _resolve_source(source, ctx.obj.config)
     cache_dir = ctx.obj.openapi_command_context.cache_dir
     proxy = ctx.obj.config.openapi.proxy
-    return _load_reader(resolved, cache_dir, proxy)
+    cache_path = openapi_cache_path(resolved, cache_dir)
+    stale = ctx.obj.config.openapi.is_stale_path(cache_path)
+    return _load_reader(resolved, cache_dir, proxy, refresh=force or stale)
 
 
 # ==============================================================================
@@ -61,9 +65,9 @@ def list_operations(
     source: str = typer.Argument(help="alias or path to the OpenAPI spec"),
     tag: str | None = typer.Option(default=None, help="filter by tag"),
     method: str | None = typer.Option(default=None, help="filter by HTTP method"),
+    force: bool = typer.Option(False, "--force", help="force re-fetch ignoring cache"),
 ) -> None:
-    # load and resolve the spec
-    reader = _get_reader(source, ctx)
+    reader = _get_reader(source, ctx, force=force)
 
     # collect all operations (path + method + identity fields)
     operations: list[dict[str, Any]] = []
@@ -106,9 +110,9 @@ def describe_operation(
     request: bool = typer.Option(
         default=False, show_default=False, help="show only request fields, omit responses"
     ),
+    force: bool = typer.Option(False, "--force", help="force re-fetch ignoring cache"),
 ) -> None:
-    # load and resolve the spec
-    reader = _get_reader(source, ctx)
+    reader = _get_reader(source, ctx, force=force)
 
     # merge path-item parameters with operation parameters
     path_item = reader.paths.get(path, {})
@@ -132,8 +136,9 @@ def describe_operation(
 def show_info(
     ctx: typer.Context,
     source: str = typer.Argument(help="alias or path to the OpenAPI spec"),
+    force: bool = typer.Option(False, "--force", help="force re-fetch ignoring cache"),
 ) -> None:
-    reader = _get_reader(source, ctx)
+    reader = _get_reader(source, ctx, force=force)
 
     META_KEYS = ("openapi", "info", "servers", "tags", "security", "externalDocs")
     meta = {k: v for k, v in reader.raw.items() if k in META_KEYS}

llm_api_scope-0.5.1/apiscope/openapi/fetch.py

@@ -0,0 +1,58 @@
+# apiscope/openapi/fetch.py
+
+import shutil
+from hashlib import sha256
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+import httpx
+
+OPENAPI_EXTENSIONS = {".json", ".yaml", ".yml"}
+
+
+def _cache_key(source: str) -> str:
+    return sha256(source.encode()).hexdigest()
+
+
+def openapi_cache_path(source: str, cache_dir: Path) -> Path:
+    if "://" in source:
+        path = unquote(urlparse(source).path).rstrip()
+    else:
+        path = source
+    suffix = Path(path).suffix.lower()
+    ext = suffix if suffix in OPENAPI_EXTENSIONS else ".json"
+    return cache_dir / f"{_cache_key(source)}{ext}"
+
+
+def _fetch_local(source: str, cache_dir: Path, refresh: bool = False) -> Path:
+    cache_path = openapi_cache_path(source, cache_dir)
+
+    if refresh or not cache_path.exists():
+        src = Path(source).expanduser().resolve()
+        shutil.copy2(src, cache_path)
+
+    return cache_path
+
+
+def _fetch_remote(
+    url: str, cache_dir: Path, proxy: str | None = None, refresh: bool = False
+) -> Path:
+    cache_path = openapi_cache_path(url, cache_dir)
+
+    if refresh or not cache_path.exists():
+        client_kwargs: dict = {}
+        if proxy is not None:
+            client_kwargs["proxy"] = proxy
+        resp = httpx.get(url, follow_redirects=True, **client_kwargs)
+        resp.raise_for_status()
+        cache_path.write_bytes(resp.content)
+
+    return cache_path
+
+
+def fetch_openapi_spec(
+    source: str, cache_dir: Path, proxy: str | None = None, refresh: bool = False
+) -> Path:
+    if "://" in source:
+        return _fetch_remote(source, cache_dir, proxy, refresh=refresh)
+    return _fetch_local(source, cache_dir, refresh=refresh)

llm_api_scope-0.5.1/apiscope/rfc/__init__.py

@@ -0,0 +1,6 @@
+# apiscope/rfc/__init__.py
+
+from apiscope.rfc.app import app as rfc_app
+from apiscope.rfc.app import check_deps as check_rfc_deps
+
+__all__ = ["rfc_app", "check_rfc_deps"]

llm_api_scope-0.5.1/apiscope/rfc/app.py

@@ -0,0 +1,403 @@
+# apiscope/rfc/app.py
+
+import json
+import shutil
+
+import typer
+
+from apiscope.config import CACHE_ROOT
+from apiscope.rfc.fetch import fetch_all_index_json, fetch_content_by_number
+from apiscope.rfc.parse_xml import TocEntry
+from apiscope.rfc.schema import ContentFormat, RfcCommandContext, RfcMetadata, RfcStatus
+from apiscope.rfc.search import match_trigram, search_content
+
+app = typer.Typer(help="browse IETF RFC documents")
+
+
+# ==============================================================================
+# public helpers
+# ==============================================================================
+
+
+def check_deps() -> str | None:
+    if shutil.which("rsync") is None:
+        return "rsync is required but not found in PATH"
+    return None
+
+
+# ==============================================================================
+# callback
+# ==============================================================================
+
+
+@app.callback()
+def rfc_callback(ctx: typer.Context) -> None:
+    # check required external tools
+    err = check_deps()
+    if err is not None:
+        typer.echo(err, err=True)
+        raise typer.Exit(code=1)
+
+    # prepare cache directories
+    rfc_cache_dir = CACHE_ROOT / "rfc"
+    index_json_dir = rfc_cache_dir / "index"
+    content_xml_dir = rfc_cache_dir / "content" / "xml"
+    content_txt_dir = rfc_cache_dir / "content" / "txt"
+    index_json_dir.mkdir(parents=True, exist_ok=True)
+    content_xml_dir.mkdir(parents=True, exist_ok=True)
+    content_txt_dir.mkdir(parents=True, exist_ok=True)
+
+    # inject context for subcommands
+    ctx.obj.rfc_command_context = RfcCommandContext(
+        index_json_dir=index_json_dir,
+        content_xml_dir=content_xml_dir,
+        content_txt_dir=content_txt_dir,
+    )
+
+
+# ==============================================================================
+# helpers
+# ==============================================================================
+
+
+def _print_toc_entry(entry: TocEntry, indent: int = 0) -> None:
+    # root entry has no self line, skip directly to children
+    if entry.id:
+        prefix = "  " * indent
+        typer.echo(f"{prefix}{entry.id}. {entry.title}")
+    child_indent = indent + (1 if entry.id else 0)
+    for child in entry.children:
+        _print_toc_entry(child, child_indent)
+
+
+def _show_page_info(content: str, number: int, rfc_ctx: RfcCommandContext) -> None:
+    from apiscope.rfc.parse_txt import page_count
+
+    total = page_count(content)
+    path = rfc_ctx.get_content_txt_path(number)
+
+    typer.echo(f"[TXT] rfc {number}  |  pages: 1-{total}")
+    typer.echo("this document has no table of contents")
+    typer.echo("use --page <number> to read a specific page")
+    typer.echo(f"grep or ripgrep at <{path}>")
+
+
+# ==============================================================================
+# commands
+# ==============================================================================
+
+
+@app.command(name="sync", help="download RFC metadata index via rsync")
+def sync_index(
+    ctx: typer.Context,
+    force: bool = typer.Option(False, "--force", help="force re-sync ignoring cache TTL"),
+) -> None:
+    rfc_ctx = ctx.obj.rfc_command_context
+    rfc_config = ctx.obj.config.rfc
+    index_dir = rfc_ctx.index_json_dir
+
+    if not force and not rfc_config.is_stale_path(index_dir):
+        typer.echo("index is up to date")
+        return
+
+    typer.echo("syncing via rsync...")
+    err = fetch_all_index_json(index_dir)
+    if err is not None:
+        typer.echo(f"sync failed\n{err}", err=True)
+        raise typer.Exit(code=1)
+    typer.echo("done")
+
+
+@app.command(name="info", help="show RFC metadata")
+def show_info(
+    ctx: typer.Context,
+    number: int = typer.Argument(help="RFC number"),
+    json_output: bool = typer.Option(False, "--json", help="output as JSON"),
+) -> None:
+    rfc_ctx = ctx.obj.rfc_command_context
+    meta = rfc_ctx.get_index_json(number)
+    if meta is None:
+        typer.echo(f"rfc {number} not found", err=True)
+        raise typer.Exit(code=1)
+
+    # output raw JSON
+    if json_output:
+        typer.echo(meta.model_dump_json())
+        return
+
+    # human-readable entry
+    typer.echo(f"[RFC-{number}]")
+    for field_name, value in meta.to_info_data():
+        # null/empty → (null)
+        if value is None or (isinstance(value, str) and not value.strip()):
+            formatted = "(null)"
+        elif isinstance(value, list) and not value:
+            formatted = "(null)"
+        elif isinstance(value, list):
+            formatted = "[" + ", ".join(str(v) for v in value) + "]"
+        elif field_name == "abstract":
+            formatted = str(value).replace("\r\n", " ").replace("\n", " ")
+        else:
+            formatted = str(value)
+        typer.echo(f"{field_name}: {formatted}")
+
+
+@app.command(name="read", help="show RFC content or table of contents")
+def read_content(
+    ctx: typer.Context,
+    number: int = typer.Argument(help="RFC number"),
+    section: str | None = typer.Option(None, "--section", help="extract by section id (XML only)"),
+    page: int | None = typer.Option(None, "--page", help="extract by page number (TXT only)"),
+    json_output: bool = typer.Option(False, "--json", help="output as JSON"),
+    force: bool = typer.Option(False, "--force", help="force re-fetch ignoring cache TTL"),
+) -> None:
+    rfc_ctx = ctx.obj.rfc_command_context
+    rfc_config = ctx.obj.config.rfc
+    meta = rfc_ctx.get_index_json(number)
+    if meta is None:
+        typer.echo(f"rfc {number} not found", err=True)
+        raise typer.Exit(code=1)
+
+    # determine content format
+    if meta.is_xml_format:
+        fmt: ContentFormat = "xml"
+        content_path = rfc_ctx.get_content_xml_path(number)
+    elif meta.is_txt_format:
+        fmt = "txt"
+        content_path = rfc_ctx.get_content_txt_path(number)
+    else:
+        typer.echo(f"rfc {number} has no readable content format", err=True)
+        raise typer.Exit(code=1)
+
+    # ensure content is available locally
+    if force or rfc_config.is_stale_path(content_path):
+        typer.echo(f"fetching <{content_path.name}> via rsync...")
+        err = fetch_content_by_number(number, content_path.parent, fmt)
+        if err is not None:
+            typer.echo(f"failed to fetch content\n{err}", err=True)
+            raise typer.Exit(code=1)
+
+    content = content_path.read_text()
+
+    # dispatch by format
+    if section is not None and page is not None:
+        typer.echo("--section and --page are mutually exclusive", err=True)
+        raise typer.Exit(code=1)
+
+    if fmt == "xml":
+        if page is not None:
+            typer.echo("--page is not available for XML format", err=True)
+            raise typer.Exit(code=1)
+
+        from apiscope.rfc.parse_xml import parse_xml_section, parse_xml_toc
+
+        if section is not None:
+            entry = parse_xml_section(content, section)
+            if json_output:
+                typer.echo(entry.model_dump_json(indent=2))
+            elif entry.content is not None:
+                typer.echo(entry.content)
+            else:
+                _print_toc_entry(entry)
+        else:
+            tree = parse_xml_toc(content)
+            if json_output:
+                typer.echo(tree.model_dump_json(indent=2))
+            else:
+                _print_toc_entry(tree)
+        return
+
+    if fmt == "txt":
+        if section is not None:
+            typer.echo("--section is not available for TXT format", err=True)
+            typer.echo("use --page <number> to read a specific page", err=True)
+            raise typer.Exit(code=1)
+
+        from apiscope.rfc.parse_txt import extract_page
+
+        if page is not None:
+            try:
+                page_content = extract_page(content, page)
+            except ValueError:
+                typer.echo(f"page {page} not found in rfc {number}", err=True)
+                raise typer.Exit(code=1)
+            if json_output:
+                typer.echo(json.dumps({"page": page, "content": page_content}))
+            else:
+                typer.echo(page_content)
+        else:
+            _show_page_info(content, number, rfc_ctx)
+        return
+
+
+@app.command(name="search", help="search RFC index or fulltext")
+def search_rfc(
+    ctx: typer.Context,
+    number: int | None = typer.Argument(None, help="RFC number for fulltext search"),
+    term: str | None = typer.Argument(None, help="search term for fulltext search"),
+    status: list[str] | None = typer.Option(None, "--status", "-s", help="filter by RFC status"),
+    since: int | None = typer.Option(None, "--since", help="filter by start year"),
+    until: int | None = typer.Option(None, "--until", help="filter by end year"),
+    author: str | None = typer.Option(None, "--author", "-a", help="filter by author name"),
+    title: str | None = typer.Option(None, "--title", "-t", help="filter by title keyword"),
+    abstract: str | None = typer.Option(None, "--abstract", help="filter by abstract keyword"),
+    keywords_field: str | None = typer.Option(None, "--keywords", help="filter by keywords field"),
+    source: str | None = typer.Option(None, "--source", help="filter by source / working group"),
+    context: int = typer.Option(1, "--context", "-C", help="lines of context around match"),
+    no_snippet: bool = typer.Option(False, "--no-snippet", help="hide content snippet"),
+    limit: int = typer.Option(20, "--limit", help="max results shown"),
+    offset: int = typer.Option(0, "--offset", help="skip first N results"),
+    force: bool = typer.Option(False, "--force", help="force re-fetch ignoring cache TTL"),
+) -> None:
+    rfc_ctx = ctx.obj.rfc_command_context
+    rfc_config = ctx.obj.config.rfc
+    # step 1: dispatch mode
+    index_filters = [status, since, until, author, title, abstract, keywords_field, source]
+    fulltext_mode = number is not None and term is not None
+    index_mode = number is None and term is None
+
+    if not fulltext_mode and not index_mode:
+        typer.echo(
+            "provide both NUMBER and TERM for fulltext search, or neither for index search",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    if fulltext_mode and any(index_filters):
+        typer.echo(
+            "filter flags only valid in index mode (search without a number)",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    if index_mode and (context != 1 or no_snippet):
+        typer.echo(
+            "--context and --no-snippet only valid in fulltext mode (search with a number)",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    # step 2: fulltext search
+    if fulltext_mode:
+        assert number is not None and term is not None
+        meta = rfc_ctx.get_index_json(number)
+        if meta is None:
+            typer.echo(f"rfc {number} not found", err=True)
+            raise typer.Exit(code=1)
+
+        content_path = rfc_ctx.get_content_txt_path(number)
+        if force or rfc_config.is_stale_path(content_path):
+            typer.echo(f"fetching <{content_path.name}> via rsync...")
+            err = fetch_content_by_number(number, content_path.parent, "txt")
+            if err is not None:
+                typer.echo(f"failed to fetch content\n{err}", err=True)
+                raise typer.Exit(code=1)
+
+        if not content_path.exists():
+            typer.echo(
+                f"rfc {number} has no TXT content, fulltext search unavailable",
+                err=True,
+            )
+            typer.echo("try reading with --section instead", err=True)
+            raise typer.Exit(code=1)
+
+        content = content_path.read_text()
+        page_matches = search_content(content, term, context)
+
+        if not page_matches:
+            typer.echo(f"no matches for '{term}' in rfc {number}")
+            return
+
+        total_ft = len(page_matches)
+        sliced_ft = page_matches[offset : offset + limit]
+
+        for page, snippet in sliced_ft:
+            if no_snippet:
+                typer.echo(f"[RFC-{number}] page {page}")
+            else:
+                typer.echo(f"[RFC-{number}] page {page}")
+                typer.echo(snippet)
+                typer.echo()
+
+        if total_ft > offset + limit:
+            shown = offset + limit
+            typer.echo(f"... ({shown} of {total_ft} matches shown, use --offset {shown} for more)")
+        return
+
+    # step 3: index search — scan all RFC metadata
+    rfc_matches = []
+
+    # validate status flags
+    wanted_status: set[RfcStatus] = set()
+    if status:
+        for raw in status:
+            s = RfcStatus.from_string(raw)
+            if s is None:
+                valid = ", ".join(f"{m.name.lower()}({m.value})" for m in RfcStatus)
+                typer.echo(f"unknown status '{raw}'. available: {valid}", err=True)
+                raise typer.Exit(code=1)
+            wanted_status.add(s)
+
+    for f in sorted(rfc_ctx.index_json_dir.glob("rfc*.json"), reverse=True):
+        meta = RfcMetadata.from_json_file(f)
+
+        # status filter
+        if wanted_status and meta.status not in wanted_status:
+            continue
+
+        # date range filter
+        if since is not None or until is not None:
+            pub_date = meta.pub_date or ""
+            year_str = pub_date.split()[-1] if pub_date else ""
+            try:
+                year = int(year_str)
+            except (ValueError, IndexError):
+                year = 0
+            if since is not None and year < since:
+                continue
+            if until is not None and year > until:
+                continue
+
+        # string filters via trigram
+        if author and not any(match_trigram(a, author) for a in meta.authors if a):
+            continue
+        if title and not match_trigram(meta.title or "", title):
+            continue
+        if abstract and not match_trigram(meta.abstract or "", abstract):
+            continue
+        if keywords_field and not any(match_trigram(k, keywords_field) for k in meta.keywords if k):
+            continue
+        if source and not match_trigram(meta.source or "", source):
+            continue
+
+        rfc_matches.append(meta)
+
+    # step 4: print index results
+    if not rfc_matches:
+        typer.echo("no matching RFCs found")
+        return
+
+    total = len(rfc_matches)
+    sliced = rfc_matches[offset : offset + limit]
+
+    for meta in sliced:
+        doc_id = (meta.doc_id or "").replace("RFC", "RFC-")
+        authors = meta.authors[:3]
+        authors_str = ", ".join(authors)
+        if len(meta.authors) > 3:
+            authors_str += ", et al."
+
+        abstract_text = (meta.abstract or "").replace("\r\n", " ").replace("\n", " ")
+        abstract_line = abstract_text.split(". ")[0] if abstract_text else ""
+        if abstract_line and not abstract_line.endswith("."):
+            abstract_line += "."
+
+        typer.echo(f"[{doc_id}] {meta.title or ''}")
+        typer.echo(f"  {meta.status or ''} | {meta.pub_date or ''} | {authors_str}")
+        if abstract_line:
+            typer.echo(f"  {abstract_line}")
+        typer.echo()
+
+    if total > offset + limit:
+        shown = offset + limit
+        typer.echo(f"... ({shown} of {total} results shown, use --offset {shown} for more)")

llm_api_scope-0.5.1/apiscope/rfc/fetch.py

@@ -0,0 +1,31 @@
+# apiscope/rfc/fetch.py
+
+import subprocess
+from pathlib import Path
+
+from apiscope.rfc.schema import ContentFormat
+
+# rsync module sources
+RFC_RSYNC_HOST = "rsync.rfc-editor.org"
+RFC_INDEX_MODULE = "rfcs-json-only"
+RFC_CONTENT_MODULE = "rfcs"
+
+
+def _do_rsync(src: str, dst: str) -> str | None:
+    result = subprocess.run(
+        ["rsync", "-az", "--delete", src, f"{dst}/"],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        return result.stderr.strip()
+    return None
+
+
+def fetch_all_index_json(dst_dir: Path) -> str | None:
+    return _do_rsync(f"{RFC_RSYNC_HOST}::{RFC_INDEX_MODULE}", str(dst_dir))
+
+
+def fetch_content_by_number(number: int, dst_dir: Path, fmt: ContentFormat) -> str | None:
+    filename = f"rfc{number}.{fmt}"
+    return _do_rsync(f"{RFC_RSYNC_HOST}::{RFC_CONTENT_MODULE}/{filename}", str(dst_dir))

llm_api_scope-0.5.1/apiscope/rfc/parse_txt.py

@@ -0,0 +1,54 @@
+# apiscope/rfc/parse_txt.py
+
+from __future__ import annotations
+
+import re
+
+PAGE_RE = re.compile(r"\[Page (\d+)\]")
+
+
+# ==============================================================================
+# public api
+# ==============================================================================
+
+
+def page_count(content: str) -> int:
+    matches = PAGE_RE.findall(content)
+    return int(matches[-1]) if matches else 1
+
+
+def split_pages(content: str) -> list[tuple[int, int, int]]:
+    # returns [(page_number, start_line, end_line), ...]
+    lines = content.split("\n")
+    boundaries: list[tuple[int, int]] = []  # [(page, line_index), ...]
+
+    for i, line in enumerate(lines):
+        m = PAGE_RE.search(line)
+        if m:
+            boundaries.append((int(m.group(1)), i))
+
+    if not boundaries:
+        return [(1, 0, len(lines))]
+
+    result: list[tuple[int, int, int]] = []
+    for idx, (page_num, line_num) in enumerate(boundaries):
+        start = 0 if idx == 0 else boundaries[idx - 1][1] + 1
+        end = line_num + 1
+        result.append((page_num, start, end))
+
+    # last page from last marker to EOF
+    last_end = boundaries[-1][1] + 1
+    if last_end < len(lines):
+        # assume next sequential page
+        next_page = boundaries[-1][0] + 1
+        result.append((next_page, last_end, len(lines)))
+
+    return result
+
+
+def extract_page(content: str, page: int) -> str:
+    pages = split_pages(content)
+    for p, start, end in pages:
+        if p == page:
+            return "\n".join(content.split("\n")[start:end])
+    raise ValueError(f"page {page} not found")

llm_api_scope-0.5.1/apiscope/rfc/parse_xml.py

@@ -0,0 +1,93 @@
+# apiscope/rfc/parse_xml.py
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+
+from pydantic import BaseModel
+
+
+class TocEntry(BaseModel):
+    # normalized section id: "3.5.1" "a.1" etc.
+    id: str
+    title: str
+    # non-None = leaf node with body text, children empty
+    # None = internal node, see children
+    content: str | None = None
+    children: list[TocEntry] = []
+
+
+# ==============================================================================
+# helpers
+# ==============================================================================
+
+
+def _pn_to_id(pn: str) -> str:
+    # pn = xml attribute: "section-3.5.1" "section-appendix.a.1"
+    if pn.startswith("section-appendix."):
+        return pn.replace("section-appendix.", "", 1)
+    if pn.startswith("section-"):
+        return pn.replace("section-", "", 1)
+    return pn
+
+
+# ==============================================================================
+# public api
+# ==============================================================================
+
+
+def parse_xml_toc(content: str) -> TocEntry:
+    tree = ET.fromstring(content)
+    children: list[TocEntry] = []
+
+    for parent_tag in ("middle", "back"):
+        parent = tree.find(parent_tag)
+        if parent is not None:
+            _collect_xml_toc(parent.findall("section"), children)
+
+    return TocEntry(id="", title="", children=children)
+
+
+def _collect_xml_toc(sections: list[ET.Element], entries: list[TocEntry]) -> None:
+    for section in sections:
+        if section.get("toc") == "exclude":
+            continue
+
+        pn = section.get("pn", "")
+        section_id = _pn_to_id(pn)
+        name_elem = section.find("name")
+        title = "".join(name_elem.itertext()).strip() if name_elem is not None else ""
+
+        child_sections = section.findall("section")
+        children: list[TocEntry] = []
+        if child_sections:
+            _collect_xml_toc(child_sections, children)
+
+        entries.append(TocEntry(id=section_id, title=title, children=children))
+
+
+def parse_xml_section(content: str, section_id: str) -> TocEntry:
+    tree = ET.fromstring(content)
+
+    pn = f"section-{section_id}"
+    section = tree.find(f".//*[@pn='{pn}']")
+    if section is None:
+        raise ValueError(f"section '{section_id}' not found")
+
+    name_elem = section.find("name")
+    title = "".join(name_elem.itertext()).strip() if name_elem is not None else ""
+
+    child_sections = section.findall("section")
+    if child_sections:
+        children: list[TocEntry] = []
+        _collect_xml_toc(child_sections, children)
+        return TocEntry(id=section_id, title=title, children=children)
+
+    # leaf: collect text from <t> elements
+    texts: list[str] = []
+    for t in section.iter("t"):
+        text = "".join(t.itertext()).strip()
+        if text:
+            texts.append(text)
+    body = "\n\n".join(texts)
+    return TocEntry(id=section_id, title=title, content=body)

llm_api_scope-0.5.1/apiscope/rfc/schema.py

@@ -0,0 +1,130 @@
+# apiscope/rfc/schema.py
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel
+
+INFO_FIELD_ORDER: list[str] = [
+    "title",
+    "authors",
+    "pub_status",
+    "status",
+    "pub_date",
+    "source",
+    "abstract",
+    "page_count",
+    "doi",
+    "draft",
+    "see_also",
+    "errata_url",
+    "obsoletes",
+    "obsoleted_by",
+    "updates",
+    "updated_by",
+]
+
+ContentFormat = Literal["xml", "txt"]
+
+
+class RfcStatus(str, Enum):
+    PS = "PROPOSED STANDARD"
+    DS = "DRAFT STANDARD"
+    STD = "INTERNET STANDARD"
+    BCP = "BEST CURRENT PRACTICE"
+    INFO = "INFORMATIONAL"
+    EXP = "EXPERIMENTAL"
+    HIST = "HISTORIC"
+    NOT_ISSUED = "NOT ISSUED"
+    UNKNOWN = "UNKNOWN"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @classmethod
+    def from_string(cls, s: str) -> RfcStatus | None:
+        key = s.strip().upper()
+        for member in cls:
+            if member.name == key or member.value == key:
+                return member
+        return None
+
+
+class RfcMetadata(BaseModel):
+    doc_id: str | None = None
+    title: str | None = None
+    authors: list[str] = []
+    pub_status: RfcStatus | None = None
+    status: RfcStatus | None = None
+    pub_date: str | None = None
+    abstract: str | None = None
+    keywords: list[str] = []
+    format: list[str] = []
+    source: str | None = None
+    page_count: str | None = None
+    doi: str | None = None
+    draft: str | None = None
+    see_also: list[str] = []
+    errata_url: str | None = None
+    obsoletes: list[str] = []
+    obsoleted_by: list[str] = []
+    updates: list[str] = []
+    updated_by: list[str] = []
+
+    @classmethod
+    def from_json_file(cls, path: Path) -> RfcMetadata:
+        return cls.model_validate_json(path.read_text())
+
+    def to_info_data(self) -> list[tuple[str, object]]:
+        dumped = self.model_dump()
+        return [(f, dumped[f]) for f in INFO_FIELD_ORDER]
+
+    def is_format(self, fmt: str) -> bool:
+        f = fmt.lower()
+        return any(f == mf.lower() for mf in self.format)
+
+    @property
+    def is_xml_format(self) -> bool:
+        return self.is_format("xml")
+
+    @property
+    def is_txt_format(self) -> bool:
+        return (
+            self.is_format("txt") or self.is_format("ascii") or not any(f for f in self.format if f)
+        )
+
+
+class RfcCommandContext(BaseModel):
+    index_json_dir: Path
+    content_xml_dir: Path
+    content_txt_dir: Path
+
+    def get_index_json_path(self, number: int) -> Path:
+        return self.index_json_dir / f"rfc{number}.json"
+
+    def get_content_xml_path(self, number: int) -> Path:
+        return self.content_xml_dir / f"rfc{number}.xml"
+
+    def get_content_txt_path(self, number: int) -> Path:
+        return self.content_txt_dir / f"rfc{number}.txt"
+
+    def get_index_json(self, number: int) -> RfcMetadata | None:
+        path = self.get_index_json_path(number)
+        if not path.exists():
+            return None
+        return RfcMetadata.from_json_file(path)
+
+    def get_content_xml(self, number: int) -> str | None:
+        path = self.get_content_xml_path(number)
+        if not path.exists():
+            return None
+        return path.read_text()
+
+    def get_content_txt(self, number: int) -> str | None:
+        path = self.get_content_txt_path(number)
+        if not path.exists():
+            return None
+        return path.read_text()

llm_api_scope-0.5.1/apiscope/rfc/search.py

@@ -0,0 +1,60 @@
+# apiscope/rfc/search.py
+
+from __future__ import annotations
+
+from difflib import SequenceMatcher
+
+from apiscope.rfc.parse_txt import split_pages
+
+
+def match_trigram(text: str, keyword: str, threshold: float = 0.7) -> bool:
+    # check if keyword appears in text via trigram similarity
+    text_lower = text.lower()
+    kw_lower = keyword.lower()
+
+    if not kw_lower:
+        return False
+
+    if kw_lower in text_lower:
+        return True
+
+    # word-level fuzzy match for typos
+    for word in text_lower.split():
+        if len(word) < 3:
+            continue
+        if SequenceMatcher(None, kw_lower, word).ratio() >= threshold:
+            return True
+
+    return False
+
+
+def search_content(content: str, keyword: str, context: int = 1) -> list[tuple[int, str]]:
+    pages = split_pages(content)
+    lines = content.split("\n")
+    kw_lower = keyword.lower()
+    results: list[tuple[int, str]] = []
+
+    for page, start, end in pages:
+        page_lines = lines[start:end]
+        match_idx = -1
+
+        for i, line in enumerate(page_lines):
+            stripped = line.strip("\x0c").strip()
+            if kw_lower in stripped.lower():
+                match_idx = i
+                break
+
+        if match_idx < 0:
+            continue
+
+        # extract context lines
+        ctx_start = max(0, match_idx - context)
+        ctx_end = min(len(page_lines), match_idx + context + 1)
+        snippet_lines = page_lines[ctx_start:ctx_end]
+
+        # strip form feed characters from snippet
+        snippet = "\n".join(snippet_lines).replace("\x0c", "").strip()
+        if snippet:
+            results.append((page, snippet))
+
+    return results

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/apiscope/schema.py

@@ -4,8 +4,10 @@ from pydantic import BaseModel
 
 from apiscope.config import Config
 from apiscope.openapi.schema import OpenapiCommandContext
+from apiscope.rfc.schema import RfcCommandContext
 
 
 class CommandContext(BaseModel):
     config: Config
     openapi_command_context: OpenapiCommandContext | None = None
+    rfc_command_context: RfcCommandContext | None = None

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/llm_api_scope.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-api-scope
-Version: 0.4.0
+Version: 0.5.1
 Summary: read and cache structured documents from remote for LLM agents
 Author-email: D7x7z49 <85430783+D7x7z49@users.noreply.github.com>
 License: MIT License
@@ -73,9 +73,14 @@ browse OpenAPI specifications with subcommands for discovering, listing, and des
 
 aliases let you register frequently used specs once and reference them by short name. fetching is transparent — local copies are cached for fast repeat access, and a proxy can be configured for restricted networks.
 
+### rfc
+
+read, search, and navigate RFC documents from the IETF.
+
+the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
+
 ## future
 
-- read RFC documents by number
 - read academic papers from arxiv
 - more formal document formats as the need arises
 

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/llm_api_scope.egg-info/SOURCES.txt

@@ -13,6 +13,13 @@ apiscope/openapi/schema.py
 apiscope/openapi/spec/__init__.py
 apiscope/openapi/spec/app.py
 apiscope/openapi/spec/schema.py
+apiscope/rfc/__init__.py
+apiscope/rfc/app.py
+apiscope/rfc/fetch.py
+apiscope/rfc/parse_txt.py
+apiscope/rfc/parse_xml.py
+apiscope/rfc/schema.py
+apiscope/rfc/search.py
 llm_api_scope.egg-info/PKG-INFO
 llm_api_scope.egg-info/SOURCES.txt
 llm_api_scope.egg-info/dependency_links.txt

{llm_api_scope-0.4.0 → llm_api_scope-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "llm-api-scope"
-version = "0.4.0"
+version = "0.5.1"
 description = "read and cache structured documents from remote for LLM agents"
 readme = "README.md"
 license = { file = "LICENSE" }

llm_api_scope-0.4.0/apiscope/openapi/fetch.py

@@ -1,49 +0,0 @@
-# apiscope/openapi/fetch.py
-
-import shutil
-from hashlib import sha256
-from pathlib import Path
-from urllib.parse import unquote, urlparse
-
-import httpx
-
-OPENAPI_EXTENSIONS = {".json", ".yaml", ".yml"}
-
-
-def _cache_key(source: str) -> str:
-    return sha256(source.encode()).hexdigest()
-
-
-def _fetch_local(source: str, cache_dir: Path) -> Path:
-    suffix = Path(source).suffix.lower()
-    ext = suffix if suffix in OPENAPI_EXTENSIONS else ".json"
-    cache_path = cache_dir / f"{_cache_key(source)}{ext}"
-
-    if not cache_path.exists():
-        src = Path(source).expanduser().resolve()
-        shutil.copy2(src, cache_path)
-
-    return cache_path
-
-
-def _fetch_remote(url: str, cache_dir: Path, proxy: str | None = None) -> Path:
-    path = unquote(urlparse(url).path).rstrip()
-    suffix = Path(path).suffix.lower()
-    ext = suffix if suffix in OPENAPI_EXTENSIONS else ".json"
-    cache_path = cache_dir / f"{_cache_key(url)}{ext}"
-
-    if not cache_path.exists():
-        client_kwargs: dict = {}
-        if proxy is not None:
-            client_kwargs["proxy"] = proxy
-        resp = httpx.get(url, follow_redirects=True, **client_kwargs)
-        resp.raise_for_status()
-        cache_path.write_bytes(resp.content)
-
-    return cache_path
-
-
-def fetch_openapi_spec(source: str, cache_dir: Path, proxy: str | None = None) -> Path:
-    if "://" in source:
-        return _fetch_remote(source, cache_dir, proxy)
-    return _fetch_local(source, cache_dir)