llm-api-scope 0.5.1__tar.gz → 0.6.0__tar.gz

{llm_api_scope-0.5.1/llm_api_scope.egg-info → llm_api_scope-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-api-scope
-Version: 0.5.1
+Version: 0.6.0
 Summary: read and cache structured documents from remote for LLM agents
 Author-email: D7x7z49 <85430783+D7x7z49@users.noreply.github.com>
 License: MIT License
@@ -79,6 +79,12 @@ read, search, and navigate RFC documents from the IETF.
 
 the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
 
+### repo
+
+sync documentation directories from any git repository to a local cache.
+
+register repositories by URL with a target subdirectory and a ref — a branch, tag, or commit. the sync command clones with shallow depth, blobless filter, and sparse checkout so only the needed tree and files come over the wire. extracted docs land in the cache under a stable hash path, and a configurable TTL avoids redundant re-fetches. provider-agnostic, zero auth.
+
 ## future
 
 - read academic papers from arxiv

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/README.md

@@ -30,6 +30,12 @@ read, search, and navigate RFC documents from the IETF.
 
 the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
 
+### repo
+
+sync documentation directories from any git repository to a local cache.
+
+register repositories by URL with a target subdirectory and a ref — a branch, tag, or commit. the sync command clones with shallow depth, blobless filter, and sparse checkout so only the needed tree and files come over the wire. extracted docs land in the cache under a stable hash path, and a configurable TTL avoids redundant re-fetches. provider-agnostic, zero auth.
+
 ## future
 
 - read academic papers from arxiv

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/apiscope/config.py

@@ -23,6 +23,8 @@ DEFAULT_CONFIG_SCHEMA_PATH = DEFAULT_ROOT / "config.schema.json"
 
 CACHE_ROOT = DEFAULT_ROOT / "cache"
 
+TMP_ROOT = DEFAULT_ROOT / "tmp"
+
 # ==============================================================================
 # config model
 # ==============================================================================
@@ -51,6 +53,21 @@ class OpenapiConfig(BaseConfig):
 
 
 class RfcConfig(BaseConfig):
+    # override
+    cache_ttl: int = Field(default=60 * 60 * 24 * 30)  # 1 month
+
+
+class RepoEntryConfig(BaseModel):
+    dir: str
+
+    # format - branch:<branch>, tag:<tag>, commit:<short-hash>
+    # e.g. branch:main, tag:v1.0, commit:abc1234
+    target: str = Field(default="branch:main")
+
+
+class RepoConfig(BaseConfig):
+    entries: dict[str, RepoEntryConfig] = Field(default_factory=dict)
+
     # override
     cache_ttl: int = Field(default=60 * 60 * 24 * 7)  # 1 week
 
@@ -58,6 +75,7 @@ class RfcConfig(BaseConfig):
 class Config(BaseModel):
     openapi: OpenapiConfig = Field(default_factory=OpenapiConfig)
     rfc: RfcConfig = Field(default_factory=RfcConfig)
+    repo: RepoConfig = Field(default_factory=RepoConfig)
 
     @classmethod
     def read(cls, path: Path) -> "Config":
@@ -85,6 +103,7 @@ class Config(BaseModel):
         merged.openapi.alias |= other.openapi.alias
         if other.openapi.proxy is not None:
             merged.openapi.proxy = other.openapi.proxy
+        merged.repo.entries = self.repo.entries | other.repo.entries
         return merged
 
 

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/apiscope/main.py

@@ -25,6 +25,7 @@ from apiscope.config import (
     get_project_config_path,
 )
 from apiscope.openapi import openapi_app
+from apiscope.repo import check_repo_deps, repo_app
 from apiscope.rfc import check_rfc_deps, rfc_app
 from apiscope.schema import CommandContext
 
@@ -57,6 +58,7 @@ def callback(ctx: typer.Context) -> None:
 
 app.add_typer(openapi_app, name="openapi")
 app.add_typer(rfc_app, name="rfc")
+app.add_typer(repo_app, name="repo")
 
 # ==============================================================================
 # commands
@@ -74,7 +76,7 @@ def health(
 
     # gather issues from all modules
     issues: list[str] = []
-    for label, check in [("rfc", check_rfc_deps)]:
+    for label, check in [("rfc", check_rfc_deps), ("repo", check_repo_deps)]:
         err = check()
         if err is not None:
             issues.append(f"[{label}] {err}")

llm_api_scope-0.6.0/apiscope/repo/__init__.py

@@ -0,0 +1,6 @@
+# apiscope/repo/__init__.py
+
+from apiscope.repo.app import app as repo_app
+from apiscope.repo.app import check_deps as check_repo_deps
+
+__all__ = ["repo_app", "check_repo_deps"]

llm_api_scope-0.6.0/apiscope/repo/app.py

@@ -0,0 +1,152 @@
+# apiscope/repo/app.py
+
+import shutil
+from pathlib import Path
+
+import typer
+
+from apiscope.config import (
+    CACHE_ROOT,
+    DEFAULT_CONFIG_PATH,
+    TMP_ROOT,
+    Config,
+    RepoEntryConfig,
+    get_project_config_path,
+)
+from apiscope.repo.fetch import sync_entry
+from apiscope.repo.schema import RepoCommandContext, RepoEntry
+
+app = typer.Typer(help="sync documentation from git repositories")
+
+
+# ==============================================================================
+# public helpers
+# ==============================================================================
+
+
+def check_deps() -> str | None:
+    if shutil.which("git") is None:
+        return "git is required but not found in PATH"
+    return None
+
+
+# ==============================================================================
+# helpers
+# ==============================================================================
+
+
+def _resolve_target(global_flag: bool) -> Path:
+    if global_flag:
+        return DEFAULT_CONFIG_PATH
+    project = get_project_config_path()
+    if project is not None and project.exists():
+        return project
+    return DEFAULT_CONFIG_PATH
+
+
+# ==============================================================================
+# callback
+# ==============================================================================
+
+
+@app.callback()
+def repo_callback(
+    ctx: typer.Context,
+    global_flag: bool = typer.Option(
+        False, "--global", "-g", help="edit global config instead of project config"
+    ),
+) -> None:
+    # check required external tools
+    err = check_deps()
+    if err is not None:
+        typer.echo(err, err=True)
+        raise typer.Exit(code=1)
+
+    # prepare cache directories
+    repo_cache_dir = CACHE_ROOT / "repo"
+    repo_tmp_dir = TMP_ROOT
+    repo_cache_dir.mkdir(parents=True, exist_ok=True)
+    repo_tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    # inject context for subcommands
+    ctx.obj.repo_command_context = RepoCommandContext(
+        cache_dir=repo_cache_dir,
+        tmp_dir=repo_tmp_dir,
+        global_flag=global_flag,
+    )
+
+
+# ==============================================================================
+# commands
+# ==============================================================================
+
+
+@app.command(name="add", help="register a repo for syncing")
+def add_repo(
+    ctx: typer.Context,
+    url: str = typer.Argument(help="git clone URL"),
+    dir: str = typer.Argument(help="directory within repo to extract"),
+    target: str = typer.Option(
+        "branch:main", "--target", help="ref target (branch:, tag:, commit:)"
+    ),
+) -> None:
+    # duplicate check against merged entries
+    if url in ctx.obj.config.repo.entries:
+        typer.echo(f"<{url}> already registered", err=True)
+        raise typer.Exit(code=1)
+
+    target_path = _resolve_target(ctx.obj.repo_command_context.global_flag)
+    with Config.edit(target_path) as cfg:
+        cfg.repo.entries[url] = RepoEntryConfig(dir=dir, target=target)
+
+    typer.echo(f"registered <{url}>")
+
+
+@app.command(name="remove", help="remove a registered repo")
+def remove_repo(
+    ctx: typer.Context,
+    url: str = typer.Argument(help="git clone URL"),
+) -> None:
+    target_path = _resolve_target(ctx.obj.repo_command_context.global_flag)
+    with Config.edit(target_path) as cfg:
+        if url not in cfg.repo.entries:
+            typer.echo(f"<{url}> not registered", err=True)
+            raise typer.Exit(code=1)
+        del cfg.repo.entries[url]
+
+    typer.echo(f"removed <{url}>")
+
+
+@app.command(name="list", help="list registered repos")
+def list_repos(ctx: typer.Context) -> None:
+    entries = RepoEntry.from_config(ctx.obj.config.repo)
+    if not entries:
+        typer.echo("no repos registered")
+        return
+
+    cache_dir = ctx.obj.repo_command_context.cache_dir
+
+    for entry in entries:
+        doc_path = cache_dir / entry.sha256_id
+        typer.echo(f"- [{entry.url}] [{entry.target}@{entry.dir}] <{doc_path}>")
+
+
+@app.command(name="sync", help="sync all registered repos to cache")
+def sync_repos(
+    ctx: typer.Context,
+    force: bool = typer.Option(False, "--force", help="force re-sync ignoring cache TTL"),
+) -> None:
+    entries = RepoEntry.from_config(ctx.obj.config.repo)
+    if not entries:
+        typer.echo("no repos registered")
+        return
+
+    rc = ctx.obj.repo_command_context
+    ttl = ctx.obj.config.repo.cache_ttl
+
+    for entry in entries:
+        err = sync_entry(entry, rc.tmp_dir, rc.cache_dir, force=force, ttl=ttl)
+        if err is not None:
+            typer.echo(f"sync failed for <{entry.url}>: {err}", err=True)
+        else:
+            typer.echo(f"synced <{entry.url}>")

llm_api_scope-0.6.0/apiscope/repo/fetch.py

@@ -0,0 +1,132 @@
+# apiscope/repo/fetch.py
+
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from apiscope.repo.schema import RepoEntry
+
+# ==============================================================================
+# git helpers
+# ==============================================================================
+
+
+def _run_git(args: list[str], cwd: Path | None = None) -> str | None:
+    result = subprocess.run(
+        ["git", *args],
+        capture_output=True,
+        text=True,
+        cwd=cwd,
+    )
+    if result.returncode != 0:
+        return result.stderr.strip()
+    return None
+
+
+def _clone_repo(url: str, work_dir: Path, branch: str | None = None) -> str | None:
+    args = [
+        "clone",
+        "--depth=1",
+        "--filter=blob:none",
+        "--sparse",
+    ]
+    if branch is not None:
+        args.extend(["--branch", branch])
+    args.extend([url, str(work_dir)])
+    return _run_git(args)
+
+
+def _fetch_commit(work_dir: Path, commit: str) -> str | None:
+    err = _run_git(["fetch", "origin", "--depth=1", commit], cwd=work_dir)
+    if err is not None:
+        return err
+    return _run_git(["checkout", "FETCH_HEAD"], cwd=work_dir)
+
+
+def _sparse_set(work_dir: Path, directory: str) -> str | None:
+    return _run_git(["sparse-checkout", "set", directory], cwd=work_dir)
+
+
+# ==============================================================================
+# sync
+# ==============================================================================
+
+
+def _copy_docs(work_dir: Path, directory: str, dst: Path) -> str | None:
+    src = work_dir / directory
+    if not src.exists():
+        return f"directory [{directory}] not found in <{work_dir}>"
+    shutil.rmtree(dst, ignore_errors=True)
+    dst.mkdir(parents=True, exist_ok=True)
+    try:
+        shutil.copytree(src, dst, dirs_exist_ok=True)
+    except shutil.Error as e:
+        return str(e)
+    return None
+
+
+def sync_entry(
+    entry: RepoEntry,
+    tmp_dir: Path,
+    cache_dir: Path,
+    *,
+    force: bool = False,
+    ttl: int,
+) -> str | None:
+    work_dir = tmp_dir / entry.sha256_id
+    dst_dir = cache_dir / entry.sha256_id
+
+    # skip if cache is fresh (not stale) and not forced
+    if not force and not _is_stale(dst_dir, ttl):
+        return None
+
+    # cleanup previous state
+    shutil.rmtree(work_dir, ignore_errors=True)
+    shutil.rmtree(dst_dir, ignore_errors=True)
+
+    ref = entry.reference
+
+    if ref is not None and ref[0] == "commit":
+        # commit ref: clone default branch first, then fetch + checkout specific commit
+        ref_name = ref[1]
+        err = _clone_repo(entry.url, work_dir)
+        if err is not None:
+            return err
+        err = _fetch_commit(work_dir, ref_name)
+        if err is not None:
+            return err
+    else:
+        # branch or tag: --branch accepts both (tags checkout in detached HEAD)
+        branch = ref[1] if ref is not None else "main"
+        err = _clone_repo(entry.url, work_dir, branch=branch)
+        if err is not None:
+            return err
+
+    # sparse checkout target directory
+    err = _sparse_set(work_dir, entry.dir)
+    if err is not None:
+        return err
+
+    # copy extracted docs to cache
+    err = _copy_docs(work_dir, entry.dir, dst_dir)
+    if err is not None:
+        return err
+
+    # cleanup working directory
+    shutil.rmtree(work_dir, ignore_errors=True)
+
+    return None
+
+
+# ==============================================================================
+# freshness
+# ==============================================================================
+
+
+def _is_stale(path: Path, ttl: int) -> bool:
+    if not path.exists():
+        return True
+    stat = path.stat()
+    latest = max(stat.st_mtime, stat.st_ctime)
+    return time.time() - latest > ttl

llm_api_scope-0.6.0/apiscope/repo/schema.py

@@ -0,0 +1,53 @@
+# apiscope/repo/schema.py
+
+import hashlib
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel
+
+if TYPE_CHECKING:
+    from apiscope.config import RepoConfig
+
+
+class RepoEntry(BaseModel):
+    url: str
+    dir: str
+    target: str
+
+    def __hash__(self) -> int:
+        return hash(self.sha256_id)
+
+    @property
+    def sha256_id(self) -> str:
+        # first 8 chars of sha256 hex digest; sufficient for collision resistance
+        return hashlib.sha256(self.url.encode()).hexdigest()[:8]
+
+    @property
+    def reference(self) -> tuple[str, str] | None:
+        if self.target.startswith("branch:"):
+            return "branch", self.target[len("branch:") :]
+        elif self.target.startswith("tag:"):
+            return "tag", self.target[len("tag:") :]
+        elif self.target.startswith("commit:"):
+            return "commit", self.target[len("commit:") :]
+        else:
+            return None
+
+    @classmethod
+    def from_config(cls, config: "RepoConfig") -> set["RepoEntry"]:
+        return {cls(url=url, dir=ec.dir, target=ec.target) for url, ec in config.entries.items()}
+
+
+# ==============================================================================
+# context
+# ==============================================================================
+
+
+class RepoCommandContext(BaseModel):
+    # computed properties
+    cache_dir: Path
+    tmp_dir: Path
+
+    # sub command options
+    global_flag: bool

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/apiscope/schema.py

@@ -4,6 +4,7 @@ from pydantic import BaseModel
 
 from apiscope.config import Config
 from apiscope.openapi.schema import OpenapiCommandContext
+from apiscope.repo.schema import RepoCommandContext
 from apiscope.rfc.schema import RfcCommandContext
 
 
@@ -11,3 +12,4 @@ class CommandContext(BaseModel):
     config: Config
     openapi_command_context: OpenapiCommandContext | None = None
     rfc_command_context: RfcCommandContext | None = None
+    repo_command_context: RepoCommandContext | None = None

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0/llm_api_scope.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-api-scope
-Version: 0.5.1
+Version: 0.6.0
 Summary: read and cache structured documents from remote for LLM agents
 Author-email: D7x7z49 <85430783+D7x7z49@users.noreply.github.com>
 License: MIT License
@@ -79,6 +79,12 @@ read, search, and navigate RFC documents from the IETF.
 
 the metadata index is mirrored once via rsync. individual text files are fetched on demand and cached locally. you can browse the table of contents, jump to a section (XML) or page (TXT), filter the index by status or source, and run keyword searches against fulltext content.
 
+### repo
+
+sync documentation directories from any git repository to a local cache.
+
+register repositories by URL with a target subdirectory and a ref — a branch, tag, or commit. the sync command clones with shallow depth, blobless filter, and sparse checkout so only the needed tree and files come over the wire. extracted docs land in the cache under a stable hash path, and a configurable TTL avoids redundant re-fetches. provider-agnostic, zero auth.
+
 ## future
 
 - read academic papers from arxiv

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/llm_api_scope.egg-info/SOURCES.txt

@@ -13,6 +13,10 @@ apiscope/openapi/schema.py
 apiscope/openapi/spec/__init__.py
 apiscope/openapi/spec/app.py
 apiscope/openapi/spec/schema.py
+apiscope/repo/__init__.py
+apiscope/repo/app.py
+apiscope/repo/fetch.py
+apiscope/repo/schema.py
 apiscope/rfc/__init__.py
 apiscope/rfc/app.py
 apiscope/rfc/fetch.py

{llm_api_scope-0.5.1 → llm_api_scope-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "llm-api-scope"
-version = "0.5.1"
+version = "0.6.0"
 description = "read and cache structured documents from remote for LLM agents"
 readme = "README.md"
 license = { file = "LICENSE" }