notations-cli 0.1.0__tar.gz

notations_cli-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

notations_cli-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,31 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+
+      - name: Run tests
+        run: pytest

notations_cli-0.1.0/.gitignore

@@ -0,0 +1,15 @@
+# Generated output
+*_notations.html
+*_notations.json
+*_notations.md
+
+# Virtual environment
+.venv/
+
+# Python
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.python-version

notations_cli-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Takashi Ishida
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

notations_cli-0.1.0/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: notations-cli
+Version: 0.1.0
+Summary: Extract notation tables from arXiv papers using LLMs
+Project-URL: Homepage, https://github.com/takashiishida/notations-cli
+Project-URL: Repository, https://github.com/takashiishida/notations-cli
+Author: Takashi Ishida
+License-Expression: MIT
+License-File: LICENSE
+Keywords: arxiv,latex,llm,notation
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Requires-Dist: arxiv-to-prompt>=0.11.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: openai>=1.99.2
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="logo.png" width="200" />
+</p>
+
+<h1 align="center">notations-cli</h1>
+
+<p align="center">
+  <a href="https://github.com/takashiishida/notations-cli/actions/workflows/tests.yml"><img src="https://github.com/takashiishida/notations-cli/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
+  <a href="https://pypi.org/project/notations-cli/"><img src="https://img.shields.io/pypi/v/notations-cli?color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/notations-cli"><img src="https://img.shields.io/pepy/dt/notations-cli?color=orange" alt="Downloads"></a>
+  <a href="https://github.com/takashiishida/notations-cli/blob/main/LICENSE"><img src="https://img.shields.io/github/license/takashiishida/notations-cli?color=green" alt="License"></a>
+</p>
+
+Extract notation tables from arXiv papers using LLMs.
+
+## Installation
+
+```bash
+pip install notations-cli
+```
+
+Or with uv:
+
+```bash
+uv add notations-cli
+```
+
+## Usage
+
+```bash
+# From arXiv ID
+notations 2006.11239
+
+# From arXiv URL
+notations https://arxiv.org/abs/2006.11239
+
+# From local .tex files
+notations /path/to/tex/folder
+
+# With options
+notations 2006.11239 --model gpt-5.2-2025-12-11 --provider openai
+notations 2006.11239 --model anthropic/claude-sonnet-4.5 --provider openrouter
+notations 2006.11239 --output my_paper    # custom base name → .json/.html/.md
+notations 2006.11239 --terminal           # also print table to terminal
+notations 2006.11239 --no-comments        # strip LaTeX comments first
+notations 2006.11239 --no-expand-macros   # disable macro expansion
+notations 2006.11239 --no-filter-body     # keep all extracted notations
+
+# Re-render from existing JSON (no LLM call)
+notations 2006_11239_notations.json
+notations 2006_11239_notations.json -t    # also print to terminal
+```
+
+By default, LaTeX macro definitions (`\newcommand`, etc.) are expanded inline before processing. Use `--no-expand-macros` to disable this.
+
+Notations are also filtered by default to only include symbols that appear in the document body (`\begin{document}...\end{document}`), removing artifacts from preamble-only macro definitions. Use `--no-filter-body` to disable this.
+
+## Configuration
+
+Set your API key as an environment variable:
+
+```bash
+# For OpenAI
+export OPENAI_API_KEY=...
+
+# For OpenRouter
+export OPENROUTER_API_KEY=...
+```
+
+## Output
+
+Generates a self-contained HTML file with:
+- Paper metadata (title, authors, arXiv link)
+- Searchable notation table
+- LaTeX rendering via KaTeX
+
+## Development
+
+```bash
+git clone https://github.com/takashiishida/notations-cli.git
+cd notations-cli
+uv sync
+uv run pytest
+```

notations_cli-0.1.0/README.md

@@ -0,0 +1,84 @@
+<p align="center">
+  <img src="logo.png" width="200" />
+</p>
+
+<h1 align="center">notations-cli</h1>
+
+<p align="center">
+  <a href="https://github.com/takashiishida/notations-cli/actions/workflows/tests.yml"><img src="https://github.com/takashiishida/notations-cli/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
+  <a href="https://pypi.org/project/notations-cli/"><img src="https://img.shields.io/pypi/v/notations-cli?color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/notations-cli"><img src="https://img.shields.io/pepy/dt/notations-cli?color=orange" alt="Downloads"></a>
+  <a href="https://github.com/takashiishida/notations-cli/blob/main/LICENSE"><img src="https://img.shields.io/github/license/takashiishida/notations-cli?color=green" alt="License"></a>
+</p>
+
+Extract notation tables from arXiv papers using LLMs.
+
+## Installation
+
+```bash
+pip install notations-cli
+```
+
+Or with uv:
+
+```bash
+uv add notations-cli
+```
+
+## Usage
+
+```bash
+# From arXiv ID
+notations 2006.11239
+
+# From arXiv URL
+notations https://arxiv.org/abs/2006.11239
+
+# From local .tex files
+notations /path/to/tex/folder
+
+# With options
+notations 2006.11239 --model gpt-5.2-2025-12-11 --provider openai
+notations 2006.11239 --model anthropic/claude-sonnet-4.5 --provider openrouter
+notations 2006.11239 --output my_paper    # custom base name → .json/.html/.md
+notations 2006.11239 --terminal           # also print table to terminal
+notations 2006.11239 --no-comments        # strip LaTeX comments first
+notations 2006.11239 --no-expand-macros   # disable macro expansion
+notations 2006.11239 --no-filter-body     # keep all extracted notations
+
+# Re-render from existing JSON (no LLM call)
+notations 2006_11239_notations.json
+notations 2006_11239_notations.json -t    # also print to terminal
+```
+
+By default, LaTeX macro definitions (`\newcommand`, etc.) are expanded inline before processing. Use `--no-expand-macros` to disable this.
+
+Notations are also filtered by default to only include symbols that appear in the document body (`\begin{document}...\end{document}`), removing artifacts from preamble-only macro definitions. Use `--no-filter-body` to disable this.
+
+## Configuration
+
+Set your API key as an environment variable:
+
+```bash
+# For OpenAI
+export OPENAI_API_KEY=...
+
+# For OpenRouter
+export OPENROUTER_API_KEY=...
+```
+
+## Output
+
+Generates a self-contained HTML file with:
+- Paper metadata (title, authors, arXiv link)
+- Searchable notation table
+- LaTeX rendering via KaTeX
+
+## Development
+
+```bash
+git clone https://github.com/takashiishida/notations-cli.git
+cd notations-cli
+uv sync
+uv run pytest
+```

notations_cli-0.1.0/notations/__init__.py

@@ -0,0 +1,3 @@
+"""Extract notation tables from arXiv papers using LLMs."""
+
+__version__ = "0.1.0"

notations_cli-0.1.0/notations/cli.py

@@ -0,0 +1,185 @@
+"""Command-line interface for notations."""
+
+import json
+import os
+import sys
+
+import click
+from arxiv_to_prompt import process_latex_source
+
+from .extractor import SUPPORTED_OPENAI_MODELS, extract_notations, filter_notations_by_body, get_paper_metadata
+from .renderer import render_html, render_json, render_markdown, render_terminal
+
+
+@click.command(context_settings={"help_option_names": ["-h", "--help"]})
+@click.argument("source")
+@click.option(
+    "--model",
+    "-m",
+    default="gpt-5.2",
+    help=f"Model to use for extraction (default: gpt-5.2). OpenAI: {', '.join(SUPPORTED_OPENAI_MODELS)}. OpenRouter: any model.",
+)
+@click.option(
+    "--provider",
+    "-p",
+    type=click.Choice(["openai", "openrouter"]),
+    default="openai",
+    help="API provider (default: openai)",
+)
+@click.option(
+    "--terminal",
+    "-t",
+    is_flag=True,
+    help="Also print notation table to terminal",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    help="Base name for output files (default: <arxiv_id>_notations). Produces .json, .html, .md.",
+)
+@click.option(
+    "--no-comments",
+    is_flag=True,
+    help="Remove comments from LaTeX source before processing",
+)
+@click.option(
+    "--no-expand-macros",
+    is_flag=True,
+    help="Disable macro expansion (\\newcommand etc.) in LaTeX source",
+)
+@click.option(
+    "--no-filter-body",
+    is_flag=True,
+    help="Disable filtering notations by document body presence",
+)
+@click.option(
+    "--no-cache",
+    is_flag=True,
+    help="Disable caching of downloaded LaTeX source (re-download from arXiv)",
+)
+def main(source: str, model: str, provider: str, terminal: bool, output: str | None, no_comments: bool, no_expand_macros: bool, no_filter_body: bool, no_cache: bool):
+    """Extract notation table from an arXiv paper.
+
+    SOURCE can be:
+
+    \b
+      - arXiv ID (e.g., 2006.11239)
+      - arXiv URL (e.g., https://arxiv.org/abs/2006.11239)
+      - Local folder path containing .tex files
+      - A .json file from a previous extraction (skips LLM call)
+    """
+    # Check if source is a JSON file (re-render mode)
+    if source.endswith(".json") and os.path.isfile(source):
+        click.echo(f"Loading notations from {source}...", err=True)
+        try:
+            with open(source, encoding="utf-8") as f:
+                data = json.load(f)
+            notations = data["notations"]
+            metadata = data["metadata"]
+        except (json.JSONDecodeError, KeyError) as e:
+            click.echo(f"Error loading JSON: {e}", err=True)
+            sys.exit(1)
+
+        click.echo(f"Loaded {len(notations)} notations", err=True)
+
+        # Derive base name from JSON filename
+        base = output or os.path.splitext(source)[0]
+    else:
+        # Determine if source is local or arxiv
+        is_local = "/" in source and "arxiv.org" not in source
+
+        # Extract arxiv_id for metadata and default filename
+        if is_local:
+            arxiv_id = None
+            metadata = {
+                "title": "Local Paper",
+                "authors": [],
+                "arxiv_id": "local",
+            }
+        else:
+            # Clean up arxiv_id
+            arxiv_id = source
+            arxiv_id = arxiv_id.replace("https://arxiv.org/abs/", "")
+            arxiv_id = arxiv_id.replace("https://arxiv.org/pdf/", "")
+            arxiv_id = arxiv_id.replace(".pdf", "")
+
+            click.echo(f"Fetching metadata for arXiv:{arxiv_id}...", err=True)
+            metadata = get_paper_metadata(arxiv_id)
+            click.echo(f"Title: {metadata['title']}", err=True)
+
+        # Get LaTeX source
+        if is_local:
+            click.echo("Fetching LaTeX source...", err=True)
+        elif no_cache:
+            click.echo("Fetching LaTeX source (cache disabled)...", err=True)
+        else:
+            click.echo("Fetching LaTeX source (using cache if available)...", err=True)
+        try:
+            if is_local:
+                latex_source = process_latex_source(
+                    local_folder=source,
+                    keep_comments=not no_comments,
+                    expand_macros_flag=not no_expand_macros,
+                )
+            else:
+                latex_source = process_latex_source(
+                    arxiv_id=arxiv_id,
+                    keep_comments=not no_comments,
+                    expand_macros_flag=not no_expand_macros,
+                    use_cache=not no_cache,
+                )
+        except Exception as e:
+            click.echo(f"Error fetching LaTeX source: {e}", err=True)
+            sys.exit(1)
+
+        click.echo(f"LaTeX source: {len(latex_source)} characters", err=True)
+
+        # Extract notations using LLM
+        click.echo(f"Extracting notations using {model} via {provider}...", err=True)
+        try:
+            notations = extract_notations(
+                latex_source=latex_source,
+                model=model,
+                provider=provider,
+            )
+        except Exception as e:
+            click.echo(f"Error extracting notations: {e}", err=True)
+            sys.exit(1)
+
+        click.echo(f"Found {len(notations)} notations", err=True)
+
+        # Filter notations by document body presence
+        if not no_filter_body:
+            before = len(notations)
+            notations = filter_notations_by_body(notations, latex_source)
+            click.echo(f"Filtered to {len(notations)}/{before} notations (body presence)", err=True)
+
+        # Derive base name
+        base_id = arxiv_id if arxiv_id else "local"
+        base_id = base_id.replace("/", "_").replace(".", "_")
+        base = output or f"{base_id}_notations"
+
+    # Write all three output files
+    json_path = f"{base}.json"
+    html_path = f"{base}.html"
+    md_path = f"{base}.md"
+
+    with open(json_path, "w", encoding="utf-8") as f:
+        f.write(render_json(notations, metadata))
+    click.echo(f"Written to {json_path}", err=True)
+
+    with open(html_path, "w", encoding="utf-8") as f:
+        f.write(render_html(notations, metadata))
+    click.echo(f"Written to {html_path}", err=True)
+
+    with open(md_path, "w", encoding="utf-8") as f:
+        f.write(render_markdown(notations, metadata))
+    click.echo(f"Written to {md_path}", err=True)
+
+    if terminal:
+        render_terminal(notations, metadata)
+
+
+if __name__ == "__main__":
+    main()

notations_cli-0.1.0/notations/extractor.py

@@ -0,0 +1,176 @@
+"""LLM-based notation extraction from LaTeX source."""
+
+import json
+import os
+from openai import OpenAI
+
+# Supported OpenAI models for notation extraction
+SUPPORTED_OPENAI_MODELS = [
+    "gpt-5.2",
+    "gpt-5.1",
+    "gpt-5-mini",
+    "gpt-5-nano",
+]
+
+SYSTEM_PROMPT = """You are a notation extractor for academic papers. Given a LaTeX source, extract all mathematical notations used in the paper.
+
+For each notation, provide:
+- symbol: The LaTeX representation (e.g., "\\theta", "\\mathcal{D}", "x_i")
+- description: A concise description of what it represents
+
+Output a JSON array of objects with "symbol" and "description" fields.
+
+Rules:
+1. Include all meaningful mathematical symbols, variables, and operators defined or used in the paper
+2. Do not include standard LaTeX commands that aren't paper-specific notations
+3. For indexed variables like x_1, x_2, ..., x_n, just include the general form x_i or x_n
+4. Keep descriptions concise but informative
+5. If a symbol is used with different meanings in different contexts, create separate entries
+6. In descriptions, use $...$ for inline math (e.g., "Model parameters with dimension $d$")
+
+Output ONLY the JSON array, no other text."""
+
+
+def extract_notations(
+    latex_source: str,
+    model: str = "gpt-4o",
+    provider: str = "openai",
+) -> list[dict]:
+    """Extract notations from LaTeX source using an LLM.
+
+    Args:
+        latex_source: The LaTeX source code of the paper
+        model: The model to use for extraction
+        provider: The API provider ("openai" or "openrouter")
+
+    Returns:
+        List of notation dicts with "symbol" and "description" keys
+    """
+    if provider == "openai":
+        if model not in SUPPORTED_OPENAI_MODELS:
+            supported = ", ".join(SUPPORTED_OPENAI_MODELS)
+            raise ValueError(f"Unsupported OpenAI model: {model}. Supported: {supported}")
+        base_url = None
+        api_key = os.environ.get("OPENAI_API_KEY")
+    elif provider == "openrouter":
+        base_url = "https://openrouter.ai/api/v1"
+        api_key = os.environ.get("OPENROUTER_API_KEY")
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+
+    if not api_key:
+        env_var = "OPENAI_API_KEY" if provider == "openai" else "OPENROUTER_API_KEY"
+        raise ValueError(f"No API key provided. Set {env_var} or pass --api-key")
+
+    client = OpenAI(api_key=api_key, base_url=base_url)
+
+    # Check if it's a reasoning model (gpt-5 series, o1, o3, o4)
+    is_reasoning_model = any(x in model for x in ["gpt-5", "o1", "o3", "o4"])
+
+    if is_reasoning_model:
+        # Use Responses API for reasoning models
+        response = client.responses.create(
+            model=model,
+            instructions=SYSTEM_PROMPT,
+            input=latex_source,
+            reasoning={"effort": "low"},
+        )
+        content = response.output_text.strip()
+    else:
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": latex_source},
+            ],
+            temperature=0,
+        )
+        content = response.choices[0].message.content.strip()
+
+    # Handle markdown code blocks if present
+    if content.startswith("```"):
+        lines = content.split("\n")
+        content = "\n".join(lines[1:-1])
+
+    return json.loads(content)
+
+
+def filter_notations_by_body(
+    notations: list[dict], latex_source: str
+) -> list[dict]:
+    """Filter notations to only those whose symbol appears in the document body.
+
+    This removes notations extracted from preamble-only macro definitions
+    that were expanded away and don't appear in the actual document.
+
+    Args:
+        notations: List of notation dicts with "symbol" key
+        latex_source: Full LaTeX source including preamble
+
+    Returns:
+        Filtered list of notations
+    """
+    import re
+
+    begin = re.search(r"\\begin\{document\}", latex_source)
+    end = re.search(r"\\end\{document\}", latex_source)
+    if begin is None or end is None:
+        return notations
+
+    body = latex_source[begin.end() : end.start()]
+
+    filtered = []
+    for n in notations:
+        symbol = n.get("symbol", "")
+        # Strip $...$ wrappers if present
+        if symbol.startswith("$") and symbol.endswith("$"):
+            symbol = symbol[1:-1]
+        if symbol and symbol in body:
+            filtered.append(n)
+    return filtered
+
+
+def get_paper_metadata(arxiv_id: str) -> dict:
+    """Fetch basic metadata for an arXiv paper.
+
+    Args:
+        arxiv_id: The arXiv ID (e.g., "2006.11239")
+
+    Returns:
+        Dict with title, authors, and arxiv_id
+    """
+    import urllib.request
+    import xml.etree.ElementTree as ET
+
+    # Clean up arxiv_id
+    arxiv_id = arxiv_id.replace("https://arxiv.org/abs/", "")
+    arxiv_id = arxiv_id.replace("https://arxiv.org/pdf/", "")
+    arxiv_id = arxiv_id.replace(".pdf", "")
+
+    url = f"http://export.arxiv.org/api/query?id_list={arxiv_id}"
+
+    with urllib.request.urlopen(url) as response:
+        xml_data = response.read()
+
+    root = ET.fromstring(xml_data)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+
+    entry = root.find("atom:entry", ns)
+    if entry is None:
+        return {"title": "Unknown", "authors": [], "arxiv_id": arxiv_id}
+
+    title = entry.find("atom:title", ns)
+    title_text = title.text.strip().replace("\n", " ") if title is not None else "Unknown"
+
+    authors = entry.findall("atom:author", ns)
+    author_names = []
+    for author in authors:
+        name = author.find("atom:name", ns)
+        if name is not None:
+            author_names.append(name.text)
+
+    return {
+        "title": title_text,
+        "authors": author_names,
+        "arxiv_id": arxiv_id,
+    }