arxiv-ingest 0.2.0__tar.gz

arxiv_ingest-0.2.0/.claude/commands/arxiv-ingest.md

@@ -0,0 +1,66 @@
+# /arxiv-ingest
+
+arXivから論文を収集し、research-wikiへ自動取り込みするワークフロー。
+
+## 実行ステップ
+
+### Step 1: 論文を収集する
+
+```bash
+cd ~/workspace/arxiv-ingest
+uv run python scripts/fetch.py
+```
+
+`data/fetched.json` に論文リストが保存される。
+
+### Step 2: スケルトンファイルを生成する
+
+```bash
+uv run python scripts/generate.py
+```
+
+`research-wiki/sources/{Category}/{slug}.md`（メタデータ）と
+`research-wiki/evidence/{Category}/{slug}.md`（空のテンプレート）と
+`research-wiki/wiki/papers/{Category}/{slug}.md`（空のテンプレート）が作成される。
+
+### Step 3: 各論文の evidence と wiki を Claude が記入する
+
+`data/fetched.json` を読み込み、各論文について以下を実行する：
+
+1. 論文のアブストラクト（`data/fetched.json` の `abstract` フィールド）を読む
+2. PDF URL（`pdf_url` フィールド）から論文を取得して内容を把握する（可能な場合）
+3. `research-wiki/evidence/{Category}/{slug}.md` を更新する：
+   - 主要な主張（3〜5点）
+   - 主要な貢献
+   - 制限・注意点
+   - ベンチマーク結果（記載があれば）
+4. `research-wiki/wiki/papers/{Category}/{slug}.md` を更新する：
+   - ソースからの事実（evidence の要点）
+   - 現時点の解釈（手法の意義・他研究との関係）
+   - 関連ページ（既存の wiki ページへのリンク）
+   - 未解決の問い
+
+### Step 4: インデックスを更新する
+
+`research-wiki/wiki/index.md` と `research-wiki/index/recent.md` に追加した論文を記録する。
+
+`research-wiki/log.md` に以下の形式で追記する：
+
+```
+## {今日の日付}
+
+- **論文追加**: {タイトル} ({著者第一著者} et al., {年})
+  - `sources/{Category}/{slug}.md` 作成
+  - `evidence/{Category}/{slug}.md` 作成
+  - `wiki/papers/{Category}/{slug}.md` 作成
+```
+
+## 設定変更
+
+キーワードや出力先は `~/workspace/arxiv-ingest/config.yaml` を編集する。
+
+## 注意
+
+- `evidence/` と `wiki/` のテンプレートに `（Claude Code が論文を読んで記入）` と書いてある箇所をすべて実際の内容に書き換えること
+- アブストラクトだけで十分に理解できる論文は PDF 取得を省略してよい
+- 既存のファイルが存在する場合は上書きせず、差分を確認してから更新する

arxiv_ingest-0.2.0/.github/workflows/daily.yml

@@ -0,0 +1,76 @@
+name: Daily arXiv Ingest
+
+on:
+  schedule:
+    - cron: '0 1 * * 1-5'  # 月〜金 JST 10:00 (UTC 01:00)
+  workflow_dispatch:
+    inputs:
+      days_back:
+        description: 'Fetch papers from last N days'
+        default: '1'
+        required: false
+
+jobs:
+  ingest:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout arxiv-ingest
+        uses: actions/checkout@v4
+
+      - name: Checkout research-wiki
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ vars.WIKI_REPO }}   # 例: yamadan96/research-wiki
+          token: ${{ secrets.GH_PAT }}
+          path: research-wiki
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Override days_back (manual trigger)
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          python - <<'EOF'
+          import yaml, re, sys
+          path = "config.yaml"
+          text = open(path).read()
+          text = re.sub(r'^days_back:.*$', f'days_back: ${{ inputs.days_back }}', text, flags=re.MULTILINE)
+          open(path, "w").write(text)
+          EOF
+
+      - name: Fetch papers from arXiv
+        run: uv run python scripts/fetch.py
+
+      - name: Generate research-wiki skeletons
+        env:
+          OUTPUT_DIR: research-wiki
+        run: |
+          python - <<'EOF'
+          import yaml, pathlib
+          cfg = yaml.safe_load(open("config.yaml"))
+          cfg["output_dir"] = "research-wiki"
+          open("config.yaml", "w").write(yaml.dump(cfg, allow_unicode=True))
+          EOF
+          uv run python scripts/generate.py
+
+      - name: Commit new files to research-wiki
+        working-directory: research-wiki
+        run: |
+          git config user.name "arxiv-ingest[bot]"
+          git config user.email "arxiv-ingest@users.noreply.github.com"
+          git add -A
+          if git diff --staged --quiet; then
+            echo "No new papers to commit."
+          else
+            COUNT=$(git diff --staged --name-only | grep '^sources/' | wc -l | tr -d ' ')
+            git commit -m "auto: arXiv ingest $(date +%Y-%m-%d) [${COUNT} papers]"
+            git push
+          fi

arxiv_ingest-0.2.0/.gitignore

@@ -0,0 +1,5 @@
+.venv/
+__pycache__/
+*.pyc
+data/fetched.json
+.python-version

arxiv_ingest-0.2.0/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: arxiv-ingest
+Version: 0.2.0
+Summary: Fetch recent arXiv papers and scaffold them into a structured research wiki
+Project-URL: Homepage, https://github.com/yamadan96/arxiv-ingest
+Project-URL: Repository, https://github.com/yamadan96/arxiv-ingest
+License: MIT
+Keywords: arxiv,knowledge-management,papers,research,wiki
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: arxiv>=4.0.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: rich>=15.0.0
+Description-Content-Type: text/markdown
+
+# arxiv-ingest
+
+**Fetch recent arXiv papers by keyword and scaffold them into structured research notes.**
+
+Works standalone as a CLI, or integrates with Claude Code for LLM-assisted evidence extraction and wiki generation.
+
+## Features
+
+- **Keyword-based collection** — list topics in `config.yaml`, no code needed
+- **3-layer output** — `sources/` (metadata) · `evidence/` (claims) · `wiki/` (synthesis)
+- **Safe re-runs** — already-filled files are never overwritten
+- **GitHub Actions template** — auto-runs Mon–Fri, `workflow_dispatch` for on-demand runs
+- **Any wiki** — point `output_dir` at any directory
+
+## Quick start
+
+```bash
+# Install (requires Python 3.11+)
+git clone https://github.com/yamadan96/arxiv-ingest
+cd arxiv-ingest
+uv sync                         # or: pip install -e .
+
+# Configure
+cp config.yaml.example config.yaml
+# Edit config.yaml: set keywords, output_dir, and category mappings
+
+# Fetch papers and generate skeletons
+arxiv-ingest run                # fetch + generate in one step
+
+# Or step by step:
+arxiv-ingest fetch              # saves data/fetched.json
+arxiv-ingest generate           # creates skeleton files in output_dir
+```
+
+## Configuration (`config.yaml`)
+
+```yaml
+output_dir: "../research-wiki"  # path to your wiki directory
+max_results: 20                 # papers per keyword
+days_back: 7                    # look back N days
+
+keywords:
+  - "Vision-Language Model transformer"
+  - "LoRA PEFT fine-tuning language model"
+  - "LLM reasoning chain-of-thought"
+
+# Only accept papers in these arXiv categories:
+allowed_arxiv_categories: [cs.CV, cs.CL, cs.LG, cs.AI, cs.RO, stat.ML]
+
+# Primary category must be in this list (prevents off-topic papers):
+require_primary_in: [cs.CV, cs.CL, cs.AI, cs.RO, stat.ML]
+
+# arXiv category → wiki folder name:
+category_map:
+  cs.CV: "Multimodal"
+  cs.CL: "Post_Training"
+  cs.AI: "Reasoning"
+  cs.RO: "Physical_AI"
+  stat.ML: "Pretraining"
+```
+
+See `config.yaml.example` for the full reference with all options documented.
+
+## Output format
+
+Skeleton files are created at:
+
+```
+output_dir/
+├── sources/{Category}/{slug}.md      # Metadata (title, authors, abstract, URL)
+├── evidence/{Category}/{slug}.md     # Claims & benchmarks — fill this in
+└── wiki/papers/{Category}/{slug}.md  # Synthesis & interpretation — fill this in
+```
+
+`sources/` files are written once and never touched again.
+`evidence/` and `wiki/` files are only (re-)created when they still contain the unfilled template placeholder, so your edits are safe across re-runs.
+
+## Using with Claude Code
+
+Copy the skill file so `/arxiv-ingest` is available as a Claude Code command:
+
+```bash
+cp .claude/commands/arxiv-ingest.md ~/.claude/commands/
+# or into your project's .claude/commands/
+```
+
+After running `arxiv-ingest run`, invoke `/arxiv-ingest` in Claude Code.
+Claude reads each paper's abstract (and PDF when available) and fills in the `evidence/` and `wiki/` files automatically.
+
+## GitHub Actions: daily auto-ingest
+
+### Setup
+
+1. Fork or use this repo as a template
+2. Create a GitHub Fine-grained PAT with **Write** access to your wiki repo
+3. Add to your repo's **Settings → Secrets and variables → Actions**:
+
+   | Kind | Name | Value |
+   |------|------|-------|
+   | Secret | `GH_PAT` | your PAT |
+   | Variable | `WIKI_REPO` | `your-name/research-wiki` |
+
+4. Go to **Actions → Daily arXiv Ingest → Run workflow** to verify
+
+### Schedule
+
+- **Auto**: Mon–Fri UTC 01:00 (JST 10:00) — `cron: '0 1 * * 1-5'`
+- **Manual**: `workflow_dispatch` with optional `days_back` override
+
+## Requirements
+
+- Python 3.11+
+- [uv](https://docs.astral.sh/uv/) (recommended) or pip
+
+## License
+
+MIT

arxiv_ingest-0.2.0/README.md

@@ -0,0 +1,117 @@
+# arxiv-ingest
+
+**Fetch recent arXiv papers by keyword and scaffold them into structured research notes.**
+
+Works standalone as a CLI, or integrates with Claude Code for LLM-assisted evidence extraction and wiki generation.
+
+## Features
+
+- **Keyword-based collection** — list topics in `config.yaml`, no code needed
+- **3-layer output** — `sources/` (metadata) · `evidence/` (claims) · `wiki/` (synthesis)
+- **Safe re-runs** — already-filled files are never overwritten
+- **GitHub Actions template** — auto-runs Mon–Fri, `workflow_dispatch` for on-demand runs
+- **Any wiki** — point `output_dir` at any directory
+
+## Quick start
+
+```bash
+# Install (requires Python 3.11+)
+git clone https://github.com/yamadan96/arxiv-ingest
+cd arxiv-ingest
+uv sync                         # or: pip install -e .
+
+# Configure
+cp config.yaml.example config.yaml
+# Edit config.yaml: set keywords, output_dir, and category mappings
+
+# Fetch papers and generate skeletons
+arxiv-ingest run                # fetch + generate in one step
+
+# Or step by step:
+arxiv-ingest fetch              # saves data/fetched.json
+arxiv-ingest generate           # creates skeleton files in output_dir
+```
+
+## Configuration (`config.yaml`)
+
+```yaml
+output_dir: "../research-wiki"  # path to your wiki directory
+max_results: 20                 # papers per keyword
+days_back: 7                    # look back N days
+
+keywords:
+  - "Vision-Language Model transformer"
+  - "LoRA PEFT fine-tuning language model"
+  - "LLM reasoning chain-of-thought"
+
+# Only accept papers in these arXiv categories:
+allowed_arxiv_categories: [cs.CV, cs.CL, cs.LG, cs.AI, cs.RO, stat.ML]
+
+# Primary category must be in this list (prevents off-topic papers):
+require_primary_in: [cs.CV, cs.CL, cs.AI, cs.RO, stat.ML]
+
+# arXiv category → wiki folder name:
+category_map:
+  cs.CV: "Multimodal"
+  cs.CL: "Post_Training"
+  cs.AI: "Reasoning"
+  cs.RO: "Physical_AI"
+  stat.ML: "Pretraining"
+```
+
+See `config.yaml.example` for the full reference with all options documented.
+
+## Output format
+
+Skeleton files are created at:
+
+```
+output_dir/
+├── sources/{Category}/{slug}.md      # Metadata (title, authors, abstract, URL)
+├── evidence/{Category}/{slug}.md     # Claims & benchmarks — fill this in
+└── wiki/papers/{Category}/{slug}.md  # Synthesis & interpretation — fill this in
+```
+
+`sources/` files are written once and never touched again.
+`evidence/` and `wiki/` files are only (re-)created when they still contain the unfilled template placeholder, so your edits are safe across re-runs.
+
+## Using with Claude Code
+
+Copy the skill file so `/arxiv-ingest` is available as a Claude Code command:
+
+```bash
+cp .claude/commands/arxiv-ingest.md ~/.claude/commands/
+# or into your project's .claude/commands/
+```
+
+After running `arxiv-ingest run`, invoke `/arxiv-ingest` in Claude Code.
+Claude reads each paper's abstract (and PDF when available) and fills in the `evidence/` and `wiki/` files automatically.
+
+## GitHub Actions: daily auto-ingest
+
+### Setup
+
+1. Fork or use this repo as a template
+2. Create a GitHub Fine-grained PAT with **Write** access to your wiki repo
+3. Add to your repo's **Settings → Secrets and variables → Actions**:
+
+   | Kind | Name | Value |
+   |------|------|-------|
+   | Secret | `GH_PAT` | your PAT |
+   | Variable | `WIKI_REPO` | `your-name/research-wiki` |
+
+4. Go to **Actions → Daily arXiv Ingest → Run workflow** to verify
+
+### Schedule
+
+- **Auto**: Mon–Fri UTC 01:00 (JST 10:00) — `cron: '0 1 * * 1-5'`
+- **Manual**: `workflow_dispatch` with optional `days_back` override
+
+## Requirements
+
+- Python 3.11+
+- [uv](https://docs.astral.sh/uv/) (recommended) or pip
+
+## License
+
+MIT

arxiv_ingest-0.2.0/config.yaml

@@ -0,0 +1,43 @@
+# arxiv-ingest configuration
+# Edit keywords and output_dir to match your setup
+
+output_dir: "../research-wiki"  # path to your research-wiki directory
+
+max_results: 20   # per keyword
+days_back: 7      # fetch papers from the last N days
+
+keywords:
+  - "vision language model transformer"
+  - "LoRA PEFT fine-tuning language model"
+  - "masked autoencoder self-supervised learning"
+  - "embodied AI robot learning manipulation"
+  - "large language model reasoning chain-of-thought"
+  - "multimodal foundation model"
+
+# arXiv category filter — only accept papers in these categories
+allowed_arxiv_categories:
+  - cs.CV
+  - cs.CL
+  - cs.LG
+  - cs.AI
+  - cs.RO
+  - stat.ML
+
+# arXiv category → research-wiki category mapping
+category_map:
+  cs.CV: "Multimodal"
+  cs.CL: "Post_Training"
+  cs.AI: "Reasoning"
+  cs.RO: "Physical_AI"
+  stat.ML: "Pretraining"
+  # cs.LG は幅広すぎるため除外 — キーワードマッチで拾う
+
+# cs.LG 論文はキーワードが一致した場合のみ受け入れる
+# allowed_arxiv_categories に cs.LG を残しつつ、
+# primary category が cs.LG 単独の場合はスキップ
+require_primary_in:
+  - cs.CV
+  - cs.CL
+  - cs.AI
+  - cs.RO
+  - stat.ML

arxiv_ingest-0.2.0/config.yaml.example

@@ -0,0 +1,59 @@
+# arxiv-ingest configuration
+# Copy this file to config.yaml and edit to match your setup.
+#
+# Quick start:
+#   cp config.yaml.example config.yaml
+#   arxiv-ingest run
+
+# Path to your research wiki directory (relative to this config file).
+# The tool creates sources/, evidence/, and wiki/papers/ subdirectories here.
+output_dir: "../research-wiki"
+
+# How many papers to fetch per keyword (arXiv API limit: 100)
+max_results: 20
+
+# Fetch papers published within the last N days
+days_back: 7
+
+# Keywords to search for on arXiv. Each keyword is queried independently.
+# Use specific phrases for higher precision, e.g. "vision language model grounding"
+# rather than just "vision".
+keywords:
+  - "Vision-Language Model transformer"
+  - "LoRA PEFT fine-tuning language model"
+  - "masked autoencoder self-supervised learning"
+  - "embodied AI robot learning manipulation"
+  - "large language model reasoning chain-of-thought"
+  - "multimodal foundation model"
+
+# Only accept papers whose arXiv categories include at least one of these.
+# Remove this section to accept all categories.
+allowed_arxiv_categories:
+  - cs.CV    # Computer Vision
+  - cs.CL    # Computation and Language (NLP)
+  - cs.LG    # Machine Learning
+  - cs.AI    # Artificial Intelligence
+  - cs.RO    # Robotics
+  - stat.ML  # Statistics - Machine Learning
+
+# Only accept papers whose PRIMARY category is in this list.
+# This prevents off-topic papers (e.g. pure math) from being included
+# even if they mention your keywords.
+# Remove or leave empty to skip this filter.
+require_primary_in:
+  - cs.CV
+  - cs.CL
+  - cs.AI
+  - cs.RO
+  - stat.ML
+
+# Map arXiv categories to your wiki's category folder names.
+# Papers are placed in wiki/papers/{category}/ based on their primary arXiv category.
+# Add or rename categories to match your wiki structure.
+category_map:
+  cs.CV: "Multimodal"
+  cs.CL: "Post_Training"
+  cs.LG: "Architecture"
+  cs.AI: "Reasoning"
+  cs.RO: "Physical_AI"
+  stat.ML: "Pretraining"

arxiv_ingest-0.2.0/main.py

@@ -0,0 +1,69 @@
+"""arxiv-ingest CLI — fetch and scaffold arXiv papers into a research wiki."""
+
+import sys
+from pathlib import Path
+
+
+def cmd_fetch(args: list[str]) -> None:
+    from scripts.fetch import main as fetch_main
+    sys.argv = ["fetch"] + args
+    fetch_main()
+
+
+def cmd_generate(args: list[str]) -> None:
+    from scripts.generate import main as generate_main
+    generate_main()
+
+
+def cmd_run(args: list[str]) -> None:
+    """Fetch + generate in one step."""
+    from scripts.fetch import main as fetch_main
+    from scripts.generate import main as generate_main
+    fetch_main()
+    generate_main()
+
+
+def print_help() -> None:
+    print(
+        "arxiv-ingest — fetch and scaffold arXiv papers into a research wiki\n"
+        "\n"
+        "Usage:\n"
+        "  arxiv-ingest fetch     Fetch recent papers from arXiv (saves data/fetched.json)\n"
+        "  arxiv-ingest generate  Generate skeleton files from fetched.json\n"
+        "  arxiv-ingest run       fetch + generate in one step\n"
+        "\n"
+        "Options:\n"
+        "  --help    Show this help message\n"
+        "\n"
+        "Configuration:\n"
+        "  Edit config.yaml to set keywords, output_dir, and category mappings.\n"
+        "  Copy config.yaml.example to config.yaml to get started.\n"
+    )
+
+
+def main() -> None:
+    args = sys.argv[1:]
+
+    if not args or args[0] in ("--help", "-h", "help"):
+        print_help()
+        return
+
+    cmd = args[0]
+    rest = args[1:]
+
+    commands = {
+        "fetch": cmd_fetch,
+        "generate": cmd_generate,
+        "run": cmd_run,
+    }
+
+    if cmd not in commands:
+        print(f"Unknown command: {cmd}", file=sys.stderr)
+        print_help()
+        sys.exit(1)
+
+    commands[cmd](rest)
+
+
+if __name__ == "__main__":
+    main()

arxiv_ingest-0.2.0/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "arxiv-ingest"
+version = "0.2.0"
+description = "Fetch recent arXiv papers and scaffold them into a structured research wiki"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+keywords = ["arxiv", "research", "papers", "wiki", "knowledge-management"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "arxiv>=4.0.0",
+    "pyyaml>=6.0.3",
+    "rich>=15.0.0",
+]
+
+[project.scripts]
+arxiv-ingest = "main:main"
+
+[project.urls]
+Homepage = "https://github.com/yamadan96/arxiv-ingest"
+Repository = "https://github.com/yamadan96/arxiv-ingest"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+include = ["main.py", "scripts/*.py"]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.0",
+]