openalex-local 0.1.0__tar.gz → 0.3.1__tar.gz

openalex_local-0.3.1/PKG-INFO

@@ -0,0 +1,288 @@
+Metadata-Version: 2.4
+Name: openalex-local
+Version: 0.3.1
+Summary: Local OpenAlex database with 284M+ works, abstracts, and semantic search
+Author-email: Yusuke Watanabe <ywatanabe@alumni.u-tokyo.ac.jp>
+License: AGPL-3.0
+Project-URL: Homepage, https://github.com/ywatanabe1989/openalex-local
+Project-URL: Repository, https://github.com/ywatanabe1989/openalex-local
+Keywords: openalex,academic,research,abstracts,semantic-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0
+Requires-Dist: awscli>=1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=0.4; extra == "mcp"
+Provides-Extra: server
+Requires-Dist: fastapi>=0.100; extra == "server"
+Requires-Dist: uvicorn>=0.23; extra == "server"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
+Requires-Dist: myst-parser>=2.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=1.25; extra == "docs"
+Provides-Extra: all
+Requires-Dist: openalex-local[dev,docs,mcp,server]; extra == "all"
+
+# OpenAlex Local
+
+Local OpenAlex database with 284M+ scholarly works, abstracts, and semantic search.
+
+[![PyPI version](https://badge.fury.io/py/openalex-local.svg)](https://badge.fury.io/py/openalex-local)
+[![Documentation](https://readthedocs.org/projects/openalex-local/badge/?version=latest)](https://openalex-local.readthedocs.io/en/latest/)
+[![Tests](https://github.com/ywatanabe1989/openalex-local/actions/workflows/test.yml/badge.svg)](https://github.com/ywatanabe1989/openalex-local/actions/workflows/test.yml)
+[![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-AGPL--3.0-blue.svg)](LICENSE)
+
+<details>
+<summary><strong>Why OpenAlex Local?</strong></summary>
+
+**Built for the LLM era** - features that matter for AI research assistants:
+
+| Feature | Benefit |
+|---------|---------|
+| **284M Works** | More coverage than CrossRef |
+| **Abstracts** | ~45-60% availability for semantic search |
+| **Concepts & Topics** | Built-in classification |
+| **Author Disambiguation** | Linked to institutions |
+| **Open Access Info** | OA status and URLs |
+
+Perfect for: RAG systems, research assistants, literature review automation.
+
+</details>
+
+<details>
+<summary><strong>Installation</strong></summary>
+
+```bash
+pip install openalex-local
+```
+
+From source:
+```bash
+git clone https://github.com/ywatanabe1989/openalex-local
+cd openalex-local && make install
+```
+
+Database setup (~300 GB, ~1-2 days to build):
+```bash
+# Check system status
+make status
+
+# 1. Download OpenAlex Works snapshot (~300GB)
+make download-screen  # runs in background
+
+# 2. Build SQLite database
+make build-db
+
+# 3. Build FTS5 index
+make build-fts
+```
+
+</details>
+
+<details>
+<summary><strong>Python API</strong></summary>
+
+```python
+from openalex_local import search, get, count
+
+# Full-text search (title + abstract)
+results = search("machine learning neural networks")
+for work in results:
+    print(f"{work.title} ({work.year})")
+    print(f"  Abstract: {work.abstract[:200]}...")
+    print(f"  Concepts: {[c['name'] for c in work.concepts]}")
+
+# Get by OpenAlex ID or DOI
+work = get("W2741809807")
+work = get("10.1038/nature12373")
+
+# Count matches
+n = count("CRISPR")
+```
+
+</details>
+
+<details>
+<summary><strong>CLI</strong></summary>
+
+```bash
+openalex-local search "CRISPR genome editing" -n 5
+openalex-local search-by-doi W2741809807
+openalex-local search-by-doi 10.1038/nature12373
+openalex-local status  # Configuration and database stats
+```
+
+With abstracts (`-a` flag):
+```
+$ openalex-local search "neural network" -n 1 -a
+
+Found 1,523,847 matches in 45.2ms
+
+1. Deep learning for neural networks (2015)
+   OpenAlex ID: W2741809807
+   Abstract: This paper presents a comprehensive overview of deep learning
+   techniques for neural network architectures...
+```
+
+</details>
+
+<details>
+<summary><strong>HTTP API</strong></summary>
+
+Start the FastAPI server:
+```bash
+openalex-local relay --host 0.0.0.0 --port 31292
+```
+
+Endpoints:
+```bash
+# Search works (FTS5)
+curl "http://localhost:31292/works?q=CRISPR&limit=10"
+
+# Get by ID or DOI
+curl "http://localhost:31292/works/W2741809807"
+curl "http://localhost:31292/works/10.1038/nature12373"
+
+# Batch lookup
+curl -X POST "http://localhost:31292/works/batch" \
+  -H "Content-Type: application/json" \
+  -d '{"ids": ["W2741809807", "10.1038/nature12373"]}'
+
+# Database info
+curl "http://localhost:31292/info"
+```
+
+HTTP mode (connect to running server):
+```bash
+# On local machine (if server is remote)
+ssh -L 31292:127.0.0.1:31292 your-server
+
+# Python client
+from openalex_local import configure_http
+configure_http("http://localhost:31292")
+
+# Or via CLI
+openalex-local --http search "CRISPR"
+```
+
+</details>
+
+<details>
+<summary><strong>MCP Server</strong></summary>
+
+Run as MCP (Model Context Protocol) server:
+```bash
+openalex-local mcp start
+```
+
+Local MCP client configuration:
+```json
+{
+  "mcpServers": {
+    "openalex-local": {
+      "command": "openalex-local",
+      "args": ["mcp", "start"],
+      "env": {
+        "OPENALEX_LOCAL_DB": "/path/to/openalex.db"
+      }
+    }
+  }
+}
+```
+
+Remote MCP via HTTP:
+```bash
+# On server: start persistent MCP server
+openalex-local mcp start -t http --host 0.0.0.0 --port 8083
+```
+```json
+{
+  "mcpServers": {
+    "openalex-remote": {
+      "url": "http://your-server:8083/mcp"
+    }
+  }
+}
+```
+
+Diagnose setup:
+```bash
+openalex-local mcp doctor        # Check dependencies and database
+openalex-local mcp list-tools    # Show available MCP tools
+openalex-local mcp installation  # Show client config examples
+```
+
+Available tools:
+- `search` - Full-text search across 284M+ papers
+- `search_by_id` - Get paper by OpenAlex ID or DOI
+- `enrich_ids` - Batch lookup with metadata
+- `status` - Database statistics
+
+</details>
+
+<details>
+<summary><strong>Related Projects</strong></summary>
+
+**[crossref-local](https://github.com/ywatanabe1989/crossref-local)** - Sister project with CrossRef data:
+
+| Feature | crossref-local | openalex-local |
+|---------|----------------|----------------|
+| Works | 167M | 284M |
+| Abstracts | ~21% | ~45-60% |
+| Update frequency | Real-time | Monthly |
+| DOI authority | Yes (source) | Uses CrossRef |
+| Citations | Raw references | Linked works |
+| Concepts/Topics | No | Yes |
+| Author IDs | No | Yes |
+| Best for | DOI lookup, raw refs | Semantic search |
+
+**When to use CrossRef**: Real-time DOI updates, raw reference parsing, authoritative metadata.
+**When to use OpenAlex**: Semantic search, citation analysis, topic discovery.
+
+</details>
+
+<details>
+<summary><strong>Documentation</strong></summary>
+
+Full documentation available at [openalex-local.readthedocs.io](https://openalex-local.readthedocs.io/en/latest/)
+
+- [Installation Guide](https://openalex-local.readthedocs.io/en/latest/installation.html)
+- [Quickstart](https://openalex-local.readthedocs.io/en/latest/quickstart.html)
+- [CLI Reference](https://openalex-local.readthedocs.io/en/latest/cli_reference.html)
+- [HTTP API Reference](https://openalex-local.readthedocs.io/en/latest/http_api.html)
+- [Python API](https://openalex-local.readthedocs.io/en/latest/api/openalex_local.html)
+
+</details>
+
+<details>
+<summary><strong>Data Source</strong></summary>
+
+Data from [OpenAlex](https://openalex.org/), an open catalog of scholarly works.
+Updated monthly from their [snapshot](https://docs.openalex.org/download-all-data/openalex-snapshot).
+
+</details>
+
+---
+
+<p align="center">
+  <a href="https://scitex.ai"><img src="docs/scitex-icon-navy-inverted.png" alt="SciTeX" width="40"/></a>
+  <br>
+  AGPL-3.0 · ywatanabe@scitex.ai
+</p>
+
+<!-- EOF -->

openalex_local-0.3.1/README.md

@@ -0,0 +1,249 @@
+# OpenAlex Local
+
+Local OpenAlex database with 284M+ scholarly works, abstracts, and semantic search.
+
+[![PyPI version](https://badge.fury.io/py/openalex-local.svg)](https://badge.fury.io/py/openalex-local)
+[![Documentation](https://readthedocs.org/projects/openalex-local/badge/?version=latest)](https://openalex-local.readthedocs.io/en/latest/)
+[![Tests](https://github.com/ywatanabe1989/openalex-local/actions/workflows/test.yml/badge.svg)](https://github.com/ywatanabe1989/openalex-local/actions/workflows/test.yml)
+[![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-AGPL--3.0-blue.svg)](LICENSE)
+
+<details>
+<summary><strong>Why OpenAlex Local?</strong></summary>
+
+**Built for the LLM era** - features that matter for AI research assistants:
+
+| Feature | Benefit |
+|---------|---------|
+| **284M Works** | More coverage than CrossRef |
+| **Abstracts** | ~45-60% availability for semantic search |
+| **Concepts & Topics** | Built-in classification |
+| **Author Disambiguation** | Linked to institutions |
+| **Open Access Info** | OA status and URLs |
+
+Perfect for: RAG systems, research assistants, literature review automation.
+
+</details>
+
+<details>
+<summary><strong>Installation</strong></summary>
+
+```bash
+pip install openalex-local
+```
+
+From source:
+```bash
+git clone https://github.com/ywatanabe1989/openalex-local
+cd openalex-local && make install
+```
+
+Database setup (~300 GB, ~1-2 days to build):
+```bash
+# Check system status
+make status
+
+# 1. Download OpenAlex Works snapshot (~300GB)
+make download-screen  # runs in background
+
+# 2. Build SQLite database
+make build-db
+
+# 3. Build FTS5 index
+make build-fts
+```
+
+</details>
+
+<details>
+<summary><strong>Python API</strong></summary>
+
+```python
+from openalex_local import search, get, count
+
+# Full-text search (title + abstract)
+results = search("machine learning neural networks")
+for work in results:
+    print(f"{work.title} ({work.year})")
+    print(f"  Abstract: {work.abstract[:200]}...")
+    print(f"  Concepts: {[c['name'] for c in work.concepts]}")
+
+# Get by OpenAlex ID or DOI
+work = get("W2741809807")
+work = get("10.1038/nature12373")
+
+# Count matches
+n = count("CRISPR")
+```
+
+</details>
+
+<details>
+<summary><strong>CLI</strong></summary>
+
+```bash
+openalex-local search "CRISPR genome editing" -n 5
+openalex-local search-by-doi W2741809807
+openalex-local search-by-doi 10.1038/nature12373
+openalex-local status  # Configuration and database stats
+```
+
+With abstracts (`-a` flag):
+```
+$ openalex-local search "neural network" -n 1 -a
+
+Found 1,523,847 matches in 45.2ms
+
+1. Deep learning for neural networks (2015)
+   OpenAlex ID: W2741809807
+   Abstract: This paper presents a comprehensive overview of deep learning
+   techniques for neural network architectures...
+```
+
+</details>
+
+<details>
+<summary><strong>HTTP API</strong></summary>
+
+Start the FastAPI server:
+```bash
+openalex-local relay --host 0.0.0.0 --port 31292
+```
+
+Endpoints:
+```bash
+# Search works (FTS5)
+curl "http://localhost:31292/works?q=CRISPR&limit=10"
+
+# Get by ID or DOI
+curl "http://localhost:31292/works/W2741809807"
+curl "http://localhost:31292/works/10.1038/nature12373"
+
+# Batch lookup
+curl -X POST "http://localhost:31292/works/batch" \
+  -H "Content-Type: application/json" \
+  -d '{"ids": ["W2741809807", "10.1038/nature12373"]}'
+
+# Database info
+curl "http://localhost:31292/info"
+```
+
+HTTP mode (connect to running server):
+```bash
+# On local machine (if server is remote)
+ssh -L 31292:127.0.0.1:31292 your-server
+
+# Python client
+from openalex_local import configure_http
+configure_http("http://localhost:31292")
+
+# Or via CLI
+openalex-local --http search "CRISPR"
+```
+
+</details>
+
+<details>
+<summary><strong>MCP Server</strong></summary>
+
+Run as MCP (Model Context Protocol) server:
+```bash
+openalex-local mcp start
+```
+
+Local MCP client configuration:
+```json
+{
+  "mcpServers": {
+    "openalex-local": {
+      "command": "openalex-local",
+      "args": ["mcp", "start"],
+      "env": {
+        "OPENALEX_LOCAL_DB": "/path/to/openalex.db"
+      }
+    }
+  }
+}
+```
+
+Remote MCP via HTTP:
+```bash
+# On server: start persistent MCP server
+openalex-local mcp start -t http --host 0.0.0.0 --port 8083
+```
+```json
+{
+  "mcpServers": {
+    "openalex-remote": {
+      "url": "http://your-server:8083/mcp"
+    }
+  }
+}
+```
+
+Diagnose setup:
+```bash
+openalex-local mcp doctor        # Check dependencies and database
+openalex-local mcp list-tools    # Show available MCP tools
+openalex-local mcp installation  # Show client config examples
+```
+
+Available tools:
+- `search` - Full-text search across 284M+ papers
+- `search_by_id` - Get paper by OpenAlex ID or DOI
+- `enrich_ids` - Batch lookup with metadata
+- `status` - Database statistics
+
+</details>
+
+<details>
+<summary><strong>Related Projects</strong></summary>
+
+**[crossref-local](https://github.com/ywatanabe1989/crossref-local)** - Sister project with CrossRef data:
+
+| Feature | crossref-local | openalex-local |
+|---------|----------------|----------------|
+| Works | 167M | 284M |
+| Abstracts | ~21% | ~45-60% |
+| Update frequency | Real-time | Monthly |
+| DOI authority | Yes (source) | Uses CrossRef |
+| Citations | Raw references | Linked works |
+| Concepts/Topics | No | Yes |
+| Author IDs | No | Yes |
+| Best for | DOI lookup, raw refs | Semantic search |
+
+**When to use CrossRef**: Real-time DOI updates, raw reference parsing, authoritative metadata.
+**When to use OpenAlex**: Semantic search, citation analysis, topic discovery.
+
+</details>
+
+<details>
+<summary><strong>Documentation</strong></summary>
+
+Full documentation available at [openalex-local.readthedocs.io](https://openalex-local.readthedocs.io/en/latest/)
+
+- [Installation Guide](https://openalex-local.readthedocs.io/en/latest/installation.html)
+- [Quickstart](https://openalex-local.readthedocs.io/en/latest/quickstart.html)
+- [CLI Reference](https://openalex-local.readthedocs.io/en/latest/cli_reference.html)
+- [HTTP API Reference](https://openalex-local.readthedocs.io/en/latest/http_api.html)
+- [Python API](https://openalex-local.readthedocs.io/en/latest/api/openalex_local.html)
+
+</details>
+
+<details>
+<summary><strong>Data Source</strong></summary>
+
+Data from [OpenAlex](https://openalex.org/), an open catalog of scholarly works.
+Updated monthly from their [snapshot](https://docs.openalex.org/download-all-data/openalex-snapshot).
+
+</details>
+
+---
+
+<p align="center">
+  <a href="https://scitex.ai"><img src="docs/scitex-icon-navy-inverted.png" alt="SciTeX" width="40"/></a>
+  <br>
+  AGPL-3.0 · ywatanabe@scitex.ai
+</p>
+
+<!-- EOF -->

{openalex_local-0.1.0 → openalex_local-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openalex-local"
-version = "0.1.0"
+version = "0.3.1"
 description = "Local OpenAlex database with 284M+ works, abstracts, and semantic search"
 readme = "README.md"
 license = {text = "AGPL-3.0"}
@@ -32,10 +32,28 @@ dependencies = [
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
+    "pytest-asyncio>=0.21",
+]
+mcp = [
+    "fastmcp>=0.4",
+]
+server = [
+    "fastapi>=0.100",
+    "uvicorn>=0.23",
+]
+docs = [
+    "sphinx>=7.0",
+    "sphinx-rtd-theme>=2.0",
+    "myst-parser>=2.0",
+    "sphinx-copybutton>=0.5",
+    "sphinx-autodoc-typehints>=1.25",
+]
+all = [
+    "openalex-local[server,mcp,dev,docs]",
 ]
 
 [project.scripts]
-openalex-local = "openalex_local.cli:main"
+openalex-local = "openalex_local._cli.cli:main"
 
 [project.urls]
 Homepage = "https://github.com/ywatanabe1989/openalex-local"

openalex_local-0.3.1/src/openalex_local/__init__.py

@@ -0,0 +1,65 @@
+"""
+OpenAlex Local - Local OpenAlex database with 284M+ works and semantic search.
+
+Example:
+    >>> from openalex_local import search, get
+    >>> results = search("machine learning neural networks")
+    >>> work = get("W2741809807")  # OpenAlex ID
+    >>> work = get("10.1038/nature12373")  # or DOI
+"""
+
+__version__ = "0.3.1"
+
+from ._core import (
+    SUPPORTED_FORMATS,
+    SearchResult,
+    Work,
+    configure,
+    count,
+    enrich,
+    enrich_ids,
+    exists,
+    get,
+    get_many,
+    get_mode,
+    info,
+    save,
+    search,
+)
+
+# Jobs module (public functions only)
+from . import jobs
+
+# Async module
+from . import aio
+
+# Cache module
+from . import cache
+
+__all__ = [
+    # Core functions
+    "search",
+    "count",
+    "get",
+    "get_many",
+    "exists",
+    "info",
+    # Enrich functions
+    "enrich",
+    "enrich_ids",
+    # Configuration
+    "configure",
+    "get_mode",
+    # Models
+    "Work",
+    "SearchResult",
+    # Export
+    "save",
+    "SUPPORTED_FORMATS",
+    # Jobs
+    "jobs",
+    # Async
+    "aio",
+    # Cache
+    "cache",
+]

openalex_local-0.3.1/src/openalex_local/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running as python -m openalex_local."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

openalex_local-0.3.1/src/openalex_local/_cache/__init__.py

@@ -0,0 +1,45 @@
+"""Cache module for openalex_local.
+
+Provides local caching of search results and works for offline analysis.
+
+Example:
+    >>> from openalex_local import cache
+    >>> # Create a cache from search
+    >>> info = cache.create("ml_papers", query="machine learning", limit=1000)
+    >>> print(f"Cached {info.count} papers")
+    >>>
+    >>> # Query the cache
+    >>> papers = cache.query("ml_papers", year_min=2020)
+    >>> # Get IDs for further processing
+    >>> ids = cache.query_ids("ml_papers")
+"""
+
+from .models import CacheInfo
+from .core import (
+    create,
+    append,
+    load,
+    query,
+    query_ids,
+    stats,
+    info,
+    exists,
+    list_caches,
+    delete,
+)
+from .export import export
+
+__all__ = [
+    "CacheInfo",
+    "create",
+    "append",
+    "load",
+    "query",
+    "query_ids",
+    "stats",
+    "info",
+    "exists",
+    "list_caches",
+    "delete",
+    "export",
+]