scitex 2.4.1__py3-none-any.whl → 2.4.3__py3-none-any.whl

scitex/scholar/citation_graph/README.md

@@ -0,0 +1,117 @@
+# Citation Graph Module
+
+Build and analyze citation networks for academic papers using CrossRef data.
+
+## Features
+
+- **Citation extraction**: Forward and reverse citation lookups
+- **Similarity metrics**: Co-citation and bibliographic coupling analysis
+- **Network building**: Construct graphs of related papers
+- **Export formats**: JSON for D3.js, vis.js, Cytoscape
+
+## Quick Start
+
+```python
+from scitex.scholar.citation_graph import CitationGraphBuilder
+
+# Initialize with CrossRef database
+builder = CitationGraphBuilder("/path/to/crossref.db")
+
+# Build citation network for a paper
+graph = builder.build("10.1038/s41586-020-2008-3", top_n=20)
+
+# Export for visualization
+builder.export_json(graph, "network.json")
+
+# Get paper summary
+summary = builder.get_paper_summary("10.1038/s41586-020-2008-3")
+```
+
+## Architecture
+
+```
+citation_graph/
+├── __init__.py          # Package exports
+├── builder.py           # CitationGraphBuilder (main interface)
+├── database.py          # Database queries and connection management
+├── models.py            # Data models (PaperNode, CitationEdge, CitationGraph)
+└── README.md            # This file
+```
+
+## Similarity Metrics
+
+### 1. Co-citation
+Papers are related if they are frequently cited together.
+- **Algorithm**: Find papers that appear together in reference lists
+- **Weight**: 2.0 (default)
+- **Use case**: Find foundational/seminal works in the same field
+
+### 2. Bibliographic Coupling
+Papers are related if they cite similar references.
+- **Algorithm**: Count shared references between papers
+- **Weight**: 2.0 (default)
+- **Use case**: Find papers addressing similar problems/methods
+
+### 3. Direct Citations
+Papers directly citing or cited by the seed paper.
+- **Weight**: 1.0 (default)
+- **Use case**: Find immediately related work
+
+## Performance
+
+Based on experiments with 47M+ citations:
+
+| Operation | Time | Status |
+|-----------|------|--------|
+| Forward citations | 0.1ms | ⚡ Excellent |
+| Reverse citations | 3.3s | ✓ Good |
+| Co-citation | 3.2s | ✓ Good |
+| Bibliographic coupling | 25s | ⚠️ Needs optimization |
+| **Full network build** | **~30s** | ✓ Acceptable |
+
+## Database Schema
+
+Requires CrossRef database with:
+- `works` table: Paper metadata
+- `citations` table: Citation relationships (citing_doi, cited_doi, citing_year)
+
+## Example Output
+
+```json
+{
+  "seed": "10.1038/s41586-020-2008-3",
+  "nodes": [
+    {
+      "id": "10.1038/s41586-020-2008-3",
+      "title": "A Randomized Controlled Trial...",
+      "year": 2020,
+      "authors": ["Smith J", "Jones A"],
+      "journal": "Nature",
+      "similarity_score": 100.0
+    },
+    ...
+  ],
+  "edges": [
+    {
+      "source": "10.1038/s41586-020-2008-3",
+      "target": "10.1016/j.cell.2019.11.025",
+      "type": "cites"
+    },
+    ...
+  ]
+}
+```
+
+## Future Enhancements
+
+- [ ] Redis caching for popular papers
+- [ ] Async/parallel query execution
+- [ ] Additional similarity metrics (topic modeling, author networks)
+- [ ] GraphQL API
+- [ ] Real-time updates
+
+## References
+
+- Co-citation: Small, H. (1973). Co-citation in the scientific literature. *J. Am. Soc. Inf. Sci.*
+- Bibliographic coupling: Kessler, M. M. (1963). Bibliographic coupling. *American Documentation*
+- Connected Papers (inspiration): https://www.connectedpapers.com/

scitex/scholar/citation_graph/__init__.py

@@ -0,0 +1,29 @@
+"""
+Citation Graph Module
+
+Build and analyze citation networks for academic papers using CrossRef data.
+
+This module provides tools to:
+- Extract citation relationships
+- Calculate paper similarity (co-citation, bibliographic coupling)
+- Build citation network graphs
+- Export for visualization (D3.js, vis.js, Cytoscape)
+
+Example:
+    >>> from scitex.scholar.citation_graph import CitationGraphBuilder
+    >>>
+    >>> builder = CitationGraphBuilder(db_path="/path/to/crossref.db")
+    >>> graph = builder.build("10.1038/s41586-020-2008-3", top_n=20)
+    >>> builder.export_json(graph, "network.json")
+"""
+
+from .builder import CitationGraphBuilder
+from .models import PaperNode, CitationEdge, CitationGraph
+
+__version__ = "0.1.0"
+__all__ = [
+    "CitationGraphBuilder",
+    "PaperNode",
+    "CitationEdge",
+    "CitationGraph",
+]

scitex/scholar/citation_graph/builder.py

@@ -0,0 +1,214 @@
+"""
+Citation Graph Builder
+
+Main interface for building citation networks from CrossRef data.
+"""
+
+import json
+from pathlib import Path
+from typing import Optional, List
+from collections import Counter
+
+from .database import CitationDatabase
+from .models import PaperNode, CitationEdge, CitationGraph
+
+
+class CitationGraphBuilder:
+    """
+    Build citation network graphs for academic papers.
+
+    Example:
+        >>> builder = CitationGraphBuilder("/path/to/crossref.db")
+        >>> graph = builder.build("10.1038/s41586-020-2008-3", top_n=20)
+        >>> builder.export_json(graph, "network.json")
+    """
+
+    def __init__(self, db_path: str):
+        """
+        Initialize builder with database path.
+
+        Args:
+            db_path: Path to CrossRef SQLite database
+        """
+        self.db_path = db_path
+        self.db = CitationDatabase(db_path)
+
+    def build(
+        self,
+        seed_doi: str,
+        top_n: int = 20,
+        weight_coupling: float = 2.0,
+        weight_cocitation: float = 2.0,
+        weight_direct: float = 1.0,
+    ) -> CitationGraph:
+        """
+        Build citation network around a seed paper.
+
+        Args:
+            seed_doi: DOI of the seed paper
+            top_n: Number of most similar papers to include
+            weight_coupling: Weight for bibliographic coupling
+            weight_cocitation: Weight for co-citation
+            weight_direct: Weight for direct citations
+
+        Returns:
+            CitationGraph object with nodes and edges
+        """
+        with self.db:
+            # Calculate similarity scores
+            scores = self.db.get_combined_similarity_scores(
+                seed_doi,
+                weight_coupling=weight_coupling,
+                weight_cocitation=weight_cocitation,
+                weight_direct=weight_direct,
+            )
+
+            # Get top N most similar papers
+            top_dois = [seed_doi] + [doi for doi, _ in scores.most_common(top_n)]
+
+            # Build nodes with metadata
+            nodes = []
+            for doi in top_dois:
+                node = self._create_paper_node(doi, scores.get(doi, 100.0))
+                nodes.append(node)
+
+            # Build edges (citations between papers in network)
+            edges = self._build_citation_edges(top_dois)
+
+            # Create graph
+            graph = CitationGraph(
+                seed_doi=seed_doi,
+                nodes=nodes,
+                edges=edges,
+                metadata={
+                    "top_n": top_n,
+                    "weights": {
+                        "coupling": weight_coupling,
+                        "cocitation": weight_cocitation,
+                        "direct": weight_direct,
+                    },
+                },
+            )
+
+            return graph
+
+    def _create_paper_node(
+        self, doi: str, similarity_score: float
+    ) -> PaperNode:
+        """
+        Create a PaperNode with metadata from database.
+
+        Args:
+            doi: DOI of the paper
+            similarity_score: Calculated similarity score
+
+        Returns:
+            PaperNode object
+        """
+        metadata = self.db.get_paper_metadata(doi)
+
+        if metadata:
+            # Extract author names
+            authors = metadata.get("author", [])
+            author_names = [
+                f"{a.get('family', '')} {a.get('given', '')[:1]}"
+                for a in authors[:3]
+            ]
+
+            # Extract year
+            year = 0
+            if "published" in metadata and "date-parts" in metadata["published"]:
+                date_parts = metadata["published"]["date-parts"]
+                if date_parts and date_parts[0]:
+                    year = date_parts[0][0] if date_parts[0][0] else 0
+
+            # Extract journal
+            journal = ""
+            if "container-title" in metadata and metadata["container-title"]:
+                journal = metadata["container-title"][0]
+
+            return PaperNode(
+                doi=doi,
+                title=metadata.get("title", ["Unknown"])[0][:200],
+                year=year,
+                authors=author_names,
+                journal=journal,
+                similarity_score=similarity_score,
+            )
+        else:
+            return PaperNode(doi=doi, similarity_score=similarity_score)
+
+    def _build_citation_edges(self, dois: List[str]) -> List[CitationEdge]:
+        """
+        Build citation edges between papers in the network.
+
+        Args:
+            dois: List of DOIs in the network
+
+        Returns:
+            List of CitationEdge objects
+        """
+        edges = []
+        doi_set = set(d.lower() for d in dois)
+
+        for doi in dois:
+            # Get references (papers this one cites)
+            refs = self.db.get_references(doi, limit=100)
+
+            for cited_doi in refs:
+                if cited_doi in doi_set:
+                    edges.append(
+                        CitationEdge(
+                            source=doi,
+                            target=cited_doi,
+                            edge_type="cites",
+                        )
+                    )
+
+        return edges
+
+    def export_json(self, graph: CitationGraph, output_path: str):
+        """
+        Export graph to JSON file for visualization.
+
+        Args:
+            graph: CitationGraph to export
+            output_path: Path to output JSON file
+        """
+        output = Path(output_path)
+        with open(output, "w") as f:
+            json.dump(graph.to_dict(), f, indent=2)
+
+    def get_paper_summary(self, doi: str) -> Optional[dict]:
+        """
+        Get summary information for a paper.
+
+        Args:
+            doi: DOI of the paper
+
+        Returns:
+            Dictionary with paper summary
+        """
+        with self.db:
+            metadata = self.db.get_paper_metadata(doi)
+
+            if not metadata:
+                return None
+
+            # Get citation counts
+            refs = self.db.get_references(doi, limit=1000)
+            citations = self.db.get_citations(doi, limit=1000)
+
+            return {
+                "doi": doi,
+                "title": metadata.get("title", ["Unknown"])[0],
+                "year": metadata.get("published", {})
+                .get("date-parts", [[0]])[0][0],
+                "authors": [
+                    f"{a.get('family', '')} {a.get('given', '')}"
+                    for a in metadata.get("author", [])[:5]
+                ],
+                "journal": metadata.get("container-title", ["Unknown"])[0],
+                "reference_count": len(refs),
+                "citation_count": len(citations),
+            }

scitex/scholar/citation_graph/database.py

@@ -0,0 +1,246 @@
+"""
+Database access layer for citation graph queries.
+
+Handles all SQL queries to the CrossRef SQLite database with
+optimized queries and connection management.
+"""
+
+import sqlite3
+import json
+from pathlib import Path
+from typing import List, Tuple, Dict, Optional
+from collections import Counter
+
+
+class CitationDatabase:
+    """
+    Database interface for citation graph operations.
+
+    Provides optimized queries for:
+    - Citation extraction (forward/reverse)
+    - Co-citation analysis
+    - Bibliographic coupling
+    - Paper metadata lookup
+    """
+
+    def __init__(self, db_path: str):
+        """
+        Initialize database connection.
+
+        Args:
+            db_path: Path to CrossRef SQLite database
+        """
+        self.db_path = Path(db_path)
+        if not self.db_path.exists():
+            raise FileNotFoundError(f"Database not found: {db_path}")
+
+        self.conn = None
+
+    def connect(self, read_only: bool = True):
+        """
+        Connect to database.
+
+        Args:
+            read_only: If True, open in read-only mode (default)
+        """
+        if read_only:
+            self.conn = sqlite3.connect(
+                f"file:{self.db_path}?mode=ro",
+                uri=True,
+                check_same_thread=False  # Allow multi-threaded access (e.g., Django)
+            )
+        else:
+            self.conn = sqlite3.connect(
+                self.db_path,
+                check_same_thread=False
+            )
+
+        self.conn.row_factory = sqlite3.Row
+
+    def close(self):
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+            self.conn = None
+
+    def __enter__(self):
+        """Context manager entry."""
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.close()
+
+    def get_references(self, doi: str, limit: int = 100) -> List[str]:
+        """
+        Get papers cited by this DOI (forward citations).
+
+        Args:
+            doi: DOI of the paper
+            limit: Maximum number of references to return
+
+        Returns:
+            List of DOIs cited by the paper
+        """
+        cursor = self.conn.execute(
+            """
+            SELECT cited_doi
+            FROM citations
+            WHERE citing_doi = ?
+            LIMIT ?
+            """,
+            (doi.lower(), limit),
+        )
+        return [row[0] for row in cursor]
+
+    def get_citations(self, doi: str, limit: int = 100) -> List[Tuple[str, int]]:
+        """
+        Get papers that cite this DOI (reverse citations).
+
+        Args:
+            doi: DOI of the paper
+            limit: Maximum number of citations to return
+
+        Returns:
+            List of (citing_doi, year) tuples
+        """
+        cursor = self.conn.execute(
+            """
+            SELECT citing_doi, citing_year
+            FROM citations
+            WHERE cited_doi = ?
+            ORDER BY citing_year DESC
+            LIMIT ?
+            """,
+            (doi.lower(), limit),
+        )
+        return [(row[0], row[1]) for row in cursor]
+
+    def get_cocited_papers(
+        self, doi: str, limit: int = 50
+    ) -> List[Tuple[str, int]]:
+        """
+        Find papers co-cited with this DOI.
+
+        Papers are co-cited if they appear together in reference lists.
+
+        Args:
+            doi: DOI of the paper
+            limit: Maximum number of results
+
+        Returns:
+            List of (cocited_doi, cocitation_count) tuples
+        """
+        cursor = self.conn.execute(
+            """
+            SELECT c2.cited_doi, COUNT(*) as cocitation_count
+            FROM citations c1
+            JOIN citations c2 ON c1.citing_doi = c2.citing_doi
+            WHERE c1.cited_doi = ?
+              AND c2.cited_doi != ?
+            GROUP BY c2.cited_doi
+            ORDER BY cocitation_count DESC
+            LIMIT ?
+            """,
+            (doi.lower(), doi.lower(), limit),
+        )
+        return [(row[0], row[1]) for row in cursor]
+
+    def get_bibliographic_coupled_papers(
+        self, doi: str, limit: int = 50
+    ) -> List[Tuple[str, int]]:
+        """
+        Find papers with similar references (bibliographic coupling).
+
+        Papers are bibliographically coupled if they cite the same references.
+
+        Args:
+            doi: DOI of the paper
+            limit: Maximum number of results
+
+        Returns:
+            List of (coupled_doi, shared_references_count) tuples
+        """
+        cursor = self.conn.execute(
+            """
+            SELECT c2.citing_doi, COUNT(*) as shared_refs
+            FROM citations c1
+            JOIN citations c2 ON c1.cited_doi = c2.cited_doi
+            WHERE c1.citing_doi = ?
+              AND c2.citing_doi != ?
+            GROUP BY c2.citing_doi
+            ORDER BY shared_refs DESC
+            LIMIT ?
+            """,
+            (doi.lower(), doi.lower(), limit),
+        )
+        return [(row[0], row[1]) for row in cursor]
+
+    def get_paper_metadata(self, doi: str) -> Optional[Dict]:
+        """
+        Get metadata for a paper from works table.
+
+        Args:
+            doi: DOI of the paper
+
+        Returns:
+            Dictionary with paper metadata, or None if not found
+        """
+        cursor = self.conn.execute(
+            "SELECT metadata FROM works WHERE doi = ?", (doi,)
+        )
+        row = cursor.fetchone()
+
+        if row:
+            return json.loads(row[0])
+        return None
+
+    def get_combined_similarity_scores(
+        self,
+        seed_doi: str,
+        weight_coupling: float = 2.0,
+        weight_cocitation: float = 2.0,
+        weight_direct: float = 1.0,
+        max_papers: int = 100,
+    ) -> Counter:
+        """
+        Calculate combined similarity scores using multiple metrics.
+
+        Combines:
+        - Bibliographic coupling (shared references)
+        - Co-citation (cited together)
+        - Direct citations (cites or is cited by)
+
+        Args:
+            seed_doi: DOI of the seed paper
+            weight_coupling: Weight for bibliographic coupling score
+            weight_cocitation: Weight for co-citation score
+            weight_direct: Weight for direct citation score
+            max_papers: Maximum papers to consider per metric
+
+        Returns:
+            Counter with {doi: combined_score}
+        """
+        scores = Counter()
+
+        # 1. Bibliographic coupling
+        coupled = self.get_bibliographic_coupled_papers(seed_doi, limit=max_papers)
+        for doi, count in coupled:
+            scores[doi] += count * weight_coupling
+
+        # 2. Co-citation
+        cocited = self.get_cocited_papers(seed_doi, limit=max_papers)
+        for doi, count in cocited:
+            scores[doi] += count * weight_cocitation
+
+        # 3. Direct citations
+        refs = self.get_references(seed_doi, limit=50)
+        for doi in refs:
+            scores[doi] += weight_direct
+
+        citations = self.get_citations(seed_doi, limit=50)
+        for doi, _ in citations:
+            scores[doi] += weight_direct
+
+        return scores

scitex/scholar/citation_graph/example.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Example usage of the citation_graph module.
+
+Run this from the scitex-code root:
+    python -m scitex.scholar.citation_graph.example
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from scitex.scholar.citation_graph import CitationGraphBuilder
+
+
+def main():
+    # Database path (adjust to your setup)
+    db_path = Path.home() / "proj/crossref_local/data/crossref.db"
+
+    if not db_path.exists():
+        print(f"❌ Database not found: {db_path}")
+        print("Please update the db_path in this script.")
+        return 1
+
+    print("="*70)
+    print("  Citation Graph Example")
+    print("="*70)
+    print(f"\nDatabase: {db_path}")
+
+    # Initialize builder
+    builder = CitationGraphBuilder(str(db_path))
+
+    # Example DOI (a well-cited paper)
+    seed_doi = "10.1001/2013.jamapsychiatry.4"
+
+    # Get paper summary
+    print(f"\n1. Getting paper summary for {seed_doi}...")
+    summary = builder.get_paper_summary(seed_doi)
+
+    if summary:
+        print(f"\nPaper: {summary['title']}")
+        print(f"Authors: {', '.join(summary['authors'][:3])}")
+        print(f"Year: {summary['year']}")
+        print(f"Journal: {summary['journal']}")
+        print(f"References: {summary['reference_count']}")
+        print(f"Citations: {summary['citation_count']}")
+    else:
+        print("Paper not found in database")
+        return 1
+
+    # Build citation network
+    print(f"\n2. Building citation network (top 20 papers)...")
+    graph = builder.build(seed_doi, top_n=20)
+
+    print(f"\nNetwork built:")
+    print(f"  Nodes: {graph.node_count}")
+    print(f"  Edges: {graph.edge_count}")
+
+    # Show top papers by similarity
+    print(f"\nTop 10 most similar papers:")
+    print(f"{'Rank':<5} {'Score':<7} {'Year':<6} {'Title':<60}")
+    print("-"*85)
+
+    sorted_nodes = sorted(
+        graph.nodes,
+        key=lambda n: n.similarity_score,
+        reverse=True
+    )
+
+    for i, node in enumerate(sorted_nodes[:11], 1):
+        if node.doi.lower() == seed_doi.lower():
+            continue
+        print(
+            f"{i:<5} {node.similarity_score:<7.1f} "
+            f"{node.year:<6} {node.title[:60]:<60}"
+        )
+
+    # Export to JSON
+    output_path = Path(__file__).parent / "example_output.json"
+    builder.export_json(graph, str(output_path))
+    print(f"\n3. Network exported to: {output_path}")
+    print(f"   File size: {output_path.stat().st_size / 1024:.1f} KB")
+
+    print("\n✅ Example complete!")
+    print("\nNext steps:")
+    print("  - Open example_output.json to see the graph data")
+    print("  - Use this JSON with D3.js, vis.js, or Cytoscape for visualization")
+    print("  - Integrate with scitex-cloud for API endpoints")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())