deeplightrag 1.0.6__tar.gz

deeplightrag-1.0.6/.env.example

@@ -0,0 +1,13 @@
+# Environment Variables Template
+# Copy this file to .env and fill in your actual values
+# NEVER commit .env to git!
+
+# LLM API Keys (Optional - only if you want to use LLM features)
+OPENAI_API_KEY=your_openai_key_here
+ANTHROPIC_API_KEY=your_anthropic_key_here
+GOOGLE_API_KEY=your_gemini_key_here
+COHERE_API_KEY=your_cohere_key_here
+
+# Other Configuration
+DEEPLIGHTRAG_STORAGE_DIR=./deeplightrag_data
+DEEPLIGHTRAG_DEVICE=auto  # auto, cpu, cuda, mps

deeplightrag-1.0.6/.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+*.sublime-project
+*.sublime-workspace
+.vim/
+
+# Project specific
+deeplightrag_data/
+config/
+*.yaml.local
+*.env.local
+*.env.*.local
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+.eggs/
+
+# Coverage reports
+htmlcov/
+.coverage
+coverage.xml
+
+# Test artifacts
+.pytest_cache/
+.tox/
+.nox/
+
+# IDE and editor files
+.vscode/
+.idea/
+*.code-workspace
+*.iml
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Temporary files
+*.tmp
+*.bak
+*.swp
+*.swo
+*~
+
+# Local test files
+test_*.py
+*_test.py
+config.yaml
+
+# Environment files with secrets
+.env
+*.env
+config.yaml

deeplightrag-1.0.6/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Visual-Graph RAG Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deeplightrag-1.0.6/MANIFEST.in

@@ -0,0 +1,18 @@
+# Include documentation
+include README.md
+include LICENSE
+include CHANGELOG.md
+
+# Include configuration files
+include pyproject.toml
+include requirements.txt
+include requirements-macos.txt
+
+# Include documentation
+recursive-include docs *.md *.rst *.txt
+
+# Exclude unnecessary files
+global-exclude *.py[cod] __pycache__ *.so *.dylib .DS_Store
+prune test*
+prune example*
+prune .git*

deeplightrag-1.0.6/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: deeplightrag
+Version: 1.0.6
+Summary: DeepLightRAG: High-performance Document Indexing and Retrieval System (use with any LLM)
+Author-email: Phuong Nguyen <nhphuong.code@gmail.com>
+Maintainer-email: Phuong Nguyen <nhphuong.code@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/png261/DeepLightRag
+Project-URL: Repository, https://github.com/png261/DeepLightRag
+Project-URL: Bug Tracker, https://github.com/png261/DeepLightRag/issues
+Project-URL: Changelog, https://github.com/png261/DeepLightRag/releases
+Keywords: rag,retrieval,augmented,generation,ocr,vision,graph,nlp,llm,deepseek,document-processing
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: networkx>=3.0
+Requires-Dist: Pillow>=10.0.0
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: typing-extensions>=4.0.0; python_version < "3.10"
+Requires-Dist: pdf2image>=1.16.0
+Requires-Dist: PyMuPDF>=1.23.0
+Requires-Dist: easyocr>=1.7.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.15.0
+Requires-Dist: transformers>=4.40.0
+Requires-Dist: accelerate>=0.24.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: setfit>=1.0.0
+Requires-Dist: gliner>=0.1.12
+Requires-Dist: faiss-cpu>=1.7.4
+Requires-Dist: toon-python>=0.1.2
+Provides-Extra: gpu
+Requires-Dist: bitsandbytes>=0.41.0; extra == "gpu"
+Provides-Extra: macos
+Requires-Dist: mlx>=0.21.0; extra == "macos"
+Requires-Dist: mlx-lm>=0.19.0; extra == "macos"
+Requires-Dist: mlx-vlm>=0.0.3; extra == "macos"
+Provides-Extra: llm
+Requires-Dist: google-generativeai>=0.3.0; extra == "llm"
+Requires-Dist: openai>=1.0.0; extra == "llm"
+Requires-Dist: anthropic>=0.25.0; extra == "llm"
+Provides-Extra: advanced-re
+Requires-Dist: opennre>=1.1.0; extra == "advanced-re"
+Provides-Extra: web
+Requires-Dist: streamlit>=1.30.0; extra == "web"
+Requires-Dist: plotly>=5.18.0; extra == "web"
+Requires-Dist: pandas>=2.0.0; extra == "web"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: pytest-xdist>=3.3.0; extra == "dev"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.11.0; extra == "dev"
+Requires-Dist: black>=23.9.0; extra == "dev"
+Requires-Dist: ruff>=0.0.290; extra == "dev"
+Requires-Dist: mypy>=1.5.0; extra == "dev"
+Requires-Dist: pre-commit>=3.3.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Requires-Dist: wheel>=0.41.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.1.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=1.24.0; extra == "docs"
+Requires-Dist: myst-parser>=1.0.0; extra == "docs"
+Provides-Extra: all
+Requires-Dist: deeplightrag[advanced-re,dev,docs,gpu,llm,web]; extra == "all"
+Dynamic: license-file
+
+# DeepLightRAG
+
+DeepLightRAG is a high-performance document indexing and retrieval system designed to work with any Large Language Model (LLM). It features a dual-layer graph architecture (Visual-Spatial and Entity-Relationship) to provide context-aware and visually-grounded retrieval.
+
+## Features
+
+- **Dual-Layer Graph**: Combines visual layout awareness with semantic entity relationships.
+- **Visual-Grounded Retrieval**: Retrieves not just text, but visual regions and their spatial context.
+- **Robust OCR**: Integrated with DeepSeek-OCR and EasyOCR fallback for reliable text extraction.
+- **Advanced NER**: Uses GLiNER for zero-shot entity recognition.
+- **Flexible LLM Support**: Compatible with OpenAI, Google Gemini, Anthropic, and local LLMs via MLX/Ollama.
+
+## Installation
+
+```bash
+pip install deeplightrag
+```
+
+## Usage
+
+Index a document:
+```bash
+deeplightrag index document.pdf
+```
+
+Query the index:
+```bash
+deeplightrag query "What is the main topic?"
+```
+
+## License
+
+MIT License

deeplightrag-1.0.6/README.md

@@ -0,0 +1,33 @@
+# DeepLightRAG
+
+DeepLightRAG is a high-performance document indexing and retrieval system designed to work with any Large Language Model (LLM). It features a dual-layer graph architecture (Visual-Spatial and Entity-Relationship) to provide context-aware and visually-grounded retrieval.
+
+## Features
+
+- **Dual-Layer Graph**: Combines visual layout awareness with semantic entity relationships.
+- **Visual-Grounded Retrieval**: Retrieves not just text, but visual regions and their spatial context.
+- **Robust OCR**: Integrated with DeepSeek-OCR and EasyOCR fallback for reliable text extraction.
+- **Advanced NER**: Uses GLiNER for zero-shot entity recognition.
+- **Flexible LLM Support**: Compatible with OpenAI, Google Gemini, Anthropic, and local LLMs via MLX/Ollama.
+
+## Installation
+
+```bash
+pip install deeplightrag
+```
+
+## Usage
+
+Index a document:
+```bash
+deeplightrag index document.pdf
+```
+
+Query the index:
+```bash
+deeplightrag query "What is the main topic?"
+```
+
+## License
+
+MIT License

deeplightrag-1.0.6/main.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""
+DeepLightRAG CLI Interface
+Main entry point for the system
+"""
+
+import argparse
+import sys
+from pathlib import Path
+import yaml
+import json
+
+from src.deeplightrag import DeepLightRAG
+
+
+def load_config(config_path: str = "config.yaml") -> dict:
+    """Load configuration from YAML file with automatic Kaggle detection"""
+    import os
+
+    # Auto-detect Kaggle environment and use appropriate config
+    is_kaggle = (
+        os.path.exists("/kaggle")
+        or "KAGGLE_KERNEL_RUN_TYPE" in os.environ
+        or "/kaggle/" in os.getcwd()
+    )
+
+    if is_kaggle and Path("config_kaggle.yaml").exists():
+        config_path = "config_kaggle.yaml"
+        print(f"🔍 Detected Kaggle environment, using {config_path}")
+
+    if Path(config_path).exists():
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+
+        # Auto-configure GPU settings
+        try:
+            import torch
+
+            if torch.cuda.is_available():
+                print(f"🎮 GPU detected: {torch.cuda.get_device_name(0)}")
+                # Update config for GPU usage
+                if "ocr" in config:
+                    config["ocr"]["device"] = "cuda"
+                if "ner" in config:
+                    config["ner"]["device"] = "cuda"
+                if "relation_extraction" in config:
+                    config["relation_extraction"]["device"] = "cuda"
+        except ImportError:
+            pass
+
+        return config
+    return {}
+
+
+def cmd_index(args, rag_system):
+    """Index a PDF document"""
+    print(f"Indexing document: {args.pdf}")
+    results = rag_system.index_document(
+        args.pdf, document_id=args.doc_id, save_to_disk=not args.no_save
+    )
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(results, f, indent=2)
+        print(f"\nResults saved to {args.output}")
+
+    return results
+
+
+def cmd_query(args, rag_system):
+    """Retrieve context for a query"""
+    if args.load_doc:
+        rag_system.load_document(args.load_doc)
+
+    result = rag_system.retrieve(args.question, override_level=args.level)
+
+    print(f"\n{'='*60}")
+    print("RETRIEVED CONTEXT")
+    print(f"{'='*60}")
+    print(f"\n{result.get('context', 'No context')}\n")
+    print(f"{'='*60}")
+    print(f"Query Level: {result.get('level_name', 'N/A')} (Level {result.get('query_level', 'N/A')})")
+    print(f"Tokens Used: {result.get('tokens_used', 'N/A')}/{result.get('token_budget', 'N/A')}")
+    print(f"Entities Found: {result.get('entities_found', 'N/A')}")
+    print(f"Nodes Retrieved: {result.get('nodes_retrieved', 'N/A')}")
+    print(f"Retrieval Time: {result.get('retrieval_time', 'N/A')}")
+    print(f"\n💡 Use this context with your own LLM for generation!")
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(result, f, indent=2, default=str)
+        print(f"\nResults saved to {args.output}")
+
+    return result
+
+
+def cmd_interactive(args, rag_system):
+    """Interactive retrieval mode"""
+    if args.load_doc:
+        rag_system.load_document(args.load_doc)
+
+    print("\n" + "=" * 60)
+    print("DeepLightRAG Interactive Retrieval Mode")
+    print("=" * 60)
+    print("Type your questions (or 'quit' to exit)")
+    print("Commands: !stats, !level N")
+    print("💡 Context retrieved - use with your own LLM for generation")
+    print()
+
+    override_level = None
+
+    while True:
+        try:
+            question = input("\nQuestion: ").strip()
+
+            if not question:
+                continue
+
+            if question.lower() in ["quit", "exit", "q"]:
+                print("Goodbye!")
+                break
+
+            # Handle commands
+            if question.startswith("!"):
+                if question == "!stats":
+                    stats = rag_system.get_statistics()
+                    print(json.dumps(stats, indent=2, default=str))
+                elif question.startswith("!level"):
+                    parts = question.split()
+                    if len(parts) == 2:
+                        override_level = int(parts[1])
+                        print(f"Query level override set to {override_level}")
+                    else:
+                        override_level = None
+                        print("Query level override cleared")
+                else:
+                    print("Unknown command")
+                continue
+
+            # Regular retrieval
+            result = rag_system.retrieve(question, override_level=override_level)
+
+            print(f"\n{'='*60}")
+            print("CONTEXT")
+            print(f"{'='*60}")
+            print(f"{result.get('context', 'No context')[:500]}...")
+            print(f"{'='*60}")
+            print(
+                f"Level: {result.get('level_name', 'N/A')} ({result.get('query_level', 'N/A')}) | "
+                f"Tokens: {result.get('tokens_used', 'N/A')}/{result.get('token_budget', 'N/A')} | "
+                f"Entities: {result.get('entities_found', 'N/A')}"
+            )
+
+        except KeyboardInterrupt:
+            print("\n\nInterrupted. Goodbye!")
+            break
+        except Exception as e:
+            print(f"Error: {e}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="DeepLightRAG: Efficient Document-based RAG System"
+    )
+    parser.add_argument("--config", default="config.yaml", help="Path to configuration file")
+    parser.add_argument(
+        "--storage", default="./deeplightrag_data", help="Storage directory for graphs and indices"
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Command to execute")
+
+    # Index command
+    index_parser = subparsers.add_parser("index", help="Index a PDF document")
+    index_parser.add_argument("pdf", help="Path to PDF file")
+    index_parser.add_argument("--doc-id", help="Document identifier")
+    index_parser.add_argument("--no-save", action="store_true", help="Don't save to disk")
+    index_parser.add_argument("--output", help="Save results to JSON file")
+
+    # Query command
+    query_parser = subparsers.add_parser("query", help="Retrieve context for a query")
+    query_parser.add_argument("question", help="Question to ask")
+    query_parser.add_argument("--load-doc", help="Load specific document")
+    query_parser.add_argument("--level", type=int, help="Override query level (1-5)")
+    query_parser.add_argument("--output", help="Save results to JSON file")
+
+    # Interactive command
+    interactive_parser = subparsers.add_parser("interactive", help="Interactive retrieval mode")
+    interactive_parser.add_argument("--load-doc", help="Load specific document")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # Load configuration
+    config = load_config(args.config)
+
+    # Initialize system
+    rag_system = DeepLightRAG(config=config, storage_dir=args.storage)
+
+    # Execute command
+    if args.command == "index":
+        cmd_index(args, rag_system)
+    elif args.command == "query":
+        cmd_query(args, rag_system)
+    elif args.command == "interactive":
+        cmd_interactive(args, rag_system)
+
+
+if __name__ == "__main__":
+    main()