open-research-lite 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_research_lite-0.1.0/LICENSE +21 -0
- open_research_lite-0.1.0/PKG-INFO +156 -0
- open_research_lite-0.1.0/README.md +128 -0
- open_research_lite-0.1.0/open_research_lite.egg-info/PKG-INFO +156 -0
- open_research_lite-0.1.0/open_research_lite.egg-info/SOURCES.txt +28 -0
- open_research_lite-0.1.0/open_research_lite.egg-info/dependency_links.txt +1 -0
- open_research_lite-0.1.0/open_research_lite.egg-info/requires.txt +18 -0
- open_research_lite-0.1.0/open_research_lite.egg-info/top_level.txt +4 -0
- open_research_lite-0.1.0/pyproject.toml +42 -0
- open_research_lite-0.1.0/setup.cfg +4 -0
- open_research_lite-0.1.0/src/legacy/__init__.py +3 -0
- open_research_lite-0.1.0/src/legacy/configuration.py +106 -0
- open_research_lite-0.1.0/src/legacy/graph.py +503 -0
- open_research_lite-0.1.0/src/legacy/multi_agent.py +488 -0
- open_research_lite-0.1.0/src/legacy/prompts.py +513 -0
- open_research_lite-0.1.0/src/legacy/state.py +73 -0
- open_research_lite-0.1.0/src/legacy/utils.py +1635 -0
- open_research_lite-0.1.0/src/open_deep_research/configuration.py +262 -0
- open_research_lite-0.1.0/src/open_deep_research/deep_researcher.py +746 -0
- open_research_lite-0.1.0/src/open_deep_research/prompts.py +368 -0
- open_research_lite-0.1.0/src/open_deep_research/state.py +96 -0
- open_research_lite-0.1.0/src/open_deep_research/utils.py +925 -0
- open_research_lite-0.1.0/src/open_research_lite/__init__.py +27 -0
- open_research_lite-0.1.0/tests/evaluators.py +174 -0
- open_research_lite-0.1.0/tests/extract_langsmith_data.py +83 -0
- open_research_lite-0.1.0/tests/pairwise_evaluation.py +128 -0
- open_research_lite-0.1.0/tests/prompts.py +257 -0
- open_research_lite-0.1.0/tests/run_evaluate.py +90 -0
- open_research_lite-0.1.0/tests/supervisor_parallel_evaluation.py +61 -0
- open_research_lite-0.1.0/tests/test_concept_diff.py +102 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 LangChain
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: open-research-lite
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents.
|
|
5
|
+
Author: DeepResearch-Lite Team
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: langgraph>=0.5.4
|
|
11
|
+
Requires-Dist: langchain-community>=0.3.9
|
|
12
|
+
Requires-Dist: langchain-openai>=0.3.28
|
|
13
|
+
Requires-Dist: langchain-anthropic>=0.3.15
|
|
14
|
+
Requires-Dist: langchain-mcp-adapters>=0.1.6
|
|
15
|
+
Requires-Dist: langchain-tavily
|
|
16
|
+
Requires-Dist: openai>=1.99.2
|
|
17
|
+
Requires-Dist: tavily-python>=0.5.0
|
|
18
|
+
Requires-Dist: pydantic>=2.0.0
|
|
19
|
+
Requires-Dist: requests>=2.32.3
|
|
20
|
+
Requires-Dist: beautifulsoup4==4.14.3
|
|
21
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
22
|
+
Requires-Dist: pytest
|
|
23
|
+
Requires-Dist: langchain-google-genai>=2.1.5
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: mypy>=1.11.1; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff>=0.6.1; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# DeepResearch-Lite ⚡
|
|
30
|
+
> **Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents**
|
|
31
|
+
|
|
32
|
+
[](https://opensource.org/licenses/MIT)
|
|
33
|
+
[](https://www.python.org/downloads/)
|
|
34
|
+
[](#benchmark--scorecard)
|
|
35
|
+
|
|
36
|
+
`DeepResearch-Lite` is an intelligent, high-performance gatekeeper layer designed for AI Research Agents (such as `langchain-ai/open_deep_research`, `smolagents`, `AutoGPT`, `CrewAI`, or custom LangGraph workflows).
|
|
37
|
+
|
|
38
|
+
It solves the **Extreme Token Bloat & Repetitive Fluff** flaw in existing deep research systems by shifting the paradigm from *"Read raw 5,000-word scraped web pages"* to **"Compile knowledge incrementally as it arrives."**
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## 🚀 Key Features
|
|
43
|
+
|
|
44
|
+
* **Dual-Layer Extraction Architecture**:
|
|
45
|
+
- **Layer 1 (LLM Mode)**: Sub-second structured JSON triplet extraction `(Subject ──► Predicate ──► Object)` via **Gemini 2.5 Flash** or **OpenAI Mini** when an API key is available.
|
|
46
|
+
- **Layer 2 (Professional Local NLP Mode)**: 0-ms offline local NLP parser (regex metric extraction, grammar triplet matching, entity resolution) when running without API keys.
|
|
47
|
+
* **Session Knowledge Graph**: Maintains an in-memory graph state of all entities, metrics, and claims learned during a research session.
|
|
48
|
+
* **Concept-Diff Engine**: Classifies scraped facts into 3 categories:
|
|
49
|
+
- `DISCARD`: Repetitive background fluff deleted immediately ($0 LLM tokens spent).
|
|
50
|
+
- `DIFF_ADD`: Novel assertions added to graph & passed in Diff Payload.
|
|
51
|
+
- `DIFF_CONFLICT`: Contradictory statements (e.g. $110 vs $140/kWh) explicitly flagged.
|
|
52
|
+
* **80%+ Token & Cost Reduction**: Delivers a ~150-word **Diff Payload** to the main reasoning model instead of 5,000 words of background history.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## ⚡ Quickstart
|
|
57
|
+
|
|
58
|
+
### 1. Installation
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install -e .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### 2. Basic Usage (Python API)
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import asyncio
|
|
68
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
69
|
+
|
|
70
|
+
async def main():
|
|
71
|
+
# Automatically uses GEMINI_API_KEY if present, otherwise uses Professional Local NLP
|
|
72
|
+
engine = ConceptDiffEngine()
|
|
73
|
+
|
|
74
|
+
raw_scraped_text = """
|
|
75
|
+
Electric vehicles have become popular over the last decade...
|
|
76
|
+
Lithium-ion batteries were invented by John Goodenough...
|
|
77
|
+
In 2026, researchers demonstrated a solid-state cell achieving 500 Wh/kg energy density.
|
|
78
|
+
Target production cell cost is $110/kWh.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
# Process raw scrape into a condensed Diff Payload
|
|
82
|
+
diff_payload = await engine.process_observation(
|
|
83
|
+
raw_text=raw_scraped_text,
|
|
84
|
+
source_url="https://tech-news.com/ev-batteries",
|
|
85
|
+
source_title="EV Battery 2026 Report"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
print(diff_payload)
|
|
89
|
+
|
|
90
|
+
if __name__ == "__main__":
|
|
91
|
+
asyncio.run(main())
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
## 📊 Benchmark & Scorecard
|
|
97
|
+
|
|
98
|
+
Running `DeepResearch-Lite` on real web searches (`2026 solid state battery Wh/kg breakthroughs`):
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
===========================================================================
|
|
102
|
+
🏆 DEEPRESEARCH-LITE HACKATHON SCORECARD
|
|
103
|
+
===========================================================================
|
|
104
|
+
METRIC | BASELINE AGENT | DEEPRESEARCH-LITE
|
|
105
|
+
---------------------------------------------------------------------------
|
|
106
|
+
Live Web Words Fetched | 9,159 words | 1,083 words
|
|
107
|
+
Input Tokens Passed to LLM | 11,906 tokens | 1,407 tokens
|
|
108
|
+
Estimated Cost per Search Run | $0.0298 | $0.0035
|
|
109
|
+
Token Reduction | 0% (Full Bloat) | 88.2% SAVED ⚡
|
|
110
|
+
Fact-to-Fluff Signal Ratio | ~15% High Signal | ~95% High Signal ⚡
|
|
111
|
+
===========================================================================
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## 🛠️ Framework Integration
|
|
117
|
+
|
|
118
|
+
### LangGraph / open_deep_research Integration
|
|
119
|
+
`DeepResearch-Lite` integrates seamlessly into `open_deep_research/deep_researcher.py` at the `researcher_tools()` node:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
123
|
+
|
|
124
|
+
diff_engine = ConceptDiffEngine()
|
|
125
|
+
|
|
126
|
+
# In researcher_tools():
|
|
127
|
+
processed_observations = []
|
|
128
|
+
for obs, tool_call in zip(observations, tool_calls):
|
|
129
|
+
if len(obs) > 100:
|
|
130
|
+
diff_payload = await diff_engine.process_observation(obs, source_title=tool_call['name'])
|
|
131
|
+
processed_observations.append(diff_payload)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## 🧪 Testing & Demonstration Scripts
|
|
137
|
+
|
|
138
|
+
- **Run Unit Tests**:
|
|
139
|
+
```bash
|
|
140
|
+
python -m pytest tests/test_concept_diff.py
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
- **Run Side-by-Side Hackathon Benchmark**:
|
|
144
|
+
```bash
|
|
145
|
+
python demo_comparison.py
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
- **Run Live Internet Search Comparison**:
|
|
149
|
+
```bash
|
|
150
|
+
python run_live_test.py "your custom research prompt"
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## 📄 License
|
|
156
|
+
MIT License. Free for open-source and commercial use.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# DeepResearch-Lite ⚡
|
|
2
|
+
> **Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents**
|
|
3
|
+
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
[](#benchmark--scorecard)
|
|
7
|
+
|
|
8
|
+
`DeepResearch-Lite` is an intelligent, high-performance gatekeeper layer designed for AI Research Agents (such as `langchain-ai/open_deep_research`, `smolagents`, `AutoGPT`, `CrewAI`, or custom LangGraph workflows).
|
|
9
|
+
|
|
10
|
+
It solves the **Extreme Token Bloat & Repetitive Fluff** flaw in existing deep research systems by shifting the paradigm from *"Read raw 5,000-word scraped web pages"* to **"Compile knowledge incrementally as it arrives."**
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 🚀 Key Features
|
|
15
|
+
|
|
16
|
+
* **Dual-Layer Extraction Architecture**:
|
|
17
|
+
- **Layer 1 (LLM Mode)**: Sub-second structured JSON triplet extraction `(Subject ──► Predicate ──► Object)` via **Gemini 2.5 Flash** or **OpenAI Mini** when an API key is available.
|
|
18
|
+
- **Layer 2 (Professional Local NLP Mode)**: 0-ms offline local NLP parser (regex metric extraction, grammar triplet matching, entity resolution) when running without API keys.
|
|
19
|
+
* **Session Knowledge Graph**: Maintains an in-memory graph state of all entities, metrics, and claims learned during a research session.
|
|
20
|
+
* **Concept-Diff Engine**: Classifies scraped facts into 3 categories:
|
|
21
|
+
- `DISCARD`: Repetitive background fluff deleted immediately ($0 LLM tokens spent).
|
|
22
|
+
- `DIFF_ADD`: Novel assertions added to graph & passed in Diff Payload.
|
|
23
|
+
- `DIFF_CONFLICT`: Contradictory statements (e.g. $110 vs $140/kWh) explicitly flagged.
|
|
24
|
+
* **80%+ Token & Cost Reduction**: Delivers a ~150-word **Diff Payload** to the main reasoning model instead of 5,000 words of background history.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## ⚡ Quickstart
|
|
29
|
+
|
|
30
|
+
### 1. Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install -e .
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 2. Basic Usage (Python API)
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import asyncio
|
|
40
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
41
|
+
|
|
42
|
+
async def main():
|
|
43
|
+
# Automatically uses GEMINI_API_KEY if present, otherwise uses Professional Local NLP
|
|
44
|
+
engine = ConceptDiffEngine()
|
|
45
|
+
|
|
46
|
+
raw_scraped_text = """
|
|
47
|
+
Electric vehicles have become popular over the last decade...
|
|
48
|
+
Lithium-ion batteries were invented by John Goodenough...
|
|
49
|
+
In 2026, researchers demonstrated a solid-state cell achieving 500 Wh/kg energy density.
|
|
50
|
+
Target production cell cost is $110/kWh.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
# Process raw scrape into a condensed Diff Payload
|
|
54
|
+
diff_payload = await engine.process_observation(
|
|
55
|
+
raw_text=raw_scraped_text,
|
|
56
|
+
source_url="https://tech-news.com/ev-batteries",
|
|
57
|
+
source_title="EV Battery 2026 Report"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
print(diff_payload)
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
asyncio.run(main())
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## 📊 Benchmark & Scorecard
|
|
69
|
+
|
|
70
|
+
Running `DeepResearch-Lite` on real web searches (`2026 solid state battery Wh/kg breakthroughs`):
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
===========================================================================
|
|
74
|
+
🏆 DEEPRESEARCH-LITE HACKATHON SCORECARD
|
|
75
|
+
===========================================================================
|
|
76
|
+
METRIC | BASELINE AGENT | DEEPRESEARCH-LITE
|
|
77
|
+
---------------------------------------------------------------------------
|
|
78
|
+
Live Web Words Fetched | 9,159 words | 1,083 words
|
|
79
|
+
Input Tokens Passed to LLM | 11,906 tokens | 1,407 tokens
|
|
80
|
+
Estimated Cost per Search Run | $0.0298 | $0.0035
|
|
81
|
+
Token Reduction | 0% (Full Bloat) | 88.2% SAVED ⚡
|
|
82
|
+
Fact-to-Fluff Signal Ratio | ~15% High Signal | ~95% High Signal ⚡
|
|
83
|
+
===========================================================================
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## 🛠️ Framework Integration
|
|
89
|
+
|
|
90
|
+
### LangGraph / open_deep_research Integration
|
|
91
|
+
`DeepResearch-Lite` integrates seamlessly into `open_deep_research/deep_researcher.py` at the `researcher_tools()` node:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
95
|
+
|
|
96
|
+
diff_engine = ConceptDiffEngine()
|
|
97
|
+
|
|
98
|
+
# In researcher_tools():
|
|
99
|
+
processed_observations = []
|
|
100
|
+
for obs, tool_call in zip(observations, tool_calls):
|
|
101
|
+
if len(obs) > 100:
|
|
102
|
+
diff_payload = await diff_engine.process_observation(obs, source_title=tool_call['name'])
|
|
103
|
+
processed_observations.append(diff_payload)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 🧪 Testing & Demonstration Scripts
|
|
109
|
+
|
|
110
|
+
- **Run Unit Tests**:
|
|
111
|
+
```bash
|
|
112
|
+
python -m pytest tests/test_concept_diff.py
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
- **Run Side-by-Side Hackathon Benchmark**:
|
|
116
|
+
```bash
|
|
117
|
+
python demo_comparison.py
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
- **Run Live Internet Search Comparison**:
|
|
121
|
+
```bash
|
|
122
|
+
python run_live_test.py "your custom research prompt"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## 📄 License
|
|
128
|
+
MIT License. Free for open-source and commercial use.
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: open-research-lite
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents.
|
|
5
|
+
Author: DeepResearch-Lite Team
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: langgraph>=0.5.4
|
|
11
|
+
Requires-Dist: langchain-community>=0.3.9
|
|
12
|
+
Requires-Dist: langchain-openai>=0.3.28
|
|
13
|
+
Requires-Dist: langchain-anthropic>=0.3.15
|
|
14
|
+
Requires-Dist: langchain-mcp-adapters>=0.1.6
|
|
15
|
+
Requires-Dist: langchain-tavily
|
|
16
|
+
Requires-Dist: openai>=1.99.2
|
|
17
|
+
Requires-Dist: tavily-python>=0.5.0
|
|
18
|
+
Requires-Dist: pydantic>=2.0.0
|
|
19
|
+
Requires-Dist: requests>=2.32.3
|
|
20
|
+
Requires-Dist: beautifulsoup4==4.14.3
|
|
21
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
22
|
+
Requires-Dist: pytest
|
|
23
|
+
Requires-Dist: langchain-google-genai>=2.1.5
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: mypy>=1.11.1; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff>=0.6.1; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# DeepResearch-Lite ⚡
|
|
30
|
+
> **Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents**
|
|
31
|
+
|
|
32
|
+
[](https://opensource.org/licenses/MIT)
|
|
33
|
+
[](https://www.python.org/downloads/)
|
|
34
|
+
[](#benchmark--scorecard)
|
|
35
|
+
|
|
36
|
+
`DeepResearch-Lite` is an intelligent, high-performance gatekeeper layer designed for AI Research Agents (such as `langchain-ai/open_deep_research`, `smolagents`, `AutoGPT`, `CrewAI`, or custom LangGraph workflows).
|
|
37
|
+
|
|
38
|
+
It solves the **Extreme Token Bloat & Repetitive Fluff** flaw in existing deep research systems by shifting the paradigm from *"Read raw 5,000-word scraped web pages"* to **"Compile knowledge incrementally as it arrives."**
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## 🚀 Key Features
|
|
43
|
+
|
|
44
|
+
* **Dual-Layer Extraction Architecture**:
|
|
45
|
+
- **Layer 1 (LLM Mode)**: Sub-second structured JSON triplet extraction `(Subject ──► Predicate ──► Object)` via **Gemini 2.5 Flash** or **OpenAI Mini** when an API key is available.
|
|
46
|
+
- **Layer 2 (Professional Local NLP Mode)**: 0-ms offline local NLP parser (regex metric extraction, grammar triplet matching, entity resolution) when running without API keys.
|
|
47
|
+
* **Session Knowledge Graph**: Maintains an in-memory graph state of all entities, metrics, and claims learned during a research session.
|
|
48
|
+
* **Concept-Diff Engine**: Classifies scraped facts into 3 categories:
|
|
49
|
+
- `DISCARD`: Repetitive background fluff deleted immediately ($0 LLM tokens spent).
|
|
50
|
+
- `DIFF_ADD`: Novel assertions added to graph & passed in Diff Payload.
|
|
51
|
+
- `DIFF_CONFLICT`: Contradictory statements (e.g. $110 vs $140/kWh) explicitly flagged.
|
|
52
|
+
* **80%+ Token & Cost Reduction**: Delivers a ~150-word **Diff Payload** to the main reasoning model instead of 5,000 words of background history.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## ⚡ Quickstart
|
|
57
|
+
|
|
58
|
+
### 1. Installation
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install -e .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### 2. Basic Usage (Python API)
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import asyncio
|
|
68
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
69
|
+
|
|
70
|
+
async def main():
|
|
71
|
+
# Automatically uses GEMINI_API_KEY if present, otherwise uses Professional Local NLP
|
|
72
|
+
engine = ConceptDiffEngine()
|
|
73
|
+
|
|
74
|
+
raw_scraped_text = """
|
|
75
|
+
Electric vehicles have become popular over the last decade...
|
|
76
|
+
Lithium-ion batteries were invented by John Goodenough...
|
|
77
|
+
In 2026, researchers demonstrated a solid-state cell achieving 500 Wh/kg energy density.
|
|
78
|
+
Target production cell cost is $110/kWh.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
# Process raw scrape into a condensed Diff Payload
|
|
82
|
+
diff_payload = await engine.process_observation(
|
|
83
|
+
raw_text=raw_scraped_text,
|
|
84
|
+
source_url="https://tech-news.com/ev-batteries",
|
|
85
|
+
source_title="EV Battery 2026 Report"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
print(diff_payload)
|
|
89
|
+
|
|
90
|
+
if __name__ == "__main__":
|
|
91
|
+
asyncio.run(main())
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
## 📊 Benchmark & Scorecard
|
|
97
|
+
|
|
98
|
+
Running `DeepResearch-Lite` on real web searches (`2026 solid state battery Wh/kg breakthroughs`):
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
===========================================================================
|
|
102
|
+
🏆 DEEPRESEARCH-LITE HACKATHON SCORECARD
|
|
103
|
+
===========================================================================
|
|
104
|
+
METRIC | BASELINE AGENT | DEEPRESEARCH-LITE
|
|
105
|
+
---------------------------------------------------------------------------
|
|
106
|
+
Live Web Words Fetched | 9,159 words | 1,083 words
|
|
107
|
+
Input Tokens Passed to LLM | 11,906 tokens | 1,407 tokens
|
|
108
|
+
Estimated Cost per Search Run | $0.0298 | $0.0035
|
|
109
|
+
Token Reduction | 0% (Full Bloat) | 88.2% SAVED ⚡
|
|
110
|
+
Fact-to-Fluff Signal Ratio | ~15% High Signal | ~95% High Signal ⚡
|
|
111
|
+
===========================================================================
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## 🛠️ Framework Integration
|
|
117
|
+
|
|
118
|
+
### LangGraph / open_deep_research Integration
|
|
119
|
+
`DeepResearch-Lite` integrates seamlessly into `open_deep_research/deep_researcher.py` at the `researcher_tools()` node:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from open_deep_research.concept_diff import ConceptDiffEngine
|
|
123
|
+
|
|
124
|
+
diff_engine = ConceptDiffEngine()
|
|
125
|
+
|
|
126
|
+
# In researcher_tools():
|
|
127
|
+
processed_observations = []
|
|
128
|
+
for obs, tool_call in zip(observations, tool_calls):
|
|
129
|
+
if len(obs) > 100:
|
|
130
|
+
diff_payload = await diff_engine.process_observation(obs, source_title=tool_call['name'])
|
|
131
|
+
processed_observations.append(diff_payload)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## 🧪 Testing & Demonstration Scripts
|
|
137
|
+
|
|
138
|
+
- **Run Unit Tests**:
|
|
139
|
+
```bash
|
|
140
|
+
python -m pytest tests/test_concept_diff.py
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
- **Run Side-by-Side Hackathon Benchmark**:
|
|
144
|
+
```bash
|
|
145
|
+
python demo_comparison.py
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
- **Run Live Internet Search Comparison**:
|
|
149
|
+
```bash
|
|
150
|
+
python run_live_test.py "your custom research prompt"
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## 📄 License
|
|
156
|
+
MIT License. Free for open-source and commercial use.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
open_research_lite.egg-info/PKG-INFO
|
|
5
|
+
open_research_lite.egg-info/SOURCES.txt
|
|
6
|
+
open_research_lite.egg-info/dependency_links.txt
|
|
7
|
+
open_research_lite.egg-info/requires.txt
|
|
8
|
+
open_research_lite.egg-info/top_level.txt
|
|
9
|
+
src/legacy/__init__.py
|
|
10
|
+
src/legacy/configuration.py
|
|
11
|
+
src/legacy/graph.py
|
|
12
|
+
src/legacy/multi_agent.py
|
|
13
|
+
src/legacy/prompts.py
|
|
14
|
+
src/legacy/state.py
|
|
15
|
+
src/legacy/utils.py
|
|
16
|
+
src/open_deep_research/configuration.py
|
|
17
|
+
src/open_deep_research/deep_researcher.py
|
|
18
|
+
src/open_deep_research/prompts.py
|
|
19
|
+
src/open_deep_research/state.py
|
|
20
|
+
src/open_deep_research/utils.py
|
|
21
|
+
src/open_research_lite/__init__.py
|
|
22
|
+
tests/evaluators.py
|
|
23
|
+
tests/extract_langsmith_data.py
|
|
24
|
+
tests/pairwise_evaluation.py
|
|
25
|
+
tests/prompts.py
|
|
26
|
+
tests/run_evaluate.py
|
|
27
|
+
tests/supervisor_parallel_evaluation.py
|
|
28
|
+
tests/test_concept_diff.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
langgraph>=0.5.4
|
|
2
|
+
langchain-community>=0.3.9
|
|
3
|
+
langchain-openai>=0.3.28
|
|
4
|
+
langchain-anthropic>=0.3.15
|
|
5
|
+
langchain-mcp-adapters>=0.1.6
|
|
6
|
+
langchain-tavily
|
|
7
|
+
openai>=1.99.2
|
|
8
|
+
tavily-python>=0.5.0
|
|
9
|
+
pydantic>=2.0.0
|
|
10
|
+
requests>=2.32.3
|
|
11
|
+
beautifulsoup4==4.14.3
|
|
12
|
+
python-dotenv>=1.0.1
|
|
13
|
+
pytest
|
|
14
|
+
langchain-google-genai>=2.1.5
|
|
15
|
+
|
|
16
|
+
[dev]
|
|
17
|
+
mypy>=1.11.1
|
|
18
|
+
ruff>=0.6.1
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "open-research-lite"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Token-Efficient Ingestion Middleware & Concept-Diff Engine for Deep Research AI Agents."
|
|
5
|
+
authors = [
|
|
6
|
+
{ name = "DeepResearch-Lite Team" }
|
|
7
|
+
]
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
license = { text = "MIT" }
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"langgraph>=0.5.4",
|
|
13
|
+
"langchain-community>=0.3.9",
|
|
14
|
+
"langchain-openai>=0.3.28",
|
|
15
|
+
"langchain-anthropic>=0.3.15",
|
|
16
|
+
"langchain-mcp-adapters>=0.1.6",
|
|
17
|
+
"langchain-tavily",
|
|
18
|
+
"openai>=1.99.2",
|
|
19
|
+
"tavily-python>=0.5.0",
|
|
20
|
+
"pydantic>=2.0.0",
|
|
21
|
+
"requests>=2.32.3",
|
|
22
|
+
"beautifulsoup4==4.14.3",
|
|
23
|
+
"python-dotenv>=1.0.1",
|
|
24
|
+
"pytest",
|
|
25
|
+
"langchain-google-genai>=2.1.5",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
dev = ["mypy>=1.11.1", "ruff>=0.6.1"]
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["setuptools>=73.0.0", "wheel"]
|
|
33
|
+
build-backend = "setuptools.build_meta"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools]
|
|
36
|
+
packages = ["open_research_lite", "open_deep_research", "legacy", "tests"]
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.package-dir]
|
|
39
|
+
"open_research_lite" = "src/open_research_lite"
|
|
40
|
+
"open_deep_research" = "src/open_deep_research"
|
|
41
|
+
"legacy" = "src/legacy"
|
|
42
|
+
"tests" = "tests"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from dataclasses import dataclass, fields
|
|
4
|
+
from typing import Any, Optional, Dict, Literal
|
|
5
|
+
|
|
6
|
+
from langchain_core.runnables import RunnableConfig
|
|
7
|
+
|
|
8
|
+
DEFAULT_REPORT_STRUCTURE = """Use this structure to create a report on the user-provided topic:
|
|
9
|
+
|
|
10
|
+
1. Introduction (no research needed)
|
|
11
|
+
- Brief overview of the topic area
|
|
12
|
+
|
|
13
|
+
2. Main Body Sections:
|
|
14
|
+
- Each section should focus on a sub-topic of the user-provided topic
|
|
15
|
+
|
|
16
|
+
3. Conclusion
|
|
17
|
+
- Aim for 1 structural element (either a list or table) that distills the main body sections
|
|
18
|
+
- Provide a concise summary of the report"""
|
|
19
|
+
|
|
20
|
+
class SearchAPI(Enum):
|
|
21
|
+
PERPLEXITY = "perplexity"
|
|
22
|
+
TAVILY = "tavily"
|
|
23
|
+
EXA = "exa"
|
|
24
|
+
ARXIV = "arxiv"
|
|
25
|
+
PUBMED = "pubmed"
|
|
26
|
+
LINKUP = "linkup"
|
|
27
|
+
DUCKDUCKGO = "duckduckgo"
|
|
28
|
+
GOOGLESEARCH = "googlesearch"
|
|
29
|
+
NONE = "none"
|
|
30
|
+
|
|
31
|
+
@dataclass(kw_only=True)
|
|
32
|
+
class Configuration:
|
|
33
|
+
"""Configuration for the workflow/graph-based implementation (graph.py)."""
|
|
34
|
+
# Common configuration
|
|
35
|
+
report_structure: str = DEFAULT_REPORT_STRUCTURE
|
|
36
|
+
search_api: SearchAPI = SearchAPI.TAVILY
|
|
37
|
+
search_api_config: Optional[Dict[str, Any]] = None
|
|
38
|
+
process_search_results: Literal["summarize", "split_and_rerank"] | None = None
|
|
39
|
+
summarization_model_provider: str = "openai"
|
|
40
|
+
summarization_model: str = "gpt-4.1"
|
|
41
|
+
max_structured_output_retries: int = 3
|
|
42
|
+
include_source_str: bool = False
|
|
43
|
+
|
|
44
|
+
# Workflow-specific configuration
|
|
45
|
+
number_of_queries: int = 2 # Number of search queries to generate per iteration
|
|
46
|
+
max_search_depth: int = 2 # Maximum number of reflection + search iterations
|
|
47
|
+
planner_provider: str = "anthropic"
|
|
48
|
+
planner_model: str = "claude-3-7-sonnet-latest"
|
|
49
|
+
planner_model_kwargs: Optional[Dict[str, Any]] = None
|
|
50
|
+
writer_provider: str = "openai"
|
|
51
|
+
writer_model: str = "gpt-4.1"
|
|
52
|
+
writer_model_kwargs: Optional[Dict[str, Any]] = None
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def from_runnable_config(
|
|
56
|
+
cls, config: Optional[RunnableConfig] = None
|
|
57
|
+
) -> "Configuration":
|
|
58
|
+
"""Create a Configuration instance from a RunnableConfig."""
|
|
59
|
+
configurable = (
|
|
60
|
+
config["configurable"] if config and "configurable" in config else {}
|
|
61
|
+
)
|
|
62
|
+
values: dict[str, Any] = {
|
|
63
|
+
f.name: os.environ.get(f.name.upper(), configurable.get(f.name))
|
|
64
|
+
for f in fields(cls)
|
|
65
|
+
if f.init
|
|
66
|
+
}
|
|
67
|
+
return cls(**{k: v for k, v in values.items() if v})
|
|
68
|
+
|
|
69
|
+
@dataclass(kw_only=True)
|
|
70
|
+
class MultiAgentConfiguration:
|
|
71
|
+
"""Configuration for the multi-agent implementation (multi_agent.py)."""
|
|
72
|
+
# Common configuration
|
|
73
|
+
search_api: SearchAPI = SearchAPI.TAVILY
|
|
74
|
+
search_api_config: Optional[Dict[str, Any]] = None
|
|
75
|
+
process_search_results: Literal["summarize", "split_and_rerank"] | None = None
|
|
76
|
+
summarization_model_provider: str = "openai"
|
|
77
|
+
summarization_model: str = "gpt-4.1"
|
|
78
|
+
include_source_str: bool = False
|
|
79
|
+
|
|
80
|
+
# Multi-agent specific configuration
|
|
81
|
+
number_of_queries: int = 2 # Number of search queries to generate per section
|
|
82
|
+
supervisor_model: str = "anthropic:claude-sonnet-4-20250514"
|
|
83
|
+
researcher_model: str = "anthropic:claude-sonnet-4-20250514"
|
|
84
|
+
ask_for_clarification: bool = False # Whether to ask for clarification from the user
|
|
85
|
+
# MCP server configuration
|
|
86
|
+
mcp_server_config: Optional[Dict[str, Any]] = None
|
|
87
|
+
mcp_prompt: Optional[str] = None
|
|
88
|
+
mcp_tools_to_include: Optional[list[str]] = None
|
|
89
|
+
|
|
90
|
+
@classmethod
|
|
91
|
+
def from_runnable_config(
|
|
92
|
+
cls, config: Optional[RunnableConfig] = None
|
|
93
|
+
) -> "MultiAgentConfiguration":
|
|
94
|
+
"""Create a MultiAgentConfiguration instance from a RunnableConfig."""
|
|
95
|
+
configurable = (
|
|
96
|
+
config["configurable"] if config and "configurable" in config else {}
|
|
97
|
+
)
|
|
98
|
+
values: dict[str, Any] = {
|
|
99
|
+
f.name: os.environ.get(f.name.upper(), configurable.get(f.name))
|
|
100
|
+
for f in fields(cls)
|
|
101
|
+
if f.init
|
|
102
|
+
}
|
|
103
|
+
return cls(**{k: v for k, v in values.items() if v})
|
|
104
|
+
|
|
105
|
+
# Keep the old Configuration class for backward compatibility
|
|
106
|
+
Configuration = Configuration
|