research-guild-ai 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- research_guild_ai-0.1.0/.gitignore +41 -0
- research_guild_ai-0.1.0/LICENSE +21 -0
- research_guild_ai-0.1.0/PKG-INFO +310 -0
- research_guild_ai-0.1.0/README.md +270 -0
- research_guild_ai-0.1.0/RELEASE_CHECKLIST.md +115 -0
- research_guild_ai-0.1.0/agent/__init__.py +1 -0
- research_guild_ai-0.1.0/agent/cli.py +1062 -0
- research_guild_ai-0.1.0/agent/cli_config.py +352 -0
- research_guild_ai-0.1.0/agent/collector.py +1229 -0
- research_guild_ai-0.1.0/agent/config.py +342 -0
- research_guild_ai-0.1.0/agent/constants.py +114 -0
- research_guild_ai-0.1.0/agent/deep_dive.py +563 -0
- research_guild_ai-0.1.0/agent/discovery.py +276 -0
- research_guild_ai-0.1.0/agent/entities.py +650 -0
- research_guild_ai-0.1.0/agent/export/__init__.py +58 -0
- research_guild_ai-0.1.0/agent/export/docx_export.py +83 -0
- research_guild_ai-0.1.0/agent/export/formats.py +41 -0
- research_guild_ai-0.1.0/agent/export/pptx_export.py +78 -0
- research_guild_ai-0.1.0/agent/export/xlsx_export.py +73 -0
- research_guild_ai-0.1.0/agent/models.py +130 -0
- research_guild_ai-0.1.0/agent/monitoring.py +274 -0
- research_guild_ai-0.1.0/agent/paths.py +86 -0
- research_guild_ai-0.1.0/agent/pipeline/__init__.py +1 -0
- research_guild_ai-0.1.0/agent/pipeline/graph.py +203 -0
- research_guild_ai-0.1.0/agent/pipeline/llm.py +124 -0
- research_guild_ai-0.1.0/agent/pipeline/loader.py +165 -0
- research_guild_ai-0.1.0/agent/pipeline/nodes.py +557 -0
- research_guild_ai-0.1.0/agent/pipeline/state.py +44 -0
- research_guild_ai-0.1.0/agent/pipeline/usage.py +322 -0
- research_guild_ai-0.1.0/agent/prune.py +313 -0
- research_guild_ai-0.1.0/agent/schemas/__init__.py +29 -0
- research_guild_ai-0.1.0/agent/schemas/claims.py +61 -0
- research_guild_ai-0.1.0/agent/schemas/extraction.py +116 -0
- research_guild_ai-0.1.0/agent/schemas/review.py +111 -0
- research_guild_ai-0.1.0/agent/schemas/snapshot.py +46 -0
- research_guild_ai-0.1.0/agent/scoping.py +271 -0
- research_guild_ai-0.1.0/agent/tools/__init__.py +1 -0
- research_guild_ai-0.1.0/agent/tools/firecrawl.py +314 -0
- research_guild_ai-0.1.0/agent/utils/__init__.py +1 -0
- research_guild_ai-0.1.0/agent/utils/dates.py +400 -0
- research_guild_ai-0.1.0/pyproject.toml +82 -0
- research_guild_ai-0.1.0/requirements.txt +17 -0
- research_guild_ai-0.1.0/tests/test_cli_config.py +205 -0
- research_guild_ai-0.1.0/tests/test_collector.py +940 -0
- research_guild_ai-0.1.0/tests/test_config.py +159 -0
- research_guild_ai-0.1.0/tests/test_dates.py +142 -0
- research_guild_ai-0.1.0/tests/test_deep_dive.py +419 -0
- research_guild_ai-0.1.0/tests/test_discovery.py +105 -0
- research_guild_ai-0.1.0/tests/test_entities.py +376 -0
- research_guild_ai-0.1.0/tests/test_export.py +50 -0
- research_guild_ai-0.1.0/tests/test_firecrawl_tool.py +195 -0
- research_guild_ai-0.1.0/tests/test_llm_provider.py +123 -0
- research_guild_ai-0.1.0/tests/test_models.py +47 -0
- research_guild_ai-0.1.0/tests/test_monitoring.py +283 -0
- research_guild_ai-0.1.0/tests/test_paths.py +108 -0
- research_guild_ai-0.1.0/tests/test_pipeline.py +257 -0
- research_guild_ai-0.1.0/tests/test_prune.py +172 -0
- research_guild_ai-0.1.0/tests/test_research_mode.py +26 -0
- research_guild_ai-0.1.0/tests/test_review_schema.py +71 -0
- research_guild_ai-0.1.0/tests/test_scoping.py +64 -0
- research_guild_ai-0.1.0/tests/test_scoping_normalize.py +15 -0
- research_guild_ai-0.1.0/tests/test_usage.py +199 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Bytecode / caches
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
.pytest_cache/
|
|
6
|
+
.mypy_cache/
|
|
7
|
+
.ruff_cache/
|
|
8
|
+
*.egg-info/
|
|
9
|
+
.eggs/
|
|
10
|
+
|
|
11
|
+
# Virtualenvs
|
|
12
|
+
.venv/
|
|
13
|
+
venv/
|
|
14
|
+
ENV/
|
|
15
|
+
|
|
16
|
+
# Build / packaging
|
|
17
|
+
build/
|
|
18
|
+
dist/
|
|
19
|
+
*.whl
|
|
20
|
+
|
|
21
|
+
# Secrets / local env
|
|
22
|
+
.env
|
|
23
|
+
.env.*
|
|
24
|
+
!.env.example
|
|
25
|
+
credentials.json
|
|
26
|
+
**/credentials.json
|
|
27
|
+
|
|
28
|
+
# Editor / OS
|
|
29
|
+
.idea/
|
|
30
|
+
.vscode/
|
|
31
|
+
*.swp
|
|
32
|
+
.DS_Store
|
|
33
|
+
|
|
34
|
+
# Runtime data (project-local override via --data-dir ./data)
|
|
35
|
+
data/
|
|
36
|
+
!data/**/.gitkeep
|
|
37
|
+
|
|
38
|
+
# Keep empty placeholder dirs in git when present
|
|
39
|
+
!data/exports/.gitkeep
|
|
40
|
+
!data/entities/.gitkeep
|
|
41
|
+
!data/entity_deep_dives/.gitkeep
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Manishkarki
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: research-guild-ai
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI for competitive intelligence research: collect, analyze, and export source-cited reports.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Manishkarki1/research-guild-ai
|
|
6
|
+
Project-URL: Documentation, https://github.com/Manishkarki1/research-guild-ai/blob/main/README.md
|
|
7
|
+
Author: Manish karki
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: cli,competitive-intelligence,firecrawl,langgraph,research,research-guild-ai
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: firecrawl-py>=2.0.0
|
|
22
|
+
Requires-Dist: keyring>=25.0.0
|
|
23
|
+
Requires-Dist: langchain-anthropic>=0.3.0
|
|
24
|
+
Requires-Dist: langchain-core>=0.3.0
|
|
25
|
+
Requires-Dist: langchain-google-genai>=2.0.0
|
|
26
|
+
Requires-Dist: langchain-openai>=0.2.0
|
|
27
|
+
Requires-Dist: langgraph>=0.2.0
|
|
28
|
+
Requires-Dist: openpyxl>=3.1.0
|
|
29
|
+
Requires-Dist: platformdirs>=4.0.0
|
|
30
|
+
Requires-Dist: pydantic>=2.0.0
|
|
31
|
+
Requires-Dist: python-docx>=1.1.0
|
|
32
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
33
|
+
Requires-Dist: python-pptx>=1.0.0
|
|
34
|
+
Requires-Dist: typer>=0.12.0
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: build>=1.0.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: twine>=5.0.0; extra == 'dev'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# AI Researcher Crawler
|
|
42
|
+
|
|
43
|
+
Research any topic, company, or competitor from your terminal — and get cited, fact-checked reports you can export to Word, Excel, or PowerPoint.
|
|
44
|
+
|
|
45
|
+
## Quick demo
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
$ pip install research-guild-ai
|
|
49
|
+
$ airc init
|
|
50
|
+
Which LLM provider would you like to use?
|
|
51
|
+
1. OpenAI
|
|
52
|
+
2. Google Gemini
|
|
53
|
+
3. Anthropic Claude
|
|
54
|
+
Choice [1]: 2
|
|
55
|
+
Enter your Google Gemini API key:
|
|
56
|
+
Enter your Firecrawl API key:
|
|
57
|
+
Saved.
|
|
58
|
+
|
|
59
|
+
Setup complete. Config stored at ~/.config/ai-researcher-crawler/
|
|
60
|
+
|
|
61
|
+
$ airc collect
|
|
62
|
+
Writing to: ~/.local/share/ai-researcher-crawler/
|
|
63
|
+
|
|
64
|
+
=== AI Researcher Crawler — Scoping ===
|
|
65
|
+
|
|
66
|
+
What are you researching?
|
|
67
|
+
|
|
68
|
+
1. Business / tech competitive intelligence (companies, products, markets)
|
|
69
|
+
2. History (eras, past events, historical figures/topics)
|
|
70
|
+
3. Current events / recent news
|
|
71
|
+
4. General research (any other topic)
|
|
72
|
+
|
|
73
|
+
Choose a number [1]: 1
|
|
74
|
+
→ Mode: competitive
|
|
75
|
+
|
|
76
|
+
Research subject (be specific, e.g. 'HR SaaS' or 'AI code assistants' — not just 'market') []: Law Firm Management Systems
|
|
77
|
+
Target geography (comma-separated countries/regions, or leave blank for global): global
|
|
78
|
+
Tech keywords (comma-separated, e.g. llm, saas, mobile): saas, legaltech, practice-management
|
|
79
|
+
Known competitors (comma-separated company names; leave blank or type 'none' to discover): Clio, Amberlo, LEAP, NetDocuments
|
|
80
|
+
Research objective (what decision or question should this answer?) []: Compare offerings, pricing signals, and geographic focus
|
|
81
|
+
Research depth (quick | deep) [quick]: quick
|
|
82
|
+
Output formats (comma-separated, e.g. markdown_report, word file, xlsx): markdown_report, docx
|
|
83
|
+
|
|
84
|
+
Scope saved: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
85
|
+
|
|
86
|
+
Starting collection (competitor pages + web + GitHub)...
|
|
87
|
+
|
|
88
|
+
=== Collection Summary ===
|
|
89
|
+
Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
90
|
+
Research mode: competitive
|
|
91
|
+
Competitors: Clio, Amberlo, LEAP, NetDocuments
|
|
92
|
+
Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
93
|
+
|
|
94
|
+
Collected:
|
|
95
|
+
- Clio (homepage): https://www.clio.com/ → .../clio-homepage.md
|
|
96
|
+
- Amberlo (homepage): https://www.amberlo.io/ → .../amberlo-homepage.md
|
|
97
|
+
- LEAP (homepage): https://www.leaplegalsoftware.com/ → .../leap-homepage.md
|
|
98
|
+
- NetDocuments (homepage): https://www.netdocuments.com/ → .../netdocuments-homepage.md
|
|
99
|
+
- web (web-search): .../web-search.md
|
|
100
|
+
- github (github-search): .../github-search.md
|
|
101
|
+
|
|
102
|
+
$ airc analyze \
|
|
103
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
|
|
104
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
105
|
+
|
|
106
|
+
=== Analysis Summary ===
|
|
107
|
+
Documents analyzed: 12
|
|
108
|
+
Confidence: medium
|
|
109
|
+
Research loops used: 0
|
|
110
|
+
Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
|
|
111
|
+
|
|
112
|
+
# From the report (excerpt):
|
|
113
|
+
# Generative and agentic AI are becoming core platform features
|
|
114
|
+
# (e.g. Clio Work, ndMAX) [clio/features, https://...; netdocuments/features, https://...]
|
|
115
|
+
#
|
|
116
|
+
# | AI Capabilities | Clio Work ... | *Not found in collected sources* | ...
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Install
|
|
120
|
+
|
|
121
|
+
Requires **Python 3.11+**.
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
pip install research-guild-ai
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Until the package is on PyPI, install from GitHub or a local wheel:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
# From this repo (subdirectory install)
|
|
131
|
+
pip install "git+https://github.com/Manishkarki1/ai-researcher.git#subdirectory=ai-researcher-crawler"
|
|
132
|
+
|
|
133
|
+
# Or build a wheel locally, then:
|
|
134
|
+
# cd ai-researcher-crawler && python -m build
|
|
135
|
+
pip install dist/ai_researcher_crawler-*.whl
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Confirm the CLI is on your PATH:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
airc --help
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## Setup
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
airc init
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
This asks which LLM provider to use (**OpenAI**, **Google Gemini**, or **Anthropic Claude**), prompts for that provider’s API key, then asks for a **Firecrawl** API key (used to collect public web sources). Keys are stored in your OS credential store when available — not in a plaintext project `.env`. If the OS keyring is unavailable, they fall back to a mode-`0600` file under the config directory.
|
|
151
|
+
|
|
152
|
+
Get keys here:
|
|
153
|
+
|
|
154
|
+
| Service | Signup |
|
|
155
|
+
|---|---|
|
|
156
|
+
| Firecrawl | https://www.firecrawl.dev/ |
|
|
157
|
+
| OpenAI | https://platform.openai.com/api-keys |
|
|
158
|
+
| Google Gemini | https://aistudio.google.com/apikey |
|
|
159
|
+
| Anthropic | https://console.anthropic.com/ |
|
|
160
|
+
|
|
161
|
+
You can also set keys later with `airc config set-key`, or via environment variables for CI (see [Configuration](#configuration-reference)).
|
|
162
|
+
|
|
163
|
+
## Research modes
|
|
164
|
+
|
|
165
|
+
| Mode | Use it for |
|
|
166
|
+
|---|---|
|
|
167
|
+
| **Competitive** | Comparing companies, products, markets |
|
|
168
|
+
| **History** | Past events, eras, historical figures |
|
|
169
|
+
| **Current events** | Recent news, ongoing situations |
|
|
170
|
+
| **General** | Anything else |
|
|
171
|
+
|
|
172
|
+
## Basic usage
|
|
173
|
+
|
|
174
|
+
### Collect sources
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
airc collect
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Interactive prompts pick a mode, save a scope file, optionally discover competitors (competitive mode), and scrape public pages into a timestamped raw folder.
|
|
181
|
+
|
|
182
|
+
```text
|
|
183
|
+
=== Collection Summary ===
|
|
184
|
+
Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
185
|
+
Research mode: competitive
|
|
186
|
+
Competitors: Clio, Amberlo, LEAP, NetDocuments
|
|
187
|
+
Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Collect and analyze in one step:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
airc collect --analyze
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Analyze into a report
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
airc analyze \
|
|
200
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
|
|
201
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
```text
|
|
205
|
+
=== Analysis Summary ===
|
|
206
|
+
Documents analyzed: 12
|
|
207
|
+
Confidence: medium
|
|
208
|
+
Research loops used: 0
|
|
209
|
+
Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
### Export
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
airc export \
|
|
216
|
+
--report ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md \
|
|
217
|
+
--formats docx,xlsx,pptx
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
```text
|
|
221
|
+
=== Exports ===
|
|
222
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.docx
|
|
223
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.xlsx
|
|
224
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.pptx
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## What you get
|
|
228
|
+
|
|
229
|
+
Every claim is **cited** and **confidence-scored**. When something isn’t in the collected sources, the report says so (`not found in collected sources` / `not publicly disclosed`) instead of guessing. Single-source claims are flagged as unverified.
|
|
230
|
+
|
|
231
|
+
Exports: **Markdown**, **Word (docx)**, **Excel (xlsx)**, and **PowerPoint (pptx)**.
|
|
232
|
+
|
|
233
|
+
## Entity deep-dive
|
|
234
|
+
|
|
235
|
+
After analysis, the tool lists people and organizations mentioned in the research and lets you follow up on selected ones. Public figures and organization representatives can get a fuller follow-up research pass. **Private individuals** mentioned in coverage get a limited, source-only summary — the tool will not go digging for more about a private person, and those names are not written into the cross-run entities index.
|
|
236
|
+
|
|
237
|
+
```text
|
|
238
|
+
Detected entities:
|
|
239
|
+
1. Clio — legal practice management vendor (organization_representative) — full deep-dive available
|
|
240
|
+
2. Jane Doe — mentioned in news coverage (incident_victim_or_private_citizen) — known-information summary only, no new research
|
|
241
|
+
|
|
242
|
+
Select entities (comma-separated numbers), or:
|
|
243
|
+
all-public → select all public_figure / organization_representative entities
|
|
244
|
+
skip → skip
|
|
245
|
+
Selection [skip]:
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
## Monitoring / re-running research
|
|
249
|
+
|
|
250
|
+
Re-run analysis with `--compare-to <prior_run_id>` to insert a **What Changed** section against a previous run’s snapshot:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
airc analyze \
|
|
254
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_YYYYMMDDTHHMMSSZ.json \
|
|
255
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/YYYYMMDDTHHMMSSZ \
|
|
256
|
+
--compare-to 20260712T173833Z
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
```text
|
|
260
|
+
What Changed section inserted (vs 20260712T173833Z)
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
## Configuration reference
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
airc config show # provider, model, masked key status, paths
|
|
267
|
+
airc config set-key openai|gemini|anthropic|firecrawl
|
|
268
|
+
airc config reset # clear stored config + credentials
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
**Data directory** (scopes, raw scrapes, reports, exports):
|
|
272
|
+
|
|
273
|
+
| Priority | Source |
|
|
274
|
+
|---|---|
|
|
275
|
+
| 1 | `airc --data-dir PATH …` |
|
|
276
|
+
| 2 | `AIRC_DATA_DIR` |
|
|
277
|
+
| 3 | OS user data dir (e.g. `~/.local/share/ai-researcher-crawler/`) |
|
|
278
|
+
|
|
279
|
+
**CI / non-interactive fallbacks** (no prompts):
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
export FIRECRAWL_API_KEY=...
|
|
283
|
+
export LLM_PROVIDER=openai # or gemini | anthropic
|
|
284
|
+
export OPENAI_API_KEY=... # or GOOGLE_API_KEY / ANTHROPIC_API_KEY
|
|
285
|
+
export AIRC_DATA_DIR=/var/lib/airc
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
CLI flags (`--firecrawl-api-key`, `--llm-provider`, `--openai-api-key`, …) override env for a single run.
|
|
289
|
+
|
|
290
|
+
## Limitations
|
|
291
|
+
|
|
292
|
+
- Collection respects site access norms (including robots.txt / terms of service via the crawl provider), so some sites will not be scraped.
|
|
293
|
+
- LLM analysis can still make mistakes; the fact-checking design reduces silent invention, it does not eliminate all error.
|
|
294
|
+
- `deep` research depth costs more in Firecrawl and LLM usage than `quick`.
|
|
295
|
+
- Competitor discovery is heuristic — always review the proposed list before accepting it.
|
|
296
|
+
- The package is early (0.1.x); expect CLI and output shape to evolve.
|
|
297
|
+
|
|
298
|
+
## Ethics / responsible use
|
|
299
|
+
|
|
300
|
+
This tool is meant for **public** information only. It does not bypass logins, paywalls, or CAPTCHAs. It includes safeguards so private individuals mentioned in coverage are not turned into dug-up personal profiles from aggregated public data. Use it for organizational research, public figures, and topics — not for building dossiers on private people.
|
|
301
|
+
|
|
302
|
+
For the full anti-hallucination design (citations, claim kinds, reviewer loop), see [DOCUMENTATION.md §11](DOCUMENTATION.md#11-anti-hallucination-design).
|
|
303
|
+
|
|
304
|
+
## Contributing
|
|
305
|
+
|
|
306
|
+
Issues and PRs welcome. Architecture, pipeline internals, and module deep-dives live in [DOCUMENTATION.md](DOCUMENTATION.md) — start there if you want to change how the system works.
|
|
307
|
+
|
|
308
|
+
## License
|
|
309
|
+
|
|
310
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# AI Researcher Crawler
|
|
2
|
+
|
|
3
|
+
Research any topic, company, or competitor from your terminal — and get cited, fact-checked reports you can export to Word, Excel, or PowerPoint.
|
|
4
|
+
|
|
5
|
+
## Quick demo
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
$ pip install research-guild-ai
|
|
9
|
+
$ airc init
|
|
10
|
+
Which LLM provider would you like to use?
|
|
11
|
+
1. OpenAI
|
|
12
|
+
2. Google Gemini
|
|
13
|
+
3. Anthropic Claude
|
|
14
|
+
Choice [1]: 2
|
|
15
|
+
Enter your Google Gemini API key:
|
|
16
|
+
Enter your Firecrawl API key:
|
|
17
|
+
Saved.
|
|
18
|
+
|
|
19
|
+
Setup complete. Config stored at ~/.config/ai-researcher-crawler/
|
|
20
|
+
|
|
21
|
+
$ airc collect
|
|
22
|
+
Writing to: ~/.local/share/ai-researcher-crawler/
|
|
23
|
+
|
|
24
|
+
=== AI Researcher Crawler — Scoping ===
|
|
25
|
+
|
|
26
|
+
What are you researching?
|
|
27
|
+
|
|
28
|
+
1. Business / tech competitive intelligence (companies, products, markets)
|
|
29
|
+
2. History (eras, past events, historical figures/topics)
|
|
30
|
+
3. Current events / recent news
|
|
31
|
+
4. General research (any other topic)
|
|
32
|
+
|
|
33
|
+
Choose a number [1]: 1
|
|
34
|
+
→ Mode: competitive
|
|
35
|
+
|
|
36
|
+
Research subject (be specific, e.g. 'HR SaaS' or 'AI code assistants' — not just 'market') []: Law Firm Management Systems
|
|
37
|
+
Target geography (comma-separated countries/regions, or leave blank for global): global
|
|
38
|
+
Tech keywords (comma-separated, e.g. llm, saas, mobile): saas, legaltech, practice-management
|
|
39
|
+
Known competitors (comma-separated company names; leave blank or type 'none' to discover): Clio, Amberlo, LEAP, NetDocuments
|
|
40
|
+
Research objective (what decision or question should this answer?) []: Compare offerings, pricing signals, and geographic focus
|
|
41
|
+
Research depth (quick | deep) [quick]: quick
|
|
42
|
+
Output formats (comma-separated, e.g. markdown_report, word file, xlsx): markdown_report, docx
|
|
43
|
+
|
|
44
|
+
Scope saved: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
45
|
+
|
|
46
|
+
Starting collection (competitor pages + web + GitHub)...
|
|
47
|
+
|
|
48
|
+
=== Collection Summary ===
|
|
49
|
+
Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
50
|
+
Research mode: competitive
|
|
51
|
+
Competitors: Clio, Amberlo, LEAP, NetDocuments
|
|
52
|
+
Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
53
|
+
|
|
54
|
+
Collected:
|
|
55
|
+
- Clio (homepage): https://www.clio.com/ → .../clio-homepage.md
|
|
56
|
+
- Amberlo (homepage): https://www.amberlo.io/ → .../amberlo-homepage.md
|
|
57
|
+
- LEAP (homepage): https://www.leaplegalsoftware.com/ → .../leap-homepage.md
|
|
58
|
+
- NetDocuments (homepage): https://www.netdocuments.com/ → .../netdocuments-homepage.md
|
|
59
|
+
- web (web-search): .../web-search.md
|
|
60
|
+
- github (github-search): .../github-search.md
|
|
61
|
+
|
|
62
|
+
$ airc analyze \
|
|
63
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
|
|
64
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
65
|
+
|
|
66
|
+
=== Analysis Summary ===
|
|
67
|
+
Documents analyzed: 12
|
|
68
|
+
Confidence: medium
|
|
69
|
+
Research loops used: 0
|
|
70
|
+
Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
|
|
71
|
+
|
|
72
|
+
# From the report (excerpt):
|
|
73
|
+
# Generative and agentic AI are becoming core platform features
|
|
74
|
+
# (e.g. Clio Work, ndMAX) [clio/features, https://...; netdocuments/features, https://...]
|
|
75
|
+
#
|
|
76
|
+
# | AI Capabilities | Clio Work ... | *Not found in collected sources* | ...
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Install
|
|
80
|
+
|
|
81
|
+
Requires **Python 3.11+**.
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install research-guild-ai
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Until the package is on PyPI, install from GitHub or a local wheel:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
# From this repo (subdirectory install)
|
|
91
|
+
pip install "git+https://github.com/Manishkarki1/ai-researcher.git#subdirectory=ai-researcher-crawler"
|
|
92
|
+
|
|
93
|
+
# Or build a wheel locally, then:
|
|
94
|
+
# cd ai-researcher-crawler && python -m build
|
|
95
|
+
pip install dist/ai_researcher_crawler-*.whl
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Confirm the CLI is on your PATH:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
airc --help
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Setup
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
airc init
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
This asks which LLM provider to use (**OpenAI**, **Google Gemini**, or **Anthropic Claude**), prompts for that provider’s API key, then asks for a **Firecrawl** API key (used to collect public web sources). Keys are stored in your OS credential store when available — not in a plaintext project `.env`. If the OS keyring is unavailable, they fall back to a mode-`0600` file under the config directory.
|
|
111
|
+
|
|
112
|
+
Get keys here:
|
|
113
|
+
|
|
114
|
+
| Service | Signup |
|
|
115
|
+
|---|---|
|
|
116
|
+
| Firecrawl | https://www.firecrawl.dev/ |
|
|
117
|
+
| OpenAI | https://platform.openai.com/api-keys |
|
|
118
|
+
| Google Gemini | https://aistudio.google.com/apikey |
|
|
119
|
+
| Anthropic | https://console.anthropic.com/ |
|
|
120
|
+
|
|
121
|
+
You can also set keys later with `airc config set-key`, or via environment variables for CI (see [Configuration](#configuration-reference)).
|
|
122
|
+
|
|
123
|
+
## Research modes
|
|
124
|
+
|
|
125
|
+
| Mode | Use it for |
|
|
126
|
+
|---|---|
|
|
127
|
+
| **Competitive** | Comparing companies, products, markets |
|
|
128
|
+
| **History** | Past events, eras, historical figures |
|
|
129
|
+
| **Current events** | Recent news, ongoing situations |
|
|
130
|
+
| **General** | Anything else |
|
|
131
|
+
|
|
132
|
+
## Basic usage
|
|
133
|
+
|
|
134
|
+
### Collect sources
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
airc collect
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Interactive prompts pick a mode, save a scope file, optionally discover competitors (competitive mode), and scrape public pages into a timestamped raw folder.
|
|
141
|
+
|
|
142
|
+
```text
|
|
143
|
+
=== Collection Summary ===
|
|
144
|
+
Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
|
|
145
|
+
Research mode: competitive
|
|
146
|
+
Competitors: Clio, Amberlo, LEAP, NetDocuments
|
|
147
|
+
Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Collect and analyze in one step:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
airc collect --analyze
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Analyze into a report
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
airc analyze \
|
|
160
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
|
|
161
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
```text
|
|
165
|
+
=== Analysis Summary ===
|
|
166
|
+
Documents analyzed: 12
|
|
167
|
+
Confidence: medium
|
|
168
|
+
Research loops used: 0
|
|
169
|
+
Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
### Export
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
airc export \
|
|
176
|
+
--report ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md \
|
|
177
|
+
--formats docx,xlsx,pptx
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
```text
|
|
181
|
+
=== Exports ===
|
|
182
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.docx
|
|
183
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.xlsx
|
|
184
|
+
- ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.pptx
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## What you get
|
|
188
|
+
|
|
189
|
+
Every claim is **cited** and **confidence-scored**. When something isn’t in the collected sources, the report says so (`not found in collected sources` / `not publicly disclosed`) instead of guessing. Single-source claims are flagged as unverified.
|
|
190
|
+
|
|
191
|
+
Exports: **Markdown**, **Word (docx)**, **Excel (xlsx)**, and **PowerPoint (pptx)**.
|
|
192
|
+
|
|
193
|
+
## Entity deep-dive
|
|
194
|
+
|
|
195
|
+
After analysis, the tool lists people and organizations mentioned in the research and lets you follow up on selected ones. Public figures and organization representatives can get a fuller follow-up research pass. **Private individuals** mentioned in coverage get a limited, source-only summary — the tool will not go digging for more about a private person, and those names are not written into the cross-run entities index.
|
|
196
|
+
|
|
197
|
+
```text
|
|
198
|
+
Detected entities:
|
|
199
|
+
1. Clio — legal practice management vendor (organization_representative) — full deep-dive available
|
|
200
|
+
2. Jane Doe — mentioned in news coverage (incident_victim_or_private_citizen) — known-information summary only, no new research
|
|
201
|
+
|
|
202
|
+
Select entities (comma-separated numbers), or:
|
|
203
|
+
all-public → select all public_figure / organization_representative entities
|
|
204
|
+
skip → skip
|
|
205
|
+
Selection [skip]:
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Monitoring / re-running research
|
|
209
|
+
|
|
210
|
+
Re-run analysis with `--compare-to <prior_run_id>` to insert a **What Changed** section against a previous run’s snapshot:
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
airc analyze \
|
|
214
|
+
--scope-file ~/.local/share/ai-researcher-crawler/scope/scope_YYYYMMDDTHHMMSSZ.json \
|
|
215
|
+
--raw-dir ~/.local/share/ai-researcher-crawler/raw/YYYYMMDDTHHMMSSZ \
|
|
216
|
+
--compare-to 20260712T173833Z
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
```text
|
|
220
|
+
What Changed section inserted (vs 20260712T173833Z)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## Configuration reference
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
airc config show # provider, model, masked key status, paths
|
|
227
|
+
airc config set-key openai|gemini|anthropic|firecrawl
|
|
228
|
+
airc config reset # clear stored config + credentials
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
**Data directory** (scopes, raw scrapes, reports, exports):
|
|
232
|
+
|
|
233
|
+
| Priority | Source |
|
|
234
|
+
|---|---|
|
|
235
|
+
| 1 | `airc --data-dir PATH …` |
|
|
236
|
+
| 2 | `AIRC_DATA_DIR` |
|
|
237
|
+
| 3 | OS user data dir (e.g. `~/.local/share/ai-researcher-crawler/`) |
|
|
238
|
+
|
|
239
|
+
**CI / non-interactive fallbacks** (no prompts):
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
export FIRECRAWL_API_KEY=...
|
|
243
|
+
export LLM_PROVIDER=openai # or gemini | anthropic
|
|
244
|
+
export OPENAI_API_KEY=... # or GOOGLE_API_KEY / ANTHROPIC_API_KEY
|
|
245
|
+
export AIRC_DATA_DIR=/var/lib/airc
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
CLI flags (`--firecrawl-api-key`, `--llm-provider`, `--openai-api-key`, …) override env for a single run.
|
|
249
|
+
|
|
250
|
+
## Limitations
|
|
251
|
+
|
|
252
|
+
- Collection respects site access norms (including robots.txt / terms of service via the crawl provider), so some sites will not be scraped.
|
|
253
|
+
- LLM analysis can still make mistakes; the fact-checking design reduces silent invention, it does not eliminate all error.
|
|
254
|
+
- `deep` research depth costs more in Firecrawl and LLM usage than `quick`.
|
|
255
|
+
- Competitor discovery is heuristic — always review the proposed list before accepting it.
|
|
256
|
+
- The package is early (0.1.x); expect CLI and output shape to evolve.
|
|
257
|
+
|
|
258
|
+
## Ethics / responsible use
|
|
259
|
+
|
|
260
|
+
This tool is meant for **public** information only. It does not bypass logins, paywalls, or CAPTCHAs. It includes safeguards so private individuals mentioned in coverage are not turned into dug-up personal profiles from aggregated public data. Use it for organizational research, public figures, and topics — not for building dossiers on private people.
|
|
261
|
+
|
|
262
|
+
For the full anti-hallucination design (citations, claim kinds, reviewer loop), see [DOCUMENTATION.md §11](DOCUMENTATION.md#11-anti-hallucination-design).
|
|
263
|
+
|
|
264
|
+
## Contributing
|
|
265
|
+
|
|
266
|
+
Issues and PRs welcome. Architecture, pipeline internals, and module deep-dives live in [DOCUMENTATION.md](DOCUMENTATION.md) — start there if you want to change how the system works.
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
[MIT](LICENSE)
|