research-guild-ai 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. research_guild_ai-0.1.0/.gitignore +41 -0
  2. research_guild_ai-0.1.0/LICENSE +21 -0
  3. research_guild_ai-0.1.0/PKG-INFO +310 -0
  4. research_guild_ai-0.1.0/README.md +270 -0
  5. research_guild_ai-0.1.0/RELEASE_CHECKLIST.md +115 -0
  6. research_guild_ai-0.1.0/agent/__init__.py +1 -0
  7. research_guild_ai-0.1.0/agent/cli.py +1062 -0
  8. research_guild_ai-0.1.0/agent/cli_config.py +352 -0
  9. research_guild_ai-0.1.0/agent/collector.py +1229 -0
  10. research_guild_ai-0.1.0/agent/config.py +342 -0
  11. research_guild_ai-0.1.0/agent/constants.py +114 -0
  12. research_guild_ai-0.1.0/agent/deep_dive.py +563 -0
  13. research_guild_ai-0.1.0/agent/discovery.py +276 -0
  14. research_guild_ai-0.1.0/agent/entities.py +650 -0
  15. research_guild_ai-0.1.0/agent/export/__init__.py +58 -0
  16. research_guild_ai-0.1.0/agent/export/docx_export.py +83 -0
  17. research_guild_ai-0.1.0/agent/export/formats.py +41 -0
  18. research_guild_ai-0.1.0/agent/export/pptx_export.py +78 -0
  19. research_guild_ai-0.1.0/agent/export/xlsx_export.py +73 -0
  20. research_guild_ai-0.1.0/agent/models.py +130 -0
  21. research_guild_ai-0.1.0/agent/monitoring.py +274 -0
  22. research_guild_ai-0.1.0/agent/paths.py +86 -0
  23. research_guild_ai-0.1.0/agent/pipeline/__init__.py +1 -0
  24. research_guild_ai-0.1.0/agent/pipeline/graph.py +203 -0
  25. research_guild_ai-0.1.0/agent/pipeline/llm.py +124 -0
  26. research_guild_ai-0.1.0/agent/pipeline/loader.py +165 -0
  27. research_guild_ai-0.1.0/agent/pipeline/nodes.py +557 -0
  28. research_guild_ai-0.1.0/agent/pipeline/state.py +44 -0
  29. research_guild_ai-0.1.0/agent/pipeline/usage.py +322 -0
  30. research_guild_ai-0.1.0/agent/prune.py +313 -0
  31. research_guild_ai-0.1.0/agent/schemas/__init__.py +29 -0
  32. research_guild_ai-0.1.0/agent/schemas/claims.py +61 -0
  33. research_guild_ai-0.1.0/agent/schemas/extraction.py +116 -0
  34. research_guild_ai-0.1.0/agent/schemas/review.py +111 -0
  35. research_guild_ai-0.1.0/agent/schemas/snapshot.py +46 -0
  36. research_guild_ai-0.1.0/agent/scoping.py +271 -0
  37. research_guild_ai-0.1.0/agent/tools/__init__.py +1 -0
  38. research_guild_ai-0.1.0/agent/tools/firecrawl.py +314 -0
  39. research_guild_ai-0.1.0/agent/utils/__init__.py +1 -0
  40. research_guild_ai-0.1.0/agent/utils/dates.py +400 -0
  41. research_guild_ai-0.1.0/pyproject.toml +82 -0
  42. research_guild_ai-0.1.0/requirements.txt +17 -0
  43. research_guild_ai-0.1.0/tests/test_cli_config.py +205 -0
  44. research_guild_ai-0.1.0/tests/test_collector.py +940 -0
  45. research_guild_ai-0.1.0/tests/test_config.py +159 -0
  46. research_guild_ai-0.1.0/tests/test_dates.py +142 -0
  47. research_guild_ai-0.1.0/tests/test_deep_dive.py +419 -0
  48. research_guild_ai-0.1.0/tests/test_discovery.py +105 -0
  49. research_guild_ai-0.1.0/tests/test_entities.py +376 -0
  50. research_guild_ai-0.1.0/tests/test_export.py +50 -0
  51. research_guild_ai-0.1.0/tests/test_firecrawl_tool.py +195 -0
  52. research_guild_ai-0.1.0/tests/test_llm_provider.py +123 -0
  53. research_guild_ai-0.1.0/tests/test_models.py +47 -0
  54. research_guild_ai-0.1.0/tests/test_monitoring.py +283 -0
  55. research_guild_ai-0.1.0/tests/test_paths.py +108 -0
  56. research_guild_ai-0.1.0/tests/test_pipeline.py +257 -0
  57. research_guild_ai-0.1.0/tests/test_prune.py +172 -0
  58. research_guild_ai-0.1.0/tests/test_research_mode.py +26 -0
  59. research_guild_ai-0.1.0/tests/test_review_schema.py +71 -0
  60. research_guild_ai-0.1.0/tests/test_scoping.py +64 -0
  61. research_guild_ai-0.1.0/tests/test_scoping_normalize.py +15 -0
  62. research_guild_ai-0.1.0/tests/test_usage.py +199 -0
@@ -0,0 +1,41 @@
1
+ # Bytecode / caches
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ .pytest_cache/
6
+ .mypy_cache/
7
+ .ruff_cache/
8
+ *.egg-info/
9
+ .eggs/
10
+
11
+ # Virtualenvs
12
+ .venv/
13
+ venv/
14
+ ENV/
15
+
16
+ # Build / packaging
17
+ build/
18
+ dist/
19
+ *.whl
20
+
21
+ # Secrets / local env
22
+ .env
23
+ .env.*
24
+ !.env.example
25
+ credentials.json
26
+ **/credentials.json
27
+
28
+ # Editor / OS
29
+ .idea/
30
+ .vscode/
31
+ *.swp
32
+ .DS_Store
33
+
34
+ # Runtime data (project-local override via --data-dir ./data)
35
+ data/
36
+ !data/**/.gitkeep
37
+
38
+ # Keep empty placeholder dirs in git when present
39
+ !data/exports/.gitkeep
40
+ !data/entities/.gitkeep
41
+ !data/entity_deep_dives/.gitkeep
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Manishkarki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,310 @@
1
+ Metadata-Version: 2.4
2
+ Name: research-guild-ai
3
+ Version: 0.1.0
4
+ Summary: CLI for competitive intelligence research: collect, analyze, and export source-cited reports.
5
+ Project-URL: Homepage, https://github.com/Manishkarki1/research-guild-ai
6
+ Project-URL: Documentation, https://github.com/Manishkarki1/research-guild-ai/blob/main/README.md
7
+ Author: Manish karki
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: cli,competitive-intelligence,firecrawl,langgraph,research,research-guild-ai
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
20
+ Requires-Python: >=3.11
21
+ Requires-Dist: firecrawl-py>=2.0.0
22
+ Requires-Dist: keyring>=25.0.0
23
+ Requires-Dist: langchain-anthropic>=0.3.0
24
+ Requires-Dist: langchain-core>=0.3.0
25
+ Requires-Dist: langchain-google-genai>=2.0.0
26
+ Requires-Dist: langchain-openai>=0.2.0
27
+ Requires-Dist: langgraph>=0.2.0
28
+ Requires-Dist: openpyxl>=3.1.0
29
+ Requires-Dist: platformdirs>=4.0.0
30
+ Requires-Dist: pydantic>=2.0.0
31
+ Requires-Dist: python-docx>=1.1.0
32
+ Requires-Dist: python-dotenv>=1.0.0
33
+ Requires-Dist: python-pptx>=1.0.0
34
+ Requires-Dist: typer>=0.12.0
35
+ Provides-Extra: dev
36
+ Requires-Dist: build>=1.0.0; extra == 'dev'
37
+ Requires-Dist: pytest>=8.0.0; extra == 'dev'
38
+ Requires-Dist: twine>=5.0.0; extra == 'dev'
39
+ Description-Content-Type: text/markdown
40
+
41
+ # AI Researcher Crawler
42
+
43
+ Research any topic, company, or competitor from your terminal — and get cited, fact-checked reports you can export to Word, Excel, or PowerPoint.
44
+
45
+ ## Quick demo
46
+
47
+ ```text
48
+ $ pip install research-guild-ai
49
+ $ airc init
50
+ Which LLM provider would you like to use?
51
+ 1. OpenAI
52
+ 2. Google Gemini
53
+ 3. Anthropic Claude
54
+ Choice [1]: 2
55
+ Enter your Google Gemini API key:
56
+ Enter your Firecrawl API key:
57
+ Saved.
58
+
59
+ Setup complete. Config stored at ~/.config/ai-researcher-crawler/
60
+
61
+ $ airc collect
62
+ Writing to: ~/.local/share/ai-researcher-crawler/
63
+
64
+ === AI Researcher Crawler — Scoping ===
65
+
66
+ What are you researching?
67
+
68
+ 1. Business / tech competitive intelligence (companies, products, markets)
69
+ 2. History (eras, past events, historical figures/topics)
70
+ 3. Current events / recent news
71
+ 4. General research (any other topic)
72
+
73
+ Choose a number [1]: 1
74
+ → Mode: competitive
75
+
76
+ Research subject (be specific, e.g. 'HR SaaS' or 'AI code assistants' — not just 'market') []: Law Firm Management Systems
77
+ Target geography (comma-separated countries/regions, or leave blank for global): global
78
+ Tech keywords (comma-separated, e.g. llm, saas, mobile): saas, legaltech, practice-management
79
+ Known competitors (comma-separated company names; leave blank or type 'none' to discover): Clio, Amberlo, LEAP, NetDocuments
80
+ Research objective (what decision or question should this answer?) []: Compare offerings, pricing signals, and geographic focus
81
+ Research depth (quick | deep) [quick]: quick
82
+ Output formats (comma-separated, e.g. markdown_report, word file, xlsx): markdown_report, docx
83
+
84
+ Scope saved: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
85
+
86
+ Starting collection (competitor pages + web + GitHub)...
87
+
88
+ === Collection Summary ===
89
+ Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
90
+ Research mode: competitive
91
+ Competitors: Clio, Amberlo, LEAP, NetDocuments
92
+ Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
93
+
94
+ Collected:
95
+ - Clio (homepage): https://www.clio.com/ → .../clio-homepage.md
96
+ - Amberlo (homepage): https://www.amberlo.io/ → .../amberlo-homepage.md
97
+ - LEAP (homepage): https://www.leaplegalsoftware.com/ → .../leap-homepage.md
98
+ - NetDocuments (homepage): https://www.netdocuments.com/ → .../netdocuments-homepage.md
99
+ - web (web-search): .../web-search.md
100
+ - github (github-search): .../github-search.md
101
+
102
+ $ airc analyze \
103
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
104
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
105
+
106
+ === Analysis Summary ===
107
+ Documents analyzed: 12
108
+ Confidence: medium
109
+ Research loops used: 0
110
+ Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
111
+
112
+ # From the report (excerpt):
113
+ # Generative and agentic AI are becoming core platform features
114
+ # (e.g. Clio Work, ndMAX) [clio/features, https://...; netdocuments/features, https://...]
115
+ #
116
+ # | AI Capabilities | Clio Work ... | *Not found in collected sources* | ...
117
+ ```
118
+
119
+ ## Install
120
+
121
+ Requires **Python 3.11+**.
122
+
123
+ ```bash
124
+ pip install research-guild-ai
125
+ ```
126
+
127
+ Until the package is on PyPI, install from GitHub or a local wheel:
128
+
129
+ ```bash
130
+ # From this repo (subdirectory install)
131
+ pip install "git+https://github.com/Manishkarki1/ai-researcher.git#subdirectory=ai-researcher-crawler"
132
+
133
+ # Or build a wheel locally, then:
134
+ # cd ai-researcher-crawler && python -m build
135
+ pip install dist/ai_researcher_crawler-*.whl
136
+ ```
137
+
138
+ Confirm the CLI is on your PATH:
139
+
140
+ ```bash
141
+ airc --help
142
+ ```
143
+
144
+ ## Setup
145
+
146
+ ```bash
147
+ airc init
148
+ ```
149
+
150
+ This asks which LLM provider to use (**OpenAI**, **Google Gemini**, or **Anthropic Claude**), prompts for that provider’s API key, then asks for a **Firecrawl** API key (used to collect public web sources). Keys are stored in your OS credential store when available — not in a plaintext project `.env`. If the OS keyring is unavailable, they fall back to a mode-`0600` file under the config directory.
151
+
152
+ Get keys here:
153
+
154
+ | Service | Signup |
155
+ |---|---|
156
+ | Firecrawl | https://www.firecrawl.dev/ |
157
+ | OpenAI | https://platform.openai.com/api-keys |
158
+ | Google Gemini | https://aistudio.google.com/apikey |
159
+ | Anthropic | https://console.anthropic.com/ |
160
+
161
+ You can also set keys later with `airc config set-key`, or via environment variables for CI (see [Configuration](#configuration-reference)).
162
+
163
+ ## Research modes
164
+
165
+ | Mode | Use it for |
166
+ |---|---|
167
+ | **Competitive** | Comparing companies, products, markets |
168
+ | **History** | Past events, eras, historical figures |
169
+ | **Current events** | Recent news, ongoing situations |
170
+ | **General** | Anything else |
171
+
172
+ ## Basic usage
173
+
174
+ ### Collect sources
175
+
176
+ ```bash
177
+ airc collect
178
+ ```
179
+
180
+ Interactive prompts pick a mode, save a scope file, optionally discover competitors (competitive mode), and scrape public pages into a timestamped raw folder.
181
+
182
+ ```text
183
+ === Collection Summary ===
184
+ Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
185
+ Research mode: competitive
186
+ Competitors: Clio, Amberlo, LEAP, NetDocuments
187
+ Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
188
+ ```
189
+
190
+ Collect and analyze in one step:
191
+
192
+ ```bash
193
+ airc collect --analyze
194
+ ```
195
+
196
+ ### Analyze into a report
197
+
198
+ ```bash
199
+ airc analyze \
200
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
201
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
202
+ ```
203
+
204
+ ```text
205
+ === Analysis Summary ===
206
+ Documents analyzed: 12
207
+ Confidence: medium
208
+ Research loops used: 0
209
+ Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
210
+ ```
211
+
212
+ ### Export
213
+
214
+ ```bash
215
+ airc export \
216
+ --report ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md \
217
+ --formats docx,xlsx,pptx
218
+ ```
219
+
220
+ ```text
221
+ === Exports ===
222
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.docx
223
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.xlsx
224
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.pptx
225
+ ```
226
+
227
+ ## What you get
228
+
229
+ Every claim is **cited** and **confidence-scored**. When something isn’t in the collected sources, the report says so (`not found in collected sources` / `not publicly disclosed`) instead of guessing. Single-source claims are flagged as unverified.
230
+
231
+ Exports: **Markdown**, **Word (docx)**, **Excel (xlsx)**, and **PowerPoint (pptx)**.
232
+
233
+ ## Entity deep-dive
234
+
235
+ After analysis, the tool lists people and organizations mentioned in the research and lets you follow up on selected ones. Public figures and organization representatives can get a fuller follow-up research pass. **Private individuals** mentioned in coverage get a limited, source-only summary — the tool will not go digging for more about a private person, and those names are not written into the cross-run entities index.
236
+
237
+ ```text
238
+ Detected entities:
239
+ 1. Clio — legal practice management vendor (organization_representative) — full deep-dive available
240
+ 2. Jane Doe — mentioned in news coverage (incident_victim_or_private_citizen) — known-information summary only, no new research
241
+
242
+ Select entities (comma-separated numbers), or:
243
+ all-public → select all public_figure / organization_representative entities
244
+ skip → skip
245
+ Selection [skip]:
246
+ ```
247
+
248
+ ## Monitoring / re-running research
249
+
250
+ Re-run analysis with `--compare-to <prior_run_id>` to insert a **What Changed** section against a previous run’s snapshot:
251
+
252
+ ```bash
253
+ airc analyze \
254
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_YYYYMMDDTHHMMSSZ.json \
255
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/YYYYMMDDTHHMMSSZ \
256
+ --compare-to 20260712T173833Z
257
+ ```
258
+
259
+ ```text
260
+ What Changed section inserted (vs 20260712T173833Z)
261
+ ```
262
+
263
+ ## Configuration reference
264
+
265
+ ```bash
266
+ airc config show # provider, model, masked key status, paths
267
+ airc config set-key openai|gemini|anthropic|firecrawl
268
+ airc config reset # clear stored config + credentials
269
+ ```
270
+
271
+ **Data directory** (scopes, raw scrapes, reports, exports):
272
+
273
+ | Priority | Source |
274
+ |---|---|
275
+ | 1 | `airc --data-dir PATH …` |
276
+ | 2 | `AIRC_DATA_DIR` |
277
+ | 3 | OS user data dir (e.g. `~/.local/share/ai-researcher-crawler/`) |
278
+
279
+ **CI / non-interactive fallbacks** (no prompts):
280
+
281
+ ```bash
282
+ export FIRECRAWL_API_KEY=...
283
+ export LLM_PROVIDER=openai # or gemini | anthropic
284
+ export OPENAI_API_KEY=... # or GOOGLE_API_KEY / ANTHROPIC_API_KEY
285
+ export AIRC_DATA_DIR=/var/lib/airc
286
+ ```
287
+
288
+ CLI flags (`--firecrawl-api-key`, `--llm-provider`, `--openai-api-key`, …) override env for a single run.
289
+
290
+ ## Limitations
291
+
292
+ - Collection respects site access norms (including robots.txt / terms of service via the crawl provider), so some sites will not be scraped.
293
+ - LLM analysis can still make mistakes; the fact-checking design reduces silent invention, it does not eliminate all error.
294
+ - `deep` research depth costs more in Firecrawl and LLM usage than `quick`.
295
+ - Competitor discovery is heuristic — always review the proposed list before accepting it.
296
+ - The package is early (0.1.x); expect CLI and output shape to evolve.
297
+
298
+ ## Ethics / responsible use
299
+
300
+ This tool is meant for **public** information only. It does not bypass logins, paywalls, or CAPTCHAs. It includes safeguards so private individuals mentioned in coverage are not turned into dug-up personal profiles from aggregated public data. Use it for organizational research, public figures, and topics — not for building dossiers on private people.
301
+
302
+ For the full anti-hallucination design (citations, claim kinds, reviewer loop), see [DOCUMENTATION.md §11](DOCUMENTATION.md#11-anti-hallucination-design).
303
+
304
+ ## Contributing
305
+
306
+ Issues and PRs welcome. Architecture, pipeline internals, and module deep-dives live in [DOCUMENTATION.md](DOCUMENTATION.md) — start there if you want to change how the system works.
307
+
308
+ ## License
309
+
310
+ [MIT](LICENSE)
@@ -0,0 +1,270 @@
1
+ # AI Researcher Crawler
2
+
3
+ Research any topic, company, or competitor from your terminal — and get cited, fact-checked reports you can export to Word, Excel, or PowerPoint.
4
+
5
+ ## Quick demo
6
+
7
+ ```text
8
+ $ pip install research-guild-ai
9
+ $ airc init
10
+ Which LLM provider would you like to use?
11
+ 1. OpenAI
12
+ 2. Google Gemini
13
+ 3. Anthropic Claude
14
+ Choice [1]: 2
15
+ Enter your Google Gemini API key:
16
+ Enter your Firecrawl API key:
17
+ Saved.
18
+
19
+ Setup complete. Config stored at ~/.config/ai-researcher-crawler/
20
+
21
+ $ airc collect
22
+ Writing to: ~/.local/share/ai-researcher-crawler/
23
+
24
+ === AI Researcher Crawler — Scoping ===
25
+
26
+ What are you researching?
27
+
28
+ 1. Business / tech competitive intelligence (companies, products, markets)
29
+ 2. History (eras, past events, historical figures/topics)
30
+ 3. Current events / recent news
31
+ 4. General research (any other topic)
32
+
33
+ Choose a number [1]: 1
34
+ → Mode: competitive
35
+
36
+ Research subject (be specific, e.g. 'HR SaaS' or 'AI code assistants' — not just 'market') []: Law Firm Management Systems
37
+ Target geography (comma-separated countries/regions, or leave blank for global): global
38
+ Tech keywords (comma-separated, e.g. llm, saas, mobile): saas, legaltech, practice-management
39
+ Known competitors (comma-separated company names; leave blank or type 'none' to discover): Clio, Amberlo, LEAP, NetDocuments
40
+ Research objective (what decision or question should this answer?) []: Compare offerings, pricing signals, and geographic focus
41
+ Research depth (quick | deep) [quick]: quick
42
+ Output formats (comma-separated, e.g. markdown_report, word file, xlsx): markdown_report, docx
43
+
44
+ Scope saved: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
45
+
46
+ Starting collection (competitor pages + web + GitHub)...
47
+
48
+ === Collection Summary ===
49
+ Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
50
+ Research mode: competitive
51
+ Competitors: Clio, Amberlo, LEAP, NetDocuments
52
+ Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
53
+
54
+ Collected:
55
+ - Clio (homepage): https://www.clio.com/ → .../clio-homepage.md
56
+ - Amberlo (homepage): https://www.amberlo.io/ → .../amberlo-homepage.md
57
+ - LEAP (homepage): https://www.leaplegalsoftware.com/ → .../leap-homepage.md
58
+ - NetDocuments (homepage): https://www.netdocuments.com/ → .../netdocuments-homepage.md
59
+ - web (web-search): .../web-search.md
60
+ - github (github-search): .../github-search.md
61
+
62
+ $ airc analyze \
63
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
64
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
65
+
66
+ === Analysis Summary ===
67
+ Documents analyzed: 12
68
+ Confidence: medium
69
+ Research loops used: 0
70
+ Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
71
+
72
+ # From the report (excerpt):
73
+ # Generative and agentic AI are becoming core platform features
74
+ # (e.g. Clio Work, ndMAX) [clio/features, https://...; netdocuments/features, https://...]
75
+ #
76
+ # | AI Capabilities | Clio Work ... | *Not found in collected sources* | ...
77
+ ```
78
+
79
+ ## Install
80
+
81
+ Requires **Python 3.11+**.
82
+
83
+ ```bash
84
+ pip install research-guild-ai
85
+ ```
86
+
87
+ Until the package is on PyPI, install from GitHub or a local wheel:
88
+
89
+ ```bash
90
+ # From this repo (subdirectory install)
91
+ pip install "git+https://github.com/Manishkarki1/ai-researcher.git#subdirectory=ai-researcher-crawler"
92
+
93
+ # Or build a wheel locally, then:
94
+ # cd ai-researcher-crawler && python -m build
95
+ pip install dist/ai_researcher_crawler-*.whl
96
+ ```
97
+
98
+ Confirm the CLI is on your PATH:
99
+
100
+ ```bash
101
+ airc --help
102
+ ```
103
+
104
+ ## Setup
105
+
106
+ ```bash
107
+ airc init
108
+ ```
109
+
110
+ This asks which LLM provider to use (**OpenAI**, **Google Gemini**, or **Anthropic Claude**), prompts for that provider’s API key, then asks for a **Firecrawl** API key (used to collect public web sources). Keys are stored in your OS credential store when available — not in a plaintext project `.env`. If the OS keyring is unavailable, they fall back to a mode-`0600` file under the config directory.
111
+
112
+ Get keys here:
113
+
114
+ | Service | Signup |
115
+ |---|---|
116
+ | Firecrawl | https://www.firecrawl.dev/ |
117
+ | OpenAI | https://platform.openai.com/api-keys |
118
+ | Google Gemini | https://aistudio.google.com/apikey |
119
+ | Anthropic | https://console.anthropic.com/ |
120
+
121
+ You can also set keys later with `airc config set-key`, or via environment variables for CI (see [Configuration](#configuration-reference)).
122
+
123
+ ## Research modes
124
+
125
+ | Mode | Use it for |
126
+ |---|---|
127
+ | **Competitive** | Comparing companies, products, markets |
128
+ | **History** | Past events, eras, historical figures |
129
+ | **Current events** | Recent news, ongoing situations |
130
+ | **General** | Anything else |
131
+
132
+ ## Basic usage
133
+
134
+ ### Collect sources
135
+
136
+ ```bash
137
+ airc collect
138
+ ```
139
+
140
+ Interactive prompts pick a mode, save a scope file, optionally discover competitors (competitive mode), and scrape public pages into a timestamped raw folder.
141
+
142
+ ```text
143
+ === Collection Summary ===
144
+ Scope saved to: ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json
145
+ Research mode: competitive
146
+ Competitors: Clio, Amberlo, LEAP, NetDocuments
147
+ Raw data directory: ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
148
+ ```
149
+
150
+ Collect and analyze in one step:
151
+
152
+ ```bash
153
+ airc collect --analyze
154
+ ```
155
+
156
+ ### Analyze into a report
157
+
158
+ ```bash
159
+ airc analyze \
160
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_20260714T095055Z.json \
161
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/20260714T095311Z
162
+ ```
163
+
164
+ ```text
165
+ === Analysis Summary ===
166
+ Documents analyzed: 12
167
+ Confidence: medium
168
+ Research loops used: 0
169
+ Report saved to: ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md
170
+ ```
171
+
172
+ ### Export
173
+
174
+ ```bash
175
+ airc export \
176
+ --report ~/.local/share/ai-researcher-crawler/reports/report_20260714T095311Z.md \
177
+ --formats docx,xlsx,pptx
178
+ ```
179
+
180
+ ```text
181
+ === Exports ===
182
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.docx
183
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.xlsx
184
+ - ~/.local/share/ai-researcher-crawler/exports/report_20260714T095311Z.pptx
185
+ ```
186
+
187
+ ## What you get
188
+
189
+ Every claim is **cited** and **confidence-scored**. When something isn’t in the collected sources, the report says so (`not found in collected sources` / `not publicly disclosed`) instead of guessing. Single-source claims are flagged as unverified.
190
+
191
+ Exports: **Markdown**, **Word (docx)**, **Excel (xlsx)**, and **PowerPoint (pptx)**.
192
+
193
+ ## Entity deep-dive
194
+
195
+ After analysis, the tool lists people and organizations mentioned in the research and lets you follow up on selected ones. Public figures and organization representatives can get a fuller follow-up research pass. **Private individuals** mentioned in coverage get a limited, source-only summary — the tool will not go digging for more about a private person, and those names are not written into the cross-run entities index.
196
+
197
+ ```text
198
+ Detected entities:
199
+ 1. Clio — legal practice management vendor (organization_representative) — full deep-dive available
200
+ 2. Jane Doe — mentioned in news coverage (incident_victim_or_private_citizen) — known-information summary only, no new research
201
+
202
+ Select entities (comma-separated numbers), or:
203
+ all-public → select all public_figure / organization_representative entities
204
+ skip → skip
205
+ Selection [skip]:
206
+ ```
207
+
208
+ ## Monitoring / re-running research
209
+
210
+ Re-run analysis with `--compare-to <prior_run_id>` to insert a **What Changed** section against a previous run’s snapshot:
211
+
212
+ ```bash
213
+ airc analyze \
214
+ --scope-file ~/.local/share/ai-researcher-crawler/scope/scope_YYYYMMDDTHHMMSSZ.json \
215
+ --raw-dir ~/.local/share/ai-researcher-crawler/raw/YYYYMMDDTHHMMSSZ \
216
+ --compare-to 20260712T173833Z
217
+ ```
218
+
219
+ ```text
220
+ What Changed section inserted (vs 20260712T173833Z)
221
+ ```
222
+
223
+ ## Configuration reference
224
+
225
+ ```bash
226
+ airc config show # provider, model, masked key status, paths
227
+ airc config set-key openai|gemini|anthropic|firecrawl
228
+ airc config reset # clear stored config + credentials
229
+ ```
230
+
231
+ **Data directory** (scopes, raw scrapes, reports, exports):
232
+
233
+ | Priority | Source |
234
+ |---|---|
235
+ | 1 | `airc --data-dir PATH …` |
236
+ | 2 | `AIRC_DATA_DIR` |
237
+ | 3 | OS user data dir (e.g. `~/.local/share/ai-researcher-crawler/`) |
238
+
239
+ **CI / non-interactive fallbacks** (no prompts):
240
+
241
+ ```bash
242
+ export FIRECRAWL_API_KEY=...
243
+ export LLM_PROVIDER=openai # or gemini | anthropic
244
+ export OPENAI_API_KEY=... # or GOOGLE_API_KEY / ANTHROPIC_API_KEY
245
+ export AIRC_DATA_DIR=/var/lib/airc
246
+ ```
247
+
248
+ CLI flags (`--firecrawl-api-key`, `--llm-provider`, `--openai-api-key`, …) override env for a single run.
249
+
250
+ ## Limitations
251
+
252
+ - Collection respects site access norms (including robots.txt / terms of service via the crawl provider), so some sites will not be scraped.
253
+ - LLM analysis can still make mistakes; the fact-checking design reduces silent invention, it does not eliminate all error.
254
+ - `deep` research depth costs more in Firecrawl and LLM usage than `quick`.
255
+ - Competitor discovery is heuristic — always review the proposed list before accepting it.
256
+ - The package is early (0.1.x); expect CLI and output shape to evolve.
257
+
258
+ ## Ethics / responsible use
259
+
260
+ This tool is meant for **public** information only. It does not bypass logins, paywalls, or CAPTCHAs. It includes safeguards so private individuals mentioned in coverage are not turned into dug-up personal profiles from aggregated public data. Use it for organizational research, public figures, and topics — not for building dossiers on private people.
261
+
262
+ For the full anti-hallucination design (citations, claim kinds, reviewer loop), see [DOCUMENTATION.md §11](DOCUMENTATION.md#11-anti-hallucination-design).
263
+
264
+ ## Contributing
265
+
266
+ Issues and PRs welcome. Architecture, pipeline internals, and module deep-dives live in [DOCUMENTATION.md](DOCUMENTATION.md) — start there if you want to change how the system works.
267
+
268
+ ## License
269
+
270
+ [MIT](LICENSE)