crawly-ai 2.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawly.py +25 -0
- crawly_ai-2.2.1.dist-info/METADATA +293 -0
- crawly_ai-2.2.1.dist-info/RECORD +14 -0
- crawly_ai-2.2.1.dist-info/WHEEL +5 -0
- crawly_ai-2.2.1.dist-info/entry_points.txt +3 -0
- crawly_ai-2.2.1.dist-info/licenses/LICENSE +674 -0
- crawly_ai-2.2.1.dist-info/top_level.txt +3 -0
- crawly_core/__init__.py +6 -0
- crawly_core/ai_engine.py +1124 -0
- crawly_core/cli.py +1274 -0
- crawly_core/downloader.py +550 -0
- crawly_core/models.py +410 -0
- crawly_core/sources.py +1001 -0
- downloader.py +70 -0
crawly.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# coding: utf-8
|
|
3
|
+
"""crawly: the literature assistant — Unified Academic Paper Discovery, Multi-Source PDF Downloader & AI Literature Review.
|
|
4
|
+
|
|
5
|
+
Repository: https://github.com/lokeshsk/crawly
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
Interactive Terminal UI (TUI):
|
|
9
|
+
python crawly.py
|
|
10
|
+
|
|
11
|
+
Direct CLI Subcommands:
|
|
12
|
+
python crawly.py search "synthetic aperture radar" -o sar_papers.csv --limit 10
|
|
13
|
+
python crawly.py download sar_papers.csv
|
|
14
|
+
python crawly.py ai sar_papers.csv --mode 1
|
|
15
|
+
python crawly.py ai sar_papers.csv --mode 2
|
|
16
|
+
python crawly.py ai sar_papers.csv --mode 3 -q "What are the main compressed sensing methods for SAR?"
|
|
17
|
+
python crawly.py pipeline "deep learning SAR despeckling" -o sar_dl --limit 8 --mode all
|
|
18
|
+
python crawly.py config --provider ollama --model llama3.2
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import sys
|
|
22
|
+
from crawly_core.cli import main
|
|
23
|
+
|
|
24
|
+
if __name__ == "__main__":
|
|
25
|
+
sys.exit(main())
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crawly-ai
|
|
3
|
+
Version: 2.2.1
|
|
4
|
+
Summary: crawly: the literature assistant — Multi-source academic paper discovery, PDF downloader & local/cloud AI literature review engine
|
|
5
|
+
Author-email: Lokesh Kumar <crawly.theliteratureassistant@gmail.com>
|
|
6
|
+
License-Expression: GPL-3.0-or-later
|
|
7
|
+
Project-URL: Homepage, https://lokeshsk.github.io/crawly/
|
|
8
|
+
Project-URL: Documentation, https://lokeshsk.github.io/crawly/
|
|
9
|
+
Project-URL: Repository, https://github.com/lokeshsk/crawly
|
|
10
|
+
Project-URL: Issues, https://github.com/lokeshsk/crawly/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/lokeshsk/crawly/releases
|
|
12
|
+
Keywords: academic,research,literature-review,papers,pdf-downloader,open-access,arxiv,openalex,semantic-scholar,europe-pmc,crossref,zotero,llm,ollama,lm-studio
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Operating System :: OS Independent
|
|
23
|
+
Classifier: Environment :: Console
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: requests>=2.31.0
|
|
28
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
29
|
+
Requires-Dist: typing-extensions>=4.0.0
|
|
30
|
+
Requires-Dist: lxml>=4.9.0
|
|
31
|
+
Requires-Dist: rich>=13.7.0
|
|
32
|
+
Requires-Dist: pypdf>=4.0.0
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# crawly: the literature assistant
|
|
36
|
+
|
|
37
|
+
[](https://github.com/lokeshsk/crawly)
|
|
38
|
+
[](https://github.com/lokeshsk/crawly/actions/workflows/ci-tests-coverage.yml)
|
|
39
|
+
[](tests/)
|
|
40
|
+
[](https://lokeshsk.github.io/crawly/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
**crawly** (**crawly: the literature assistant**) is an end-to-end academic research assistant for discovering scientific papers across 8+ official Open Access databases, downloading full-text PDFs within a **strict Legal Open-Access framework by default**, exporting BibTeX citations, and generating AI-powered literature reviews using either **local self-hosted LLMs** (Ollama, LM Studio, vLLM, llama.cpp) or **commercial/OpenAI-compatible APIs** (OpenAI, Gemini, Anthropic, OpenRouter, Groq, DeepSeek).
|
|
44
|
+
|
|
45
|
+
- **Repository:** [https://github.com/lokeshsk/crawly](https://github.com/lokeshsk/crawly)
|
|
46
|
+
- **Documentation & Command Builder Website:** [https://lokeshsk.github.io/crawly/](https://lokeshsk.github.io/crawly/)
|
|
47
|
+
- **Maintainer:** Lokesh Kumar (`crawly.theliteratureassistant@gmail.com`)
|
|
48
|
+
|
|
49
|
+
> [!CAUTION]
|
|
50
|
+
> ### Legal Warning & Non-Liability Disclaimer
|
|
51
|
+
> 1. **Not Recommended by Developers:** The developers, authors, and contributors of **crawly** (`crawly: the literature assistant`) **do NOT recommend, endorse, or encourage** the use of Sci-Hub, shadow libraries, or automated scraping of websites that prohibit automated access or bypass copyright/paywall protections.
|
|
52
|
+
> 2. **Strict Legal Open-Access Default:** By default, **crawly** operates strictly within a legal framework using official Open Access APIs and repositories (**OpenAlex**, **Unpaywall**, **Semantic Scholar**, **arXiv**, **Europe PMC / PubMed Central**, **Crossref**, **DOAJ**, **HAL Open Archive**, and **OpenAIRE**).
|
|
53
|
+
> 3. **Use at Your Own Sole Risk & Zero Developer Liability:** Any feature that goes beyond official Open Access APIs (such as third-party mirror fallbacks or HTML scraping) is disabled by default (`"use_scihub_fallback": false` in [`config.json`](config.json)) and guarded by an explicit risk-acceptance prompt. If a user chooses to enable or continue with such external websites or tools, **they do so strictly at their own sole risk and legal responsibility**. The developers and contributors **shall NOT be held liable or responsible** for any copyright claims, Terms of Service violations, ISP blocks, legal consequences, or any other harm or damages arising from the user's actions.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## 1. Quick Start for Users (No `pip install` or Dependencies Required!)
|
|
58
|
+
|
|
59
|
+
### ⚡ One-Line Instant Install (Zero Setup)
|
|
60
|
+
- **Linux / macOS / WSL:**
|
|
61
|
+
```bash
|
|
62
|
+
curl -fsSL https://lokeshsk.github.io/crawly/install.sh | bash
|
|
63
|
+
```
|
|
64
|
+
- **Windows (PowerShell):**
|
|
65
|
+
```powershell
|
|
66
|
+
irm https://lokeshsk.github.io/crawly/install.ps1 | iex
|
|
67
|
+
```
|
|
68
|
+
- **Python / PyPI (`pip`):**
|
|
69
|
+
```bash
|
|
70
|
+
pip install crawly-ai
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
### Manual Download & Repository Clone
|
|
76
|
+
|
|
77
|
+
### Option A: Universal Cross-Platform Bundle (Works on BOTH Windows & Linux / WSL)
|
|
78
|
+
Unzip [`dist/crawly-universal-windows-linux.zip`](dist/crawly-universal-windows-linux.zip) (or run directly from [`dist/`](dist/)):
|
|
79
|
+
- **On Windows (Command Prompt / PowerShell / Double-Click):**
|
|
80
|
+
```cmd
|
|
81
|
+
dist\crawly.bat
|
|
82
|
+
```
|
|
83
|
+
- **On Linux / WSL / macOS:**
|
|
84
|
+
```bash
|
|
85
|
+
./dist/crawly
|
|
86
|
+
```
|
|
87
|
+
- **Direct Single-File Archive (`crawly.pyz`):**
|
|
88
|
+
```bash
|
|
89
|
+
python dist/crawly.pyz
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Option B: Native Standalone Binary (Zero Python Setup Needed)
|
|
93
|
+
- **Linux / WSL:**
|
|
94
|
+
```bash
|
|
95
|
+
./dist/crawly-linux-x86_64
|
|
96
|
+
```
|
|
97
|
+
- **Windows:**
|
|
98
|
+
```powershell
|
|
99
|
+
.\dist\crawly-windows-x86_64.exe
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## 2. Setting Up & Customizing [`config.json`](config.json)
|
|
105
|
+
|
|
106
|
+
All configuration values—including AI providers, model names, base URLs, API keys, temperature, PDF parsing limits, default search databases, timeouts, user-agents, and API endpoints—are stored in [**`config.json`**](config.json) in the project root (or current working directory).
|
|
107
|
+
|
|
108
|
+
Nothing is hardcoded: if `config.json` is deleted, **crawly** automatically regenerates it with clean default values and built-in `_documentation` labels.
|
|
109
|
+
|
|
110
|
+
### Three Ways to Change Settings in `config.json`
|
|
111
|
+
|
|
112
|
+
1. **Edit [`config.json`](config.json) directly** in any text editor (VS Code, Notepad, nano, vim).
|
|
113
|
+
2. **Interactive Terminal UI:** Launch `./dist/crawly` and choose **Option `5` (Configure AI Models & Settings)**.
|
|
114
|
+
3. **CLI Command:**
|
|
115
|
+
```bash
|
|
116
|
+
# View current config.json status
|
|
117
|
+
./dist/crawly config --show
|
|
118
|
+
|
|
119
|
+
# Update values in config.json via CLI flags
|
|
120
|
+
./dist/crawly config --provider ollama --model llama3.2 --temperature 0.2
|
|
121
|
+
./dist/crawly config --provider lmstudio --base-url http://localhost:1234/v1 --model local-model
|
|
122
|
+
./dist/crawly config --provider openai_compatible --base-url https://api.openai.com/v1 --api-key sk-... --model gpt-4o-mini
|
|
123
|
+
./dist/crawly config --provider gemini --api-key YOUR_GEMINI_KEY --model gemini-2.5-flash
|
|
124
|
+
./dist/crawly config --provider anthropic --api-key YOUR_ANTHROPIC_KEY --model claude-3-5-sonnet-latest
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
### Complete `config.json` Field Reference
|
|
130
|
+
|
|
131
|
+
| Variable Name | Default Value | What to Put / Allowed Values |
|
|
132
|
+
| :--- | :--- | :--- |
|
|
133
|
+
| **`ai_provider`** | `"ollama"` | Which AI engine to use:<br>• `"ollama"` — Local self-hosted Ollama server<br>• `"lmstudio"` — Local self-hosted LM Studio server<br>• `"openai_compatible"` — Any OpenAI-compatible server (OpenAI, OpenRouter, Groq, DeepSeek, Together, vLLM, llama.cpp)<br>• `"gemini"` — Google Gemini API<br>• `"anthropic"` — Anthropic Claude API<br>• `"extractive"` — Offline NLP & BM25 extractive engine (no LLM server or API key needed) |
|
|
134
|
+
| **`ai_model`** | `"llama3.2"` | Model identifier for your chosen provider:<br>• Ollama: `"llama3.2"`, `"qwen2.5:7b"`, `"mistral"`, `"deepseek-r1:8b"`<br>• LM Studio: `"local-model"` or the loaded GGUF model ID<br>• OpenAI / OpenRouter: `"gpt-4o-mini"`, `"deepseek-chat"`, `"meta-llama/llama-3.1-8b-instruct"`<br>• Gemini: `"gemini-2.5-flash"`, `"gemini-2.5-pro"`<br>• Anthropic: `"claude-3-5-sonnet-latest"` |
|
|
135
|
+
| **`ai_base_url`** | `"http://localhost:11434"` | Server base URL for local or OpenAI-compatible providers:<br>• Ollama: `"http://localhost:11434"`<br>• LM Studio: `"http://localhost:1234/v1"`<br>• vLLM / llama.cpp: `"http://localhost:8000/v1"`<br>• OpenAI: `"https://api.openai.com/v1"`<br>• OpenRouter: `"https://openrouter.ai/api/v1"`<br>• Groq: `"https://api.groq.com/openai/v1"` |
|
|
136
|
+
| **`ai_api_key`** | `""` | API key for commercial/cloud providers (`openai_compatible`, `gemini`, `anthropic`). Leave `""` for local Ollama or LM Studio. Can also be set via env vars (`CRAWLY_AI_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`). |
|
|
137
|
+
| **`temperature`** | `0.2` | Float between `0.0` and `1.0`. Lower values (`0.1`–`0.3`) produce more factual, deterministic literature summaries. |
|
|
138
|
+
| **`max_pdf_pages`** | `25` | Maximum number of PDF pages to extract per paper with `pypdf` for AI summarization and RAG indexing. |
|
|
139
|
+
| **`max_text_chars`** | `14000` | Maximum characters of extracted full-text PDF content passed to the LLM per paper. |
|
|
140
|
+
| **`default_sources`** | `["openalex", "semantic_scholar", "arxiv", "europe_pmc", "crossref", "doaj", "hal", "openaire"]` | List of open academic databases searched by default when `--sources` is not specified. |
|
|
141
|
+
| **`default_limit_per_source`** | `10` | Default maximum number of papers to fetch from each database per search query. |
|
|
142
|
+
| **`default_sort`** | `"relevance"` | Default sorting order for search results: `"relevance"`, `"desc"` (newest year first), `"asc"` (oldest year first), or `"citations"` (most cited first). |
|
|
143
|
+
| **`request_timeout`** | `12` | HTTP timeout in seconds for database queries and metadata resolution. |
|
|
144
|
+
| **`unpaywall_email`** | `"crawly.theliteratureassistant@gmail.com"` | Polite contact email sent to the Unpaywall Legal Open-Access API when resolving OA PDF links by DOI. |
|
|
145
|
+
| **`semantic_scholar_api_key`** | `""` | Optional Semantic Scholar API key for higher rate limits (leave `""` for public tier). |
|
|
146
|
+
| **`polite_user_agent`** | `"crawly/2.0 (crawly: the literature assistant; https://github.com/lokeshsk/crawly; mailto:crawly.theliteratureassistant@gmail.com)"` | User-Agent header used for polite academic API requests (OpenAlex, Crossref, Unpaywall, Europe PMC). |
|
|
147
|
+
| **`browser_user_agent`** | `"Mozilla/5.0 ..."` | Browser User-Agent header used when downloading PDFs from publisher repositories. |
|
|
148
|
+
| **`use_scihub_fallback`** | `false` | Keep `false` for strict Legal Open-Access compliance. Even if set to `true`, **crawly** displays the Legal Warning & Non-Liability Disclaimer requiring explicit risk acceptance. |
|
|
149
|
+
| **`scihub_cdn_url`** | `"https://sci.bban.top/pdf/{doi}.pdf"` | Template URL (`{doi}` placeholder) used only if the user explicitly accepts legal risk for third-party fallback. |
|
|
150
|
+
| **`scihub_mirrors`** | `["https://sci-hub.ee", ...]` | List of mirror URLs tried only if the user explicitly accepts legal risk for third-party fallback. |
|
|
151
|
+
| **`api_endpoints`** | `{...}` | Dictionary of base URLs for OpenAlex, Semantic Scholar, arXiv, Europe PMC, Crossref, DOAJ, HAL, OpenAIRE, Unpaywall, Gemini, and Anthropic. |
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
### Example `config.json` Setups for Popular AI Providers
|
|
156
|
+
|
|
157
|
+
#### A. Local Ollama (Default — Free & Private)
|
|
158
|
+
1. Install Ollama (`https://ollama.com`) and pull a model: `ollama pull llama3.2`
|
|
159
|
+
2. Set in `config.json`:
|
|
160
|
+
```json
|
|
161
|
+
{
|
|
162
|
+
"ai_provider": "ollama",
|
|
163
|
+
"ai_model": "llama3.2",
|
|
164
|
+
"ai_base_url": "http://localhost:11434",
|
|
165
|
+
"ai_api_key": "",
|
|
166
|
+
"temperature": 0.2
|
|
167
|
+
}
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
#### B. Local LM Studio (Free & Private GUI Server)
|
|
171
|
+
1. Open LM Studio, load any GGUF model, and start the Local Server on port `1234`.
|
|
172
|
+
2. Set in `config.json`:
|
|
173
|
+
```json
|
|
174
|
+
{
|
|
175
|
+
"ai_provider": "lmstudio",
|
|
176
|
+
"ai_model": "local-model",
|
|
177
|
+
"ai_base_url": "http://localhost:1234/v1",
|
|
178
|
+
"ai_api_key": "",
|
|
179
|
+
"temperature": 0.2
|
|
180
|
+
}
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
#### C. OpenAI / OpenRouter / Groq / DeepSeek (`openai_compatible`)
|
|
184
|
+
```json
|
|
185
|
+
{
|
|
186
|
+
"ai_provider": "openai_compatible",
|
|
187
|
+
"ai_model": "gpt-4o-mini",
|
|
188
|
+
"ai_base_url": "https://api.openai.com/v1",
|
|
189
|
+
"ai_api_key": "sk-proj-YOUR_API_KEY",
|
|
190
|
+
"temperature": 0.2
|
|
191
|
+
}
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
#### D. Google Gemini API (`gemini`)
|
|
195
|
+
```json
|
|
196
|
+
{
|
|
197
|
+
"ai_provider": "gemini",
|
|
198
|
+
"ai_model": "gemini-2.5-flash",
|
|
199
|
+
"ai_api_key": "AIzaSyYOUR_GEMINI_API_KEY",
|
|
200
|
+
"temperature": 0.2
|
|
201
|
+
}
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
#### E. Offline Extractive Engine (`extractive` — No LLM or Internet Needed)
|
|
205
|
+
```json
|
|
206
|
+
{
|
|
207
|
+
"ai_provider": "extractive",
|
|
208
|
+
"ai_model": "extractive-bm25"
|
|
209
|
+
}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
## 3. CLI Subcommands & Usage
|
|
215
|
+
|
|
216
|
+
All commands display modern multi-column progress bars (spinner, progress bar, percentage, `M/N` counter, elapsed time, and ETA) and work identically whether you invoke `./dist/crawly`, `dist\crawly.bat`, `./dist/crawly-linux-x86_64`, or `python crawly.py`:
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
# 1. Launch Interactive Terminal UI Dashboard
|
|
220
|
+
./dist/crawly
|
|
221
|
+
|
|
222
|
+
# 2. Search across all legal open databases (saves CSV + BibTeX)
|
|
223
|
+
./dist/crawly search "synthetic aperture radar" -o sar_papers.csv --limit 10
|
|
224
|
+
|
|
225
|
+
# 3. Citation Graph Snowballing (trace forward citing & backward referenced works)
|
|
226
|
+
./dist/crawly snowball "10.1109/TGRS.2023.123456" -l 10
|
|
227
|
+
./dist/crawly snowball "synthetic aperture radar imaging" -o snowball_radar.csv
|
|
228
|
+
|
|
229
|
+
# 4. Download PDFs using strictly Legal Open Access repositories (Default)
|
|
230
|
+
./dist/crawly download sar_papers.csv
|
|
231
|
+
|
|
232
|
+
# 5. Run AI Literature Review Modes
|
|
233
|
+
# Mode 1: Per-Paper Structured Summaries (Objective, Methodology, Key Findings, Limitations)
|
|
234
|
+
./dist/crawly ai sar_papers.csv --mode 1
|
|
235
|
+
# Mode 2: Synthesized Cross-Paper Literature Review & Markdown Comparison Matrix
|
|
236
|
+
./dist/crawly ai sar_papers.csv --mode 2
|
|
237
|
+
# Mode 3: Citation-Grounded RAG Q&A over Downloaded PDFs
|
|
238
|
+
./dist/crawly ai sar_papers.csv --mode 3 -q "What are the main compressed sensing methods for SAR?"
|
|
239
|
+
# Mode gaps: Automated Research Gaps & Open Challenges Roadmap
|
|
240
|
+
./dist/crawly ai sar_papers.csv --mode gaps
|
|
241
|
+
|
|
242
|
+
# 6. Generate an Interactive Single-File HTML Review Dashboard
|
|
243
|
+
./dist/crawly report sar_papers.csv
|
|
244
|
+
|
|
245
|
+
# 7. Sync Papers & Full-Text PDFs to Local Zotero Desktop
|
|
246
|
+
./dist/crawly zotero sar_papers.csv -c "SAR_Survey"
|
|
247
|
+
|
|
248
|
+
# 8. Literature Watchdog (Scan latest preprints and generate alerts digest)
|
|
249
|
+
./dist/crawly watch "SAR despeckling" -o watchdog_radar.md
|
|
250
|
+
|
|
251
|
+
# 9. Run the complete end-to-end pipeline (Search -> Download Legal OA PDFs -> AI Review)
|
|
252
|
+
./dist/crawly pipeline "deep learning SAR despeckling" -o sar_dl --limit 8 --mode all
|
|
253
|
+
|
|
254
|
+
# 10. Export any CSV dataset to BibTeX (.bib) for Zotero / Mendeley / Overleaf
|
|
255
|
+
./dist/crawly export-bibtex sar_papers.csv
|
|
256
|
+
|
|
257
|
+
# 11. Run System & API Health Doctor
|
|
258
|
+
./dist/crawly doctor
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
## 4. For Contributors & Developers
|
|
264
|
+
|
|
265
|
+
Every Pull Request submitted to **crawly: the literature assistant** is verified by the automated GitHub Actions quality gate ([`.github/workflows/ci-tests-coverage.yml`](.github/workflows/ci-tests-coverage.yml)).
|
|
266
|
+
|
|
267
|
+
> [!IMPORTANT]
|
|
268
|
+
> **Mandatory Pull Request Policy:** All unit tests in [`tests/`](tests/) must pass and total code coverage across `crawly.py`, `downloader.py`, and `crawly_core/*` must be **100%** (`fail_under = 100` in [`.coveragerc`](.coveragerc)). If any test case fails or code coverage drops below 100%, the CI workflow will block the PR and automatically comment on the PR asking the contributor to fix the failing tests or add missing test coverage before it comes to the maintainer for review and merging.
|
|
269
|
+
|
|
270
|
+
```bash
|
|
271
|
+
# 1. Install developer, test, and build dependencies
|
|
272
|
+
pip install -r requirements.txt coverage pytest pytest-cov pyinstaller
|
|
273
|
+
|
|
274
|
+
# 2. Run the full test suite and enforce 100% code coverage
|
|
275
|
+
python -m coverage run -m unittest discover -s tests -v
|
|
276
|
+
python -m coverage report -m --fail-under=100
|
|
277
|
+
|
|
278
|
+
# 3. Run from source
|
|
279
|
+
python crawly.py
|
|
280
|
+
python downloader.py
|
|
281
|
+
|
|
282
|
+
# 4. Rebuild dist/crawly.pyz, launchers, universal zip, and native binary
|
|
283
|
+
python build_release.py --native
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
### Contact & Credits
|
|
287
|
+
- **Author & Maintainer:** Lokesh Kumar (`crawly.theliteratureassistant@gmail.com`)
|
|
288
|
+
- **Credits:** Original v1 Google Scholar script inspired by `https://jovian.ai/saini-9`.
|
|
289
|
+
|
|
290
|
+
### Support / Donation
|
|
291
|
+
If you like **crawly: the literature assistant**, you can buy me a cup of tea :)
|
|
292
|
+
|
|
293
|
+
[](https://paypal.me/lokeshsk755)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
crawly.py,sha256=a5LgIgbIJXApCzLRGNu3iomK4gh4xXKym32WGAMvxng,903
|
|
2
|
+
downloader.py,sha256=PE5ghyNk3lgCzaj2p1W3IE8YT6kgaMUdweD0wi72bn8,1896
|
|
3
|
+
crawly_ai-2.2.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
4
|
+
crawly_core/__init__.py,sha256=xbHDhlBGdGUvLYfmDlHh9pOALF0E4n-aFlPUcWFCNhY,279
|
|
5
|
+
crawly_core/ai_engine.py,sha256=5OdYUPI1FvSUDnTmU1wQmzDoGEUb8UJUYy12iiN7D6M,44853
|
|
6
|
+
crawly_core/cli.py,sha256=BnFVmGFcDGRX36Ap9tM5Fp-cSIHEVQBvojmJ5PyvFz4,61647
|
|
7
|
+
crawly_core/downloader.py,sha256=NDU0WaVqLyUFMY7nPr3PNCBhKvK3eL2OiNPdjZK5lhs,21001
|
|
8
|
+
crawly_core/models.py,sha256=ZNg-GMH98PL1Zqr3OKSmTf0gALQfLrVtISqJGaadQ_0,16481
|
|
9
|
+
crawly_core/sources.py,sha256=VIeyo-LdrxWTI-rgtFmbKSNUgmMH5YzyNHaxynijGN8,39683
|
|
10
|
+
crawly_ai-2.2.1.dist-info/METADATA,sha256=GDpJlYMVEyQIQEpNxmZmcIP4UGaJtsWAf2XfW8o-FhU,16281
|
|
11
|
+
crawly_ai-2.2.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
12
|
+
crawly_ai-2.2.1.dist-info/entry_points.txt,sha256=AULX2LbKf2VFiwsXbvphbiK446OajU29IhCSoGCEOKc,82
|
|
13
|
+
crawly_ai-2.2.1.dist-info/top_level.txt,sha256=1C2jCEXD3Ig925-DWhgmluB34yYpS5rMWIsm5R8NUus,30
|
|
14
|
+
crawly_ai-2.2.1.dist-info/RECORD,,
|