polyglotminer 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyglotminer-1.0.1/LICENSE +21 -0
- polyglotminer-1.0.1/PKG-INFO +262 -0
- polyglotminer-1.0.1/README.md +230 -0
- polyglotminer-1.0.1/polyglotminer/__init__.py +3 -0
- polyglotminer-1.0.1/polyglotminer/cache.py +246 -0
- polyglotminer-1.0.1/polyglotminer/campaign.py +1079 -0
- polyglotminer-1.0.1/polyglotminer/cli.py +1231 -0
- polyglotminer-1.0.1/polyglotminer/cochange.py +648 -0
- polyglotminer-1.0.1/polyglotminer/completion.py +264 -0
- polyglotminer-1.0.1/polyglotminer/config.py +141 -0
- polyglotminer-1.0.1/polyglotminer/db.py +1372 -0
- polyglotminer-1.0.1/polyglotminer/doctor.py +281 -0
- polyglotminer-1.0.1/polyglotminer/evolution.py +1281 -0
- polyglotminer-1.0.1/polyglotminer/languages.py +749 -0
- polyglotminer-1.0.1/polyglotminer/media/logo.png +0 -0
- polyglotminer-1.0.1/polyglotminer/models.py +440 -0
- polyglotminer-1.0.1/polyglotminer/scanner.py +527 -0
- polyglotminer-1.0.1/polyglotminer/server.py +708 -0
- polyglotminer-1.0.1/polyglotminer/storage.py +318 -0
- polyglotminer-1.0.1/polyglotminer/templates/base.html +1232 -0
- polyglotminer-1.0.1/polyglotminer/templates/dashboard.html +1094 -0
- polyglotminer-1.0.1/polyglotminer/templates/discovery.html +923 -0
- polyglotminer-1.0.1/polyglotminer/templates/evolution_dashboard.html +3588 -0
- polyglotminer-1.0.1/polyglotminer/templates/evolution_report.html +2572 -0
- polyglotminer-1.0.1/polyglotminer/templates/report.html +1409 -0
- polyglotminer-1.0.1/polyglotminer/wizard.py +144 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/PKG-INFO +262 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/SOURCES.txt +43 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/dependency_links.txt +1 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/entry_points.txt +3 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/requires.txt +5 -0
- polyglotminer-1.0.1/polyglotminer.egg-info/top_level.txt +1 -0
- polyglotminer-1.0.1/setup.cfg +4 -0
- polyglotminer-1.0.1/setup.py +44 -0
- polyglotminer-1.0.1/tests/test_campaign.py +334 -0
- polyglotminer-1.0.1/tests/test_cli_smoke.py +249 -0
- polyglotminer-1.0.1/tests/test_db_authoritative.py +587 -0
- polyglotminer-1.0.1/tests/test_directed_campaign.py +186 -0
- polyglotminer-1.0.1/tests/test_evolution_audit_fixes.py +221 -0
- polyglotminer-1.0.1/tests/test_false_positive_migrations.py +136 -0
- polyglotminer-1.0.1/tests/test_header_migration_exclusion.py +208 -0
- polyglotminer-1.0.1/tests/test_identical_rename_exclusion.py +114 -0
- polyglotminer-1.0.1/tests/test_migration_heuristics_fixes.py +187 -0
- polyglotminer-1.0.1/tests/test_stratified_harvesting.py +190 -0
- polyglotminer-1.0.1/tests/test_token_fallback.py +119 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sari Yanouche
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: polyglotminer
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: Empirical research framework for Git repository analysis and cross-language migration mining
|
|
5
|
+
Home-page: https://github.com/SariYanouche/PolyglotMiner
|
|
6
|
+
Author: Sari Yanouche
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Source, https://github.com/SariYanouche/PolyglotMiner
|
|
9
|
+
Project-URL: Tracker, https://github.com/SariYanouche/PolyglotMiner/issues
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Topic :: Software Development :: Version Control :: Git
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: gitpython>=3.1.40
|
|
17
|
+
Requires-Dist: jinja2>=3.1.0
|
|
18
|
+
Requires-Dist: rich>=13.0.0
|
|
19
|
+
Requires-Dist: rich-argparse>=1.4.0
|
|
20
|
+
Requires-Dist: pygments>=2.15.0
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: description
|
|
24
|
+
Dynamic: description-content-type
|
|
25
|
+
Dynamic: home-page
|
|
26
|
+
Dynamic: license
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
Dynamic: project-url
|
|
29
|
+
Dynamic: requires-dist
|
|
30
|
+
Dynamic: requires-python
|
|
31
|
+
Dynamic: summary
|
|
32
|
+
|
|
33
|
+
<div align="center">
|
|
34
|
+
|
|
35
|
+
# PolyglotMiner
|
|
36
|
+
|
|
37
|
+
### Empirical Git Repository & Migration Intelligence Platform
|
|
38
|
+
|
|
39
|
+
[](https://pypi.org/project/polyglotminer/)
|
|
40
|
+
[](https://www.python.org/)
|
|
41
|
+
[](https://github.com/SariYanouche/PolyglotMiner/releases)
|
|
42
|
+
[](LICENSE)
|
|
43
|
+
[](https://www.sqlite.org/)
|
|
44
|
+
[](http://localhost:8080)
|
|
45
|
+
|
|
46
|
+
**PolyglotMiner** is a high-throughput research platform and CLI tool suite for analyzing longitudinal codebase evolution, multi-language composition dynamics, and cross-language software migrations across Git repositories at scale.
|
|
47
|
+
|
|
48
|
+
[Installation & Quick Start](#installation--quick-start) •
|
|
49
|
+
[Overview & Methodology](#overview--methodology) •
|
|
50
|
+
[CLI Command Reference](#cli-command-reference) •
|
|
51
|
+
[Web Studio & API](#web-analytics-studio) •
|
|
52
|
+
[Architecture](#database-architecture) •
|
|
53
|
+
[Citation](#citation)
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Installation & Quick Start
|
|
60
|
+
|
|
61
|
+
### 1. Install via pip (Recommended)
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install polyglotminer
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Both `polyglotminer` and its short alias `pgminer` are immediately available in your terminal.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
# Verify environment health
|
|
71
|
+
polyglotminer doctor
|
|
72
|
+
|
|
73
|
+
# Launch the interactive terminal wizard (TUI)
|
|
74
|
+
polyglotminer
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
<details>
|
|
78
|
+
<summary><b>Or install from source / Git (for development)</b></summary>
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
git clone https://github.com/SariYanouche/PolyglotMiner.git
|
|
82
|
+
cd PolyglotMiner
|
|
83
|
+
python -m venv .venv
|
|
84
|
+
source .venv/bin/activate # On Windows: .\.venv\Scripts\Activate.ps1
|
|
85
|
+
pip install -e .
|
|
86
|
+
```
|
|
87
|
+
</details>
|
|
88
|
+
|
|
89
|
+
### 2. Configure Autocompletion & Auth (Optional)
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
# Enable tab autocompletion for your shell (PowerShell, Bash, Zsh, Fish)
|
|
93
|
+
polyglotminer completion --install
|
|
94
|
+
|
|
95
|
+
# (Optional) High-volume candidate harvesting without GitHub rate-limits (60 -> 5,000 req/hr)
|
|
96
|
+
gh auth login # or: export GITHUB_TOKEN="ghp_..."
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Overview & Methodology
|
|
102
|
+
|
|
103
|
+
PolyglotMiner traverses repository histories without checking out physical file trees or downloading file blobs, achieving mining throughputs exceeding **70+ repositories per minute**.
|
|
104
|
+
|
|
105
|
+
* **$O(1)$ Memory Blobless Tree Diffing**: Traverses commit trees via native `git ls-tree -r` streaming with zero blob network calls.
|
|
106
|
+
* **Native Subprocess Streaming**: Direct `git rev-list` and `git log --name-status` pipelines eliminate commit-object memory allocation overhead.
|
|
107
|
+
* **Dual Execution Modes**: Fast **Empirical Module Mode** (default, file-turnover analysis) or **Physical LOC Mode** (`--full-loc`, batched `git cat-file --batch` streaming).
|
|
108
|
+
* **Noise & False-Positive Elimination**: Excludes C/C++/Objective-C header pairings (`.h` vs `.c`/`.cpp`/`.m`), identical renames/moves, DevOps/CI glue scripts (`.sh`, `.bat`, Dockerfiles), and vendor directories (`node_modules/`, `Pods/`, `vendor/`).
|
|
109
|
+
|
|
110
|
+
### Empirical Migration Archetypes
|
|
111
|
+
|
|
112
|
+
| Archetype | Defining Empirical Criteria |
|
|
113
|
+
| :--- | :--- |
|
|
114
|
+
| **Direct File Replacement** | Verified 1-to-1 file conversion (e.g. `Parser.java` $\to$ `Parser.kt`) in co-changes |
|
|
115
|
+
| **Incremental Migration** | Macro inverse displacement with Pearson correlation $r \le -0.25$ and source contraction $\ge 25\%$ |
|
|
116
|
+
| **Heterogeneous Interop** | Long-term concurrent multi-language co-existence ($r \approx 0$) |
|
|
117
|
+
| **Polyglot Co-Growth** | Concurrent expansion of both languages across milestones ($r > 0$) |
|
|
118
|
+
|
|
119
|
+
$$\text{Milestone Pearson Correlation: } r = \frac{\sum (s_i - \bar{s})(t_i - \bar{t})}{\sqrt{\sum (s_i - \bar{s})^2 \sum (t_i - \bar{t})^2}} \le -0.25$$
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## CLI Command Reference
|
|
124
|
+
|
|
125
|
+
Both `polyglotminer` and its short alias `pgminer` are registered executable commands.
|
|
126
|
+
|
|
127
|
+
| Command | Aliases | Description | Quick Example |
|
|
128
|
+
| :--- | :--- | :--- | :--- |
|
|
129
|
+
| **`scan`** | `s` | Point-in-time language distribution & LOC metrics | `polyglotminer scan . --open` |
|
|
130
|
+
| **`history`** | `h`, `evol` | Longitudinal equidistant milestone trajectories | `polyglotminer history repo -k 10 --open` |
|
|
131
|
+
| **`compare`** | `c`, `diff` | Pairwise 2-commit delta & churn inspector | `polyglotminer compare . HEAD~10 HEAD` |
|
|
132
|
+
| **`mine`** | `cochange` | Mined multi-language commits & file replacements | `polyglotminer mine repo --min-loc 10` |
|
|
133
|
+
| **`campaign`** | `harvest` | Multiprocess batch discovery & mining pipeline | `polyglotminer campaign -p "C->C++,JS->TS"` |
|
|
134
|
+
| **`ui`** | `serve` | Launch interactive Web Analytics Studio | `polyglotminer ui --port 8080` |
|
|
135
|
+
| **`saved`** | `bookmarks`| List and export curated benchmark corpus | `polyglotminer saved --export-csv corpus.csv` |
|
|
136
|
+
| **`bookmark`** | `star` | Tag or annotate a repository in SQLite | `polyglotminer bookmark repo -c "Java->Kotlin"` |
|
|
137
|
+
| **`rebuild`** | `b` | Instant HTML dashboard re-compilation | `polyglotminer rebuild --open` |
|
|
138
|
+
| **`list`** | `ls` | List stored snapshots, runs, and repositories | `polyglotminer list` |
|
|
139
|
+
| **`clean`** | `rm` | Delete reports, snapshots, or database records | `polyglotminer clean my-repo --yes` |
|
|
140
|
+
| **`doctor`** | `check` | System diagnostics & dependency health-check | `polyglotminer doctor` |
|
|
141
|
+
| **`completion`**| | Auto-detect shell and install tab-completion | `polyglotminer completion --install` |
|
|
142
|
+
|
|
143
|
+
### Key Command Examples
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
# 1. Point-in-time profiling (local directory or remote repository)
|
|
147
|
+
polyglotminer scan https://github.com/facebook/react --commit v18.2.0 --open
|
|
148
|
+
|
|
149
|
+
# 2. Longitudinal evolution trajectory across 10 milestone intervals
|
|
150
|
+
polyglotminer history handmux/handmux --samples 10 --open
|
|
151
|
+
|
|
152
|
+
# 3. Directed migration campaign (target specific transitions e.g. C -> C++ or JS -> TS)
|
|
153
|
+
polyglotminer campaign -p "C->C++,JS->TS" --workers 4 --limit 50 --open
|
|
154
|
+
|
|
155
|
+
# 4. Continuous overnight mining loop with stratified star sampling
|
|
156
|
+
polyglotminer campaign --min-stars 50 --max-stars 5000 --seed 42 --workers 2 --loop
|
|
157
|
+
|
|
158
|
+
# 5. Mine multi-language co-changes with minimum LOC delta threshold
|
|
159
|
+
polyglotminer mine carlphilipp/chicago-commutes --source-lang Java --target-lang Kotlin --export-csv pairs.csv
|
|
160
|
+
|
|
161
|
+
# 6. Pairwise 2-commit turnover comparison
|
|
162
|
+
polyglotminer compare vuejs/vue v2.6.0 v3.0.0 --open
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
<div align="center">
|
|
166
|
+
<img src=".github/assets/cli_history_demo.png" alt="PolyglotMiner CLI Terminal Evolution Output" width="95%">
|
|
167
|
+
</div>
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Web Analytics Studio
|
|
172
|
+
|
|
173
|
+
Launch the dedicated read-only analytics suite with `polyglotminer ui` (default: `http://localhost:8080`).
|
|
174
|
+
|
|
175
|
+
<div align="center">
|
|
176
|
+
<img src=".github/assets/web_studio_trajectory.png" alt="PolyglotMiner Interactive Analytics Studio" width="75%">
|
|
177
|
+
</div>
|
|
178
|
+
|
|
179
|
+
* **Graph Navigator & Spotlight Visualizations**: Instant navigation across transition ecosystems, turnover archetypes, star tier distributions, and commit depth brackets.
|
|
180
|
+
* **Live Heuristic Tuning**: Interactive threshold sliders (candidate pairs threshold, minimum confidence score, archetype filtering) with instant client-side recalculation.
|
|
181
|
+
* **Instant SSR & Skeleton Shimmers**: Virtualized tables and lightweight SQLite streaming (<10MB RAM) for lag-free exploration of large datasets.
|
|
182
|
+
* **Academic Export Suite**: 1-click export of publication-ready high-resolution figures (300 DPI PNG), LaTeX `booktabs` tables, and CSV datasets.
|
|
183
|
+
* **REST API**: Integrates `/api/summary`, `/api/corpus`, `/api/repositories`, `/api/repositories/saved`, `/api/cochanges`, and dynamic `/<owner>/<repo>/evolution_latest.html` routing.
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## Database Architecture
|
|
188
|
+
|
|
189
|
+
Analytical state is stored in an embedded SQLite database (`reports/polyglotminer.db`) with Write-Ahead Logging (`WAL`):
|
|
190
|
+
|
|
191
|
+
| Table | Primary Key | Description |
|
|
192
|
+
| :--- | :--- | :--- |
|
|
193
|
+
| **`repositories`** | `name_with_owner` | Discovered corpus metadata, stars, composition, bookmarks, and status |
|
|
194
|
+
| **`snapshots`** | `id` | Point-in-time scan metrics, file counts, code/comment/blank lines |
|
|
195
|
+
| **`evolution_runs`** | `run_id` | Longitudinal trajectories, milestone sample counts, and migration pairs |
|
|
196
|
+
| **`cochange_commits`** | `id` | Commits modifying $\ge 2$ languages with net directional churn deltas |
|
|
197
|
+
| **`migration_pairs`** | `id` | 1-to-1 file replacements, deleted/added snippets, and similarity confidence |
|
|
198
|
+
|
|
199
|
+
* **Process Recycling**: Campaign workers recycle processes every 25 repositories (`maxtasksperchild=25`) for 100% OS memory reclamation.
|
|
200
|
+
* **Watchdog & Failure Resilience**: 75-second per-repository watchdog timeout. Failed or timeout repositories are permanently tracked in SQLite so resume queues never stall.
|
|
201
|
+
|
|
202
|
+
---
|
|
203
|
+
|
|
204
|
+
## Verification & Tests
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
# Run full automated test suite
|
|
208
|
+
python -m unittest discover tests
|
|
209
|
+
|
|
210
|
+
# Run specific verification suites
|
|
211
|
+
python -m unittest tests/test_cli_smoke.py
|
|
212
|
+
python -m unittest tests/test_directed_campaign.py
|
|
213
|
+
python -m unittest tests/test_header_migration_exclusion.py
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Repository Structure
|
|
219
|
+
|
|
220
|
+
```text
|
|
221
|
+
PolyglotMiner/
|
|
222
|
+
├── polyglotminer/ # Core research package
|
|
223
|
+
│ ├── __init__.py # Version and package metadata (v1.0.1)
|
|
224
|
+
│ ├── cli.py # Unified CLI parser and router
|
|
225
|
+
│ ├── scanner.py # Single-commit line-counting and tokenizer
|
|
226
|
+
│ ├── evolution.py # Milestone sampler and trajectory engine
|
|
227
|
+
│ ├── cochange.py # Multi-language commit and snippet extractor
|
|
228
|
+
│ ├── campaign.py # Multiprocess mass-mining campaign engine
|
|
229
|
+
│ ├── cache.py # Git clone cache pool (.cache/repos/)
|
|
230
|
+
│ ├── languages.py # Language specifications and color maps
|
|
231
|
+
│ ├── config.py # Vendor and asset exclusion rules
|
|
232
|
+
│ ├── db.py # SQLite persistence layer (WAL mode)
|
|
233
|
+
│ ├── storage.py # Storage orchestration
|
|
234
|
+
│ ├── server.py # Embedded REST API and Web Studio server
|
|
235
|
+
│ ├── doctor.py # System diagnostics & dependency check
|
|
236
|
+
│ ├── completion.py # Shell autocompletion generator
|
|
237
|
+
│ ├── wizard.py # Interactive quick-start terminal wizard (TUI)
|
|
238
|
+
│ ├── models.py # Analytical data structures
|
|
239
|
+
│ ├── reporters/ # CLI and HTML formatters
|
|
240
|
+
│ └── templates/ # Web Studio Jinja2 templates
|
|
241
|
+
├── tests/ # Automated verification test suite
|
|
242
|
+
├── main.py # Universal entry point
|
|
243
|
+
├── requirements.txt # Package dependencies
|
|
244
|
+
├── setup.py # Setuptools build configuration
|
|
245
|
+
└── README.md # Documentation
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
250
|
+
## Citation
|
|
251
|
+
|
|
252
|
+
If you use **PolyglotMiner** in academic publications, please cite:
|
|
253
|
+
|
|
254
|
+
```bibtex
|
|
255
|
+
@software{polyglotminer2026,
|
|
256
|
+
author = {Sari, Yanouche},
|
|
257
|
+
title = {PolyglotMiner: An Empirical Research Platform for Longitudinal Software Evolution and Cross-Language Migration Mining},
|
|
258
|
+
year = {2026},
|
|
259
|
+
version = {1.0.1},
|
|
260
|
+
url = {https://github.com/SariYanouche/PolyglotMiner}
|
|
261
|
+
}
|
|
262
|
+
```
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# PolyglotMiner
|
|
4
|
+
|
|
5
|
+
### Empirical Git Repository & Migration Intelligence Platform
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/polyglotminer/)
|
|
8
|
+
[](https://www.python.org/)
|
|
9
|
+
[](https://github.com/SariYanouche/PolyglotMiner/releases)
|
|
10
|
+
[](LICENSE)
|
|
11
|
+
[](https://www.sqlite.org/)
|
|
12
|
+
[](http://localhost:8080)
|
|
13
|
+
|
|
14
|
+
**PolyglotMiner** is a high-throughput research platform and CLI tool suite for analyzing longitudinal codebase evolution, multi-language composition dynamics, and cross-language software migrations across Git repositories at scale.
|
|
15
|
+
|
|
16
|
+
[Installation & Quick Start](#installation--quick-start) •
|
|
17
|
+
[Overview & Methodology](#overview--methodology) •
|
|
18
|
+
[CLI Command Reference](#cli-command-reference) •
|
|
19
|
+
[Web Studio & API](#web-analytics-studio) •
|
|
20
|
+
[Architecture](#database-architecture) •
|
|
21
|
+
[Citation](#citation)
|
|
22
|
+
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## Installation & Quick Start
|
|
28
|
+
|
|
29
|
+
### 1. Install via pip (Recommended)
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install polyglotminer
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Both `polyglotminer` and its short alias `pgminer` are immediately available in your terminal.
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Verify environment health
|
|
39
|
+
polyglotminer doctor
|
|
40
|
+
|
|
41
|
+
# Launch the interactive terminal wizard (TUI)
|
|
42
|
+
polyglotminer
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
<details>
|
|
46
|
+
<summary><b>Or install from source / Git (for development)</b></summary>
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/SariYanouche/PolyglotMiner.git
|
|
50
|
+
cd PolyglotMiner
|
|
51
|
+
python -m venv .venv
|
|
52
|
+
source .venv/bin/activate # On Windows: .\.venv\Scripts\Activate.ps1
|
|
53
|
+
pip install -e .
|
|
54
|
+
```
|
|
55
|
+
</details>
|
|
56
|
+
|
|
57
|
+
### 2. Configure Autocompletion & Auth (Optional)
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# Enable tab autocompletion for your shell (PowerShell, Bash, Zsh, Fish)
|
|
61
|
+
polyglotminer completion --install
|
|
62
|
+
|
|
63
|
+
# (Optional) High-volume candidate harvesting without GitHub rate-limits (60 -> 5,000 req/hr)
|
|
64
|
+
gh auth login # or: export GITHUB_TOKEN="ghp_..."
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Overview & Methodology
|
|
70
|
+
|
|
71
|
+
PolyglotMiner traverses repository histories without checking out physical file trees or downloading file blobs, achieving mining throughputs exceeding **70+ repositories per minute**.
|
|
72
|
+
|
|
73
|
+
* **$O(1)$ Memory Blobless Tree Diffing**: Traverses commit trees via native `git ls-tree -r` streaming with zero blob network calls.
|
|
74
|
+
* **Native Subprocess Streaming**: Direct `git rev-list` and `git log --name-status` pipelines eliminate commit-object memory allocation overhead.
|
|
75
|
+
* **Dual Execution Modes**: Fast **Empirical Module Mode** (default, file-turnover analysis) or **Physical LOC Mode** (`--full-loc`, batched `git cat-file --batch` streaming).
|
|
76
|
+
* **Noise & False-Positive Elimination**: Excludes C/C++/Objective-C header pairings (`.h` vs `.c`/`.cpp`/`.m`), identical renames/moves, DevOps/CI glue scripts (`.sh`, `.bat`, Dockerfiles), and vendor directories (`node_modules/`, `Pods/`, `vendor/`).
|
|
77
|
+
|
|
78
|
+
### Empirical Migration Archetypes
|
|
79
|
+
|
|
80
|
+
| Archetype | Defining Empirical Criteria |
|
|
81
|
+
| :--- | :--- |
|
|
82
|
+
| **Direct File Replacement** | Verified 1-to-1 file conversion (e.g. `Parser.java` $\to$ `Parser.kt`) in co-changes |
|
|
83
|
+
| **Incremental Migration** | Macro inverse displacement with Pearson correlation $r \le -0.25$ and source contraction $\ge 25\%$ |
|
|
84
|
+
| **Heterogeneous Interop** | Long-term concurrent multi-language co-existence ($r \approx 0$) |
|
|
85
|
+
| **Polyglot Co-Growth** | Concurrent expansion of both languages across milestones ($r > 0$) |
|
|
86
|
+
|
|
87
|
+
$$\text{Milestone Pearson Correlation: } r = \frac{\sum (s_i - \bar{s})(t_i - \bar{t})}{\sqrt{\sum (s_i - \bar{s})^2 \sum (t_i - \bar{t})^2}} \le -0.25$$
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## CLI Command Reference
|
|
92
|
+
|
|
93
|
+
Both `polyglotminer` and its short alias `pgminer` are registered executable commands.
|
|
94
|
+
|
|
95
|
+
| Command | Aliases | Description | Quick Example |
|
|
96
|
+
| :--- | :--- | :--- | :--- |
|
|
97
|
+
| **`scan`** | `s` | Point-in-time language distribution & LOC metrics | `polyglotminer scan . --open` |
|
|
98
|
+
| **`history`** | `h`, `evol` | Longitudinal equidistant milestone trajectories | `polyglotminer history repo -k 10 --open` |
|
|
99
|
+
| **`compare`** | `c`, `diff` | Pairwise 2-commit delta & churn inspector | `polyglotminer compare . HEAD~10 HEAD` |
|
|
100
|
+
| **`mine`** | `cochange` | Mined multi-language commits & file replacements | `polyglotminer mine repo --min-loc 10` |
|
|
101
|
+
| **`campaign`** | `harvest` | Multiprocess batch discovery & mining pipeline | `polyglotminer campaign -p "C->C++,JS->TS"` |
|
|
102
|
+
| **`ui`** | `serve` | Launch interactive Web Analytics Studio | `polyglotminer ui --port 8080` |
|
|
103
|
+
| **`saved`** | `bookmarks`| List and export curated benchmark corpus | `polyglotminer saved --export-csv corpus.csv` |
|
|
104
|
+
| **`bookmark`** | `star` | Tag or annotate a repository in SQLite | `polyglotminer bookmark repo -c "Java->Kotlin"` |
|
|
105
|
+
| **`rebuild`** | `b` | Instant HTML dashboard re-compilation | `polyglotminer rebuild --open` |
|
|
106
|
+
| **`list`** | `ls` | List stored snapshots, runs, and repositories | `polyglotminer list` |
|
|
107
|
+
| **`clean`** | `rm` | Delete reports, snapshots, or database records | `polyglotminer clean my-repo --yes` |
|
|
108
|
+
| **`doctor`** | `check` | System diagnostics & dependency health-check | `polyglotminer doctor` |
|
|
109
|
+
| **`completion`**| | Auto-detect shell and install tab-completion | `polyglotminer completion --install` |
|
|
110
|
+
|
|
111
|
+
### Key Command Examples
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# 1. Point-in-time profiling (local directory or remote repository)
|
|
115
|
+
polyglotminer scan https://github.com/facebook/react --commit v18.2.0 --open
|
|
116
|
+
|
|
117
|
+
# 2. Longitudinal evolution trajectory across 10 milestone intervals
|
|
118
|
+
polyglotminer history handmux/handmux --samples 10 --open
|
|
119
|
+
|
|
120
|
+
# 3. Directed migration campaign (target specific transitions e.g. C -> C++ or JS -> TS)
|
|
121
|
+
polyglotminer campaign -p "C->C++,JS->TS" --workers 4 --limit 50 --open
|
|
122
|
+
|
|
123
|
+
# 4. Continuous overnight mining loop with stratified star sampling
|
|
124
|
+
polyglotminer campaign --min-stars 50 --max-stars 5000 --seed 42 --workers 2 --loop
|
|
125
|
+
|
|
126
|
+
# 5. Mine multi-language co-changes with minimum LOC delta threshold
|
|
127
|
+
polyglotminer mine carlphilipp/chicago-commutes --source-lang Java --target-lang Kotlin --export-csv pairs.csv
|
|
128
|
+
|
|
129
|
+
# 6. Pairwise 2-commit turnover comparison
|
|
130
|
+
polyglotminer compare vuejs/vue v2.6.0 v3.0.0 --open
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
<div align="center">
|
|
134
|
+
<img src=".github/assets/cli_history_demo.png" alt="PolyglotMiner CLI Terminal Evolution Output" width="95%">
|
|
135
|
+
</div>
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Web Analytics Studio
|
|
140
|
+
|
|
141
|
+
Launch the dedicated read-only analytics suite with `polyglotminer ui` (default: `http://localhost:8080`).
|
|
142
|
+
|
|
143
|
+
<div align="center">
|
|
144
|
+
<img src=".github/assets/web_studio_trajectory.png" alt="PolyglotMiner Interactive Analytics Studio" width="75%">
|
|
145
|
+
</div>
|
|
146
|
+
|
|
147
|
+
* **Graph Navigator & Spotlight Visualizations**: Instant navigation across transition ecosystems, turnover archetypes, star tier distributions, and commit depth brackets.
|
|
148
|
+
* **Live Heuristic Tuning**: Interactive threshold sliders (candidate pairs threshold, minimum confidence score, archetype filtering) with instant client-side recalculation.
|
|
149
|
+
* **Instant SSR & Skeleton Shimmers**: Virtualized tables and lightweight SQLite streaming (<10MB RAM) for lag-free exploration of large datasets.
|
|
150
|
+
* **Academic Export Suite**: 1-click export of publication-ready high-resolution figures (300 DPI PNG), LaTeX `booktabs` tables, and CSV datasets.
|
|
151
|
+
* **REST API**: Integrates `/api/summary`, `/api/corpus`, `/api/repositories`, `/api/repositories/saved`, `/api/cochanges`, and dynamic `/<owner>/<repo>/evolution_latest.html` routing.
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## Database Architecture
|
|
156
|
+
|
|
157
|
+
Analytical state is stored in an embedded SQLite database (`reports/polyglotminer.db`) with Write-Ahead Logging (`WAL`):
|
|
158
|
+
|
|
159
|
+
| Table | Primary Key | Description |
|
|
160
|
+
| :--- | :--- | :--- |
|
|
161
|
+
| **`repositories`** | `name_with_owner` | Discovered corpus metadata, stars, composition, bookmarks, and status |
|
|
162
|
+
| **`snapshots`** | `id` | Point-in-time scan metrics, file counts, code/comment/blank lines |
|
|
163
|
+
| **`evolution_runs`** | `run_id` | Longitudinal trajectories, milestone sample counts, and migration pairs |
|
|
164
|
+
| **`cochange_commits`** | `id` | Commits modifying $\ge 2$ languages with net directional churn deltas |
|
|
165
|
+
| **`migration_pairs`** | `id` | 1-to-1 file replacements, deleted/added snippets, and similarity confidence |
|
|
166
|
+
|
|
167
|
+
* **Process Recycling**: Campaign workers recycle processes every 25 repositories (`maxtasksperchild=25`) for 100% OS memory reclamation.
|
|
168
|
+
* **Watchdog & Failure Resilience**: 75-second per-repository watchdog timeout. Failed or timeout repositories are permanently tracked in SQLite so resume queues never stall.
|
|
169
|
+
|
|
170
|
+
---
|
|
171
|
+
|
|
172
|
+
## Verification & Tests
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
# Run full automated test suite
|
|
176
|
+
python -m unittest discover tests
|
|
177
|
+
|
|
178
|
+
# Run specific verification suites
|
|
179
|
+
python -m unittest tests/test_cli_smoke.py
|
|
180
|
+
python -m unittest tests/test_directed_campaign.py
|
|
181
|
+
python -m unittest tests/test_header_migration_exclusion.py
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Repository Structure
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
PolyglotMiner/
|
|
190
|
+
├── polyglotminer/ # Core research package
|
|
191
|
+
│ ├── __init__.py # Version and package metadata (v1.0.1)
|
|
192
|
+
│ ├── cli.py # Unified CLI parser and router
|
|
193
|
+
│ ├── scanner.py # Single-commit line-counting and tokenizer
|
|
194
|
+
│ ├── evolution.py # Milestone sampler and trajectory engine
|
|
195
|
+
│ ├── cochange.py # Multi-language commit and snippet extractor
|
|
196
|
+
│ ├── campaign.py # Multiprocess mass-mining campaign engine
|
|
197
|
+
│ ├── cache.py # Git clone cache pool (.cache/repos/)
|
|
198
|
+
│ ├── languages.py # Language specifications and color maps
|
|
199
|
+
│ ├── config.py # Vendor and asset exclusion rules
|
|
200
|
+
│ ├── db.py # SQLite persistence layer (WAL mode)
|
|
201
|
+
│ ├── storage.py # Storage orchestration
|
|
202
|
+
│ ├── server.py # Embedded REST API and Web Studio server
|
|
203
|
+
│ ├── doctor.py # System diagnostics & dependency check
|
|
204
|
+
│ ├── completion.py # Shell autocompletion generator
|
|
205
|
+
│ ├── wizard.py # Interactive quick-start terminal wizard (TUI)
|
|
206
|
+
│ ├── models.py # Analytical data structures
|
|
207
|
+
│ ├── reporters/ # CLI and HTML formatters
|
|
208
|
+
│ └── templates/ # Web Studio Jinja2 templates
|
|
209
|
+
├── tests/ # Automated verification test suite
|
|
210
|
+
├── main.py # Universal entry point
|
|
211
|
+
├── requirements.txt # Package dependencies
|
|
212
|
+
├── setup.py # Setuptools build configuration
|
|
213
|
+
└── README.md # Documentation
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Citation
|
|
219
|
+
|
|
220
|
+
If you use **PolyglotMiner** in academic publications, please cite:
|
|
221
|
+
|
|
222
|
+
```bibtex
|
|
223
|
+
@software{polyglotminer2026,
|
|
224
|
+
author = {Sari, Yanouche},
|
|
225
|
+
title = {PolyglotMiner: An Empirical Research Platform for Longitudinal Software Evolution and Cross-Language Migration Mining},
|
|
226
|
+
year = {2026},
|
|
227
|
+
version = {1.0.1},
|
|
228
|
+
url = {https://github.com/SariYanouche/PolyglotMiner}
|
|
229
|
+
}
|
|
230
|
+
```
|