polyglotminer 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. polyglotminer-1.0.1/LICENSE +21 -0
  2. polyglotminer-1.0.1/PKG-INFO +262 -0
  3. polyglotminer-1.0.1/README.md +230 -0
  4. polyglotminer-1.0.1/polyglotminer/__init__.py +3 -0
  5. polyglotminer-1.0.1/polyglotminer/cache.py +246 -0
  6. polyglotminer-1.0.1/polyglotminer/campaign.py +1079 -0
  7. polyglotminer-1.0.1/polyglotminer/cli.py +1231 -0
  8. polyglotminer-1.0.1/polyglotminer/cochange.py +648 -0
  9. polyglotminer-1.0.1/polyglotminer/completion.py +264 -0
  10. polyglotminer-1.0.1/polyglotminer/config.py +141 -0
  11. polyglotminer-1.0.1/polyglotminer/db.py +1372 -0
  12. polyglotminer-1.0.1/polyglotminer/doctor.py +281 -0
  13. polyglotminer-1.0.1/polyglotminer/evolution.py +1281 -0
  14. polyglotminer-1.0.1/polyglotminer/languages.py +749 -0
  15. polyglotminer-1.0.1/polyglotminer/media/logo.png +0 -0
  16. polyglotminer-1.0.1/polyglotminer/models.py +440 -0
  17. polyglotminer-1.0.1/polyglotminer/scanner.py +527 -0
  18. polyglotminer-1.0.1/polyglotminer/server.py +708 -0
  19. polyglotminer-1.0.1/polyglotminer/storage.py +318 -0
  20. polyglotminer-1.0.1/polyglotminer/templates/base.html +1232 -0
  21. polyglotminer-1.0.1/polyglotminer/templates/dashboard.html +1094 -0
  22. polyglotminer-1.0.1/polyglotminer/templates/discovery.html +923 -0
  23. polyglotminer-1.0.1/polyglotminer/templates/evolution_dashboard.html +3588 -0
  24. polyglotminer-1.0.1/polyglotminer/templates/evolution_report.html +2572 -0
  25. polyglotminer-1.0.1/polyglotminer/templates/report.html +1409 -0
  26. polyglotminer-1.0.1/polyglotminer/wizard.py +144 -0
  27. polyglotminer-1.0.1/polyglotminer.egg-info/PKG-INFO +262 -0
  28. polyglotminer-1.0.1/polyglotminer.egg-info/SOURCES.txt +43 -0
  29. polyglotminer-1.0.1/polyglotminer.egg-info/dependency_links.txt +1 -0
  30. polyglotminer-1.0.1/polyglotminer.egg-info/entry_points.txt +3 -0
  31. polyglotminer-1.0.1/polyglotminer.egg-info/requires.txt +5 -0
  32. polyglotminer-1.0.1/polyglotminer.egg-info/top_level.txt +1 -0
  33. polyglotminer-1.0.1/setup.cfg +4 -0
  34. polyglotminer-1.0.1/setup.py +44 -0
  35. polyglotminer-1.0.1/tests/test_campaign.py +334 -0
  36. polyglotminer-1.0.1/tests/test_cli_smoke.py +249 -0
  37. polyglotminer-1.0.1/tests/test_db_authoritative.py +587 -0
  38. polyglotminer-1.0.1/tests/test_directed_campaign.py +186 -0
  39. polyglotminer-1.0.1/tests/test_evolution_audit_fixes.py +221 -0
  40. polyglotminer-1.0.1/tests/test_false_positive_migrations.py +136 -0
  41. polyglotminer-1.0.1/tests/test_header_migration_exclusion.py +208 -0
  42. polyglotminer-1.0.1/tests/test_identical_rename_exclusion.py +114 -0
  43. polyglotminer-1.0.1/tests/test_migration_heuristics_fixes.py +187 -0
  44. polyglotminer-1.0.1/tests/test_stratified_harvesting.py +190 -0
  45. polyglotminer-1.0.1/tests/test_token_fallback.py +119 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sari Yanouche
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,262 @@
1
+ Metadata-Version: 2.4
2
+ Name: polyglotminer
3
+ Version: 1.0.1
4
+ Summary: Empirical research framework for Git repository analysis and cross-language migration mining
5
+ Home-page: https://github.com/SariYanouche/PolyglotMiner
6
+ Author: Sari Yanouche
7
+ License: MIT
8
+ Project-URL: Source, https://github.com/SariYanouche/PolyglotMiner
9
+ Project-URL: Tracker, https://github.com/SariYanouche/PolyglotMiner/issues
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Topic :: Software Development :: Version Control :: Git
13
+ Requires-Python: >=3.10
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENSE
16
+ Requires-Dist: gitpython>=3.1.40
17
+ Requires-Dist: jinja2>=3.1.0
18
+ Requires-Dist: rich>=13.0.0
19
+ Requires-Dist: rich-argparse>=1.4.0
20
+ Requires-Dist: pygments>=2.15.0
21
+ Dynamic: author
22
+ Dynamic: classifier
23
+ Dynamic: description
24
+ Dynamic: description-content-type
25
+ Dynamic: home-page
26
+ Dynamic: license
27
+ Dynamic: license-file
28
+ Dynamic: project-url
29
+ Dynamic: requires-dist
30
+ Dynamic: requires-python
31
+ Dynamic: summary
32
+
33
+ <div align="center">
34
+
35
+ # PolyglotMiner
36
+
37
+ ### Empirical Git Repository & Migration Intelligence Platform
38
+
39
+ [![PyPI](https://img.shields.io/pypi/v/polyglotminer?color=4F46E5&label=PyPI)](https://pypi.org/project/polyglotminer/)
40
+ [![Python Version](https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/)
41
+ [![Version](https://img.shields.io/badge/Release-v1.0.1-4F46E5?style=flat)](https://github.com/SariYanouche/PolyglotMiner/releases)
42
+ [![License: MIT](https://img.shields.io/badge/License-MIT-059669.svg)](LICENSE)
43
+ [![Storage: SQLite](https://img.shields.io/badge/Storage-SQLite%203%20%28WAL%29-003B57?style=flat&logo=sqlite&logoColor=white)](https://www.sqlite.org/)
44
+ [![UI: Web Studio](https://img.shields.io/badge/Interface-Interactive%20Studio-0284C7?style=flat)](http://localhost:8080)
45
+
46
+ **PolyglotMiner** is a high-throughput research platform and CLI tool suite for analyzing longitudinal codebase evolution, multi-language composition dynamics, and cross-language software migrations across Git repositories at scale.
47
+
48
+ [Installation & Quick Start](#installation--quick-start) •
49
+ [Overview & Methodology](#overview--methodology) •
50
+ [CLI Command Reference](#cli-command-reference) •
51
+ [Web Studio & API](#web-analytics-studio) •
52
+ [Architecture](#database-architecture) •
53
+ [Citation](#citation)
54
+
55
+ </div>
56
+
57
+ ---
58
+
59
+ ## Installation & Quick Start
60
+
61
+ ### 1. Install via pip (Recommended)
62
+
63
+ ```bash
64
+ pip install polyglotminer
65
+ ```
66
+
67
+ Both `polyglotminer` and its short alias `pgminer` are immediately available in your terminal.
68
+
69
+ ```bash
70
+ # Verify environment health
71
+ polyglotminer doctor
72
+
73
+ # Launch the interactive terminal wizard (TUI)
74
+ polyglotminer
75
+ ```
76
+
77
+ <details>
78
+ <summary><b>Or install from source / Git (for development)</b></summary>
79
+
80
+ ```bash
81
+ git clone https://github.com/SariYanouche/PolyglotMiner.git
82
+ cd PolyglotMiner
83
+ python -m venv .venv
84
+ source .venv/bin/activate # On Windows: .\.venv\Scripts\Activate.ps1
85
+ pip install -e .
86
+ ```
87
+ </details>
88
+
89
+ ### 2. Configure Autocompletion & Auth (Optional)
90
+
91
+ ```bash
92
+ # Enable tab autocompletion for your shell (PowerShell, Bash, Zsh, Fish)
93
+ polyglotminer completion --install
94
+
95
+ # (Optional) High-volume candidate harvesting without GitHub rate-limits (60 -> 5,000 req/hr)
96
+ gh auth login # or: export GITHUB_TOKEN="ghp_..."
97
+ ```
98
+
99
+ ---
100
+
101
+ ## Overview & Methodology
102
+
103
+ PolyglotMiner traverses repository histories without checking out physical file trees or downloading file blobs, achieving mining throughputs exceeding **70+ repositories per minute**.
104
+
105
+ * **$O(1)$ Memory Blobless Tree Diffing**: Traverses commit trees via native `git ls-tree -r` streaming with zero blob network calls.
106
+ * **Native Subprocess Streaming**: Direct `git rev-list` and `git log --name-status` pipelines eliminate commit-object memory allocation overhead.
107
+ * **Dual Execution Modes**: Fast **Empirical Module Mode** (default, file-turnover analysis) or **Physical LOC Mode** (`--full-loc`, batched `git cat-file --batch` streaming).
108
+ * **Noise & False-Positive Elimination**: Excludes C/C++/Objective-C header pairings (`.h` vs `.c`/`.cpp`/`.m`), identical renames/moves, DevOps/CI glue scripts (`.sh`, `.bat`, Dockerfiles), and vendor directories (`node_modules/`, `Pods/`, `vendor/`).
109
+
110
+ ### Empirical Migration Archetypes
111
+
112
+ | Archetype | Defining Empirical Criteria |
113
+ | :--- | :--- |
114
+ | **Direct File Replacement** | Verified 1-to-1 file conversion (e.g. `Parser.java` $\to$ `Parser.kt`) in co-changes |
115
+ | **Incremental Migration** | Macro inverse displacement with Pearson correlation $r \le -0.25$ and source contraction $\ge 25\%$ |
116
+ | **Heterogeneous Interop** | Long-term concurrent multi-language co-existence ($r \approx 0$) |
117
+ | **Polyglot Co-Growth** | Concurrent expansion of both languages across milestones ($r > 0$) |
118
+
119
+ $$\text{Milestone Pearson Correlation: } r = \frac{\sum (s_i - \bar{s})(t_i - \bar{t})}{\sqrt{\sum (s_i - \bar{s})^2 \sum (t_i - \bar{t})^2}} \le -0.25$$
120
+
121
+ ---
122
+
123
+ ## CLI Command Reference
124
+
125
+ Both `polyglotminer` and its short alias `pgminer` are registered executable commands.
126
+
127
+ | Command | Aliases | Description | Quick Example |
128
+ | :--- | :--- | :--- | :--- |
129
+ | **`scan`** | `s` | Point-in-time language distribution & LOC metrics | `polyglotminer scan . --open` |
130
+ | **`history`** | `h`, `evol` | Longitudinal equidistant milestone trajectories | `polyglotminer history repo -k 10 --open` |
131
+ | **`compare`** | `c`, `diff` | Pairwise 2-commit delta & churn inspector | `polyglotminer compare . HEAD~10 HEAD` |
132
+ | **`mine`** | `cochange` | Mined multi-language commits & file replacements | `polyglotminer mine repo --min-loc 10` |
133
+ | **`campaign`** | `harvest` | Multiprocess batch discovery & mining pipeline | `polyglotminer campaign -p "C->C++,JS->TS"` |
134
+ | **`ui`** | `serve` | Launch interactive Web Analytics Studio | `polyglotminer ui --port 8080` |
135
+ | **`saved`** | `bookmarks`| List and export curated benchmark corpus | `polyglotminer saved --export-csv corpus.csv` |
136
+ | **`bookmark`** | `star` | Tag or annotate a repository in SQLite | `polyglotminer bookmark repo -c "Java->Kotlin"` |
137
+ | **`rebuild`** | `b` | Instant HTML dashboard re-compilation | `polyglotminer rebuild --open` |
138
+ | **`list`** | `ls` | List stored snapshots, runs, and repositories | `polyglotminer list` |
139
+ | **`clean`** | `rm` | Delete reports, snapshots, or database records | `polyglotminer clean my-repo --yes` |
140
+ | **`doctor`** | `check` | System diagnostics & dependency health-check | `polyglotminer doctor` |
141
+ | **`completion`**| | Auto-detect shell and install tab-completion | `polyglotminer completion --install` |
142
+
143
+ ### Key Command Examples
144
+
145
+ ```bash
146
+ # 1. Point-in-time profiling (local directory or remote repository)
147
+ polyglotminer scan https://github.com/facebook/react --commit v18.2.0 --open
148
+
149
+ # 2. Longitudinal evolution trajectory across 10 milestone intervals
150
+ polyglotminer history handmux/handmux --samples 10 --open
151
+
152
+ # 3. Directed migration campaign (target specific transitions e.g. C -> C++ or JS -> TS)
153
+ polyglotminer campaign -p "C->C++,JS->TS" --workers 4 --limit 50 --open
154
+
155
+ # 4. Continuous overnight mining loop with stratified star sampling
156
+ polyglotminer campaign --min-stars 50 --max-stars 5000 --seed 42 --workers 2 --loop
157
+
158
+ # 5. Mine multi-language co-changes with minimum LOC delta threshold
159
+ polyglotminer mine carlphilipp/chicago-commutes --source-lang Java --target-lang Kotlin --export-csv pairs.csv
160
+
161
+ # 6. Pairwise 2-commit turnover comparison
162
+ polyglotminer compare vuejs/vue v2.6.0 v3.0.0 --open
163
+ ```
164
+
165
+ <div align="center">
166
+ <img src=".github/assets/cli_history_demo.png" alt="PolyglotMiner CLI Terminal Evolution Output" width="95%">
167
+ </div>
168
+
169
+ ---
170
+
171
+ ## Web Analytics Studio
172
+
173
+ Launch the dedicated read-only analytics suite with `polyglotminer ui` (default: `http://localhost:8080`).
174
+
175
+ <div align="center">
176
+ <img src=".github/assets/web_studio_trajectory.png" alt="PolyglotMiner Interactive Analytics Studio" width="75%">
177
+ </div>
178
+
179
+ * **Graph Navigator & Spotlight Visualizations**: Instant navigation across transition ecosystems, turnover archetypes, star tier distributions, and commit depth brackets.
180
+ * **Live Heuristic Tuning**: Interactive threshold sliders (candidate pairs threshold, minimum confidence score, archetype filtering) with instant client-side recalculation.
181
+ * **Instant SSR & Skeleton Shimmers**: Virtualized tables and lightweight SQLite streaming (<10MB RAM) for lag-free exploration of large datasets.
182
+ * **Academic Export Suite**: 1-click export of publication-ready high-resolution figures (300 DPI PNG), LaTeX `booktabs` tables, and CSV datasets.
183
+ * **REST API**: Integrates `/api/summary`, `/api/corpus`, `/api/repositories`, `/api/repositories/saved`, `/api/cochanges`, and dynamic `/<owner>/<repo>/evolution_latest.html` routing.
184
+
185
+ ---
186
+
187
+ ## Database Architecture
188
+
189
+ Analytical state is stored in an embedded SQLite database (`reports/polyglotminer.db`) with Write-Ahead Logging (`WAL`):
190
+
191
+ | Table | Primary Key | Description |
192
+ | :--- | :--- | :--- |
193
+ | **`repositories`** | `name_with_owner` | Discovered corpus metadata, stars, composition, bookmarks, and status |
194
+ | **`snapshots`** | `id` | Point-in-time scan metrics, file counts, code/comment/blank lines |
195
+ | **`evolution_runs`** | `run_id` | Longitudinal trajectories, milestone sample counts, and migration pairs |
196
+ | **`cochange_commits`** | `id` | Commits modifying $\ge 2$ languages with net directional churn deltas |
197
+ | **`migration_pairs`** | `id` | 1-to-1 file replacements, deleted/added snippets, and similarity confidence |
198
+
199
+ * **Process Recycling**: Campaign workers recycle processes every 25 repositories (`maxtasksperchild=25`) for 100% OS memory reclamation.
200
+ * **Watchdog & Failure Resilience**: 75-second per-repository watchdog timeout. Failed or timeout repositories are permanently tracked in SQLite so resume queues never stall.
201
+
202
+ ---
203
+
204
+ ## Verification & Tests
205
+
206
+ ```bash
207
+ # Run full automated test suite
208
+ python -m unittest discover tests
209
+
210
+ # Run specific verification suites
211
+ python -m unittest tests/test_cli_smoke.py
212
+ python -m unittest tests/test_directed_campaign.py
213
+ python -m unittest tests/test_header_migration_exclusion.py
214
+ ```
215
+
216
+ ---
217
+
218
+ ## Repository Structure
219
+
220
+ ```text
221
+ PolyglotMiner/
222
+ ├── polyglotminer/ # Core research package
223
+ │ ├── __init__.py # Version and package metadata (v1.0.1)
224
+ │ ├── cli.py # Unified CLI parser and router
225
+ │ ├── scanner.py # Single-commit line-counting and tokenizer
226
+ │ ├── evolution.py # Milestone sampler and trajectory engine
227
+ │ ├── cochange.py # Multi-language commit and snippet extractor
228
+ │ ├── campaign.py # Multiprocess mass-mining campaign engine
229
+ │ ├── cache.py # Git clone cache pool (.cache/repos/)
230
+ │ ├── languages.py # Language specifications and color maps
231
+ │ ├── config.py # Vendor and asset exclusion rules
232
+ │ ├── db.py # SQLite persistence layer (WAL mode)
233
+ │ ├── storage.py # Storage orchestration
234
+ │ ├── server.py # Embedded REST API and Web Studio server
235
+ │ ├── doctor.py # System diagnostics & dependency check
236
+ │ ├── completion.py # Shell autocompletion generator
237
+ │ ├── wizard.py # Interactive quick-start terminal wizard (TUI)
238
+ │ ├── models.py # Analytical data structures
239
+ │ ├── reporters/ # CLI and HTML formatters
240
+ │ └── templates/ # Web Studio Jinja2 templates
241
+ ├── tests/ # Automated verification test suite
242
+ ├── main.py # Universal entry point
243
+ ├── requirements.txt # Package dependencies
244
+ ├── setup.py # Setuptools build configuration
245
+ └── README.md # Documentation
246
+ ```
247
+
248
+ ---
249
+
250
+ ## Citation
251
+
252
+ If you use **PolyglotMiner** in academic publications, please cite:
253
+
254
+ ```bibtex
255
+ @software{polyglotminer2026,
256
+ author = {Sari, Yanouche},
257
+ title = {PolyglotMiner: An Empirical Research Platform for Longitudinal Software Evolution and Cross-Language Migration Mining},
258
+ year = {2026},
259
+ version = {1.0.1},
260
+ url = {https://github.com/SariYanouche/PolyglotMiner}
261
+ }
262
+ ```
@@ -0,0 +1,230 @@
1
+ <div align="center">
2
+
3
+ # PolyglotMiner
4
+
5
+ ### Empirical Git Repository & Migration Intelligence Platform
6
+
7
+ [![PyPI](https://img.shields.io/pypi/v/polyglotminer?color=4F46E5&label=PyPI)](https://pypi.org/project/polyglotminer/)
8
+ [![Python Version](https://img.shields.io/badge/Python-3.10%2B-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/)
9
+ [![Version](https://img.shields.io/badge/Release-v1.0.1-4F46E5?style=flat)](https://github.com/SariYanouche/PolyglotMiner/releases)
10
+ [![License: MIT](https://img.shields.io/badge/License-MIT-059669.svg)](LICENSE)
11
+ [![Storage: SQLite](https://img.shields.io/badge/Storage-SQLite%203%20%28WAL%29-003B57?style=flat&logo=sqlite&logoColor=white)](https://www.sqlite.org/)
12
+ [![UI: Web Studio](https://img.shields.io/badge/Interface-Interactive%20Studio-0284C7?style=flat)](http://localhost:8080)
13
+
14
+ **PolyglotMiner** is a high-throughput research platform and CLI tool suite for analyzing longitudinal codebase evolution, multi-language composition dynamics, and cross-language software migrations across Git repositories at scale.
15
+
16
+ [Installation & Quick Start](#installation--quick-start) •
17
+ [Overview & Methodology](#overview--methodology) •
18
+ [CLI Command Reference](#cli-command-reference) •
19
+ [Web Studio & API](#web-analytics-studio) •
20
+ [Architecture](#database-architecture) •
21
+ [Citation](#citation)
22
+
23
+ </div>
24
+
25
+ ---
26
+
27
+ ## Installation & Quick Start
28
+
29
+ ### 1. Install via pip (Recommended)
30
+
31
+ ```bash
32
+ pip install polyglotminer
33
+ ```
34
+
35
+ Both `polyglotminer` and its short alias `pgminer` are immediately available in your terminal.
36
+
37
+ ```bash
38
+ # Verify environment health
39
+ polyglotminer doctor
40
+
41
+ # Launch the interactive terminal wizard (TUI)
42
+ polyglotminer
43
+ ```
44
+
45
+ <details>
46
+ <summary><b>Or install from source / Git (for development)</b></summary>
47
+
48
+ ```bash
49
+ git clone https://github.com/SariYanouche/PolyglotMiner.git
50
+ cd PolyglotMiner
51
+ python -m venv .venv
52
+ source .venv/bin/activate # On Windows: .\.venv\Scripts\Activate.ps1
53
+ pip install -e .
54
+ ```
55
+ </details>
56
+
57
+ ### 2. Configure Autocompletion & Auth (Optional)
58
+
59
+ ```bash
60
+ # Enable tab autocompletion for your shell (PowerShell, Bash, Zsh, Fish)
61
+ polyglotminer completion --install
62
+
63
+ # (Optional) High-volume candidate harvesting without GitHub rate-limits (60 -> 5,000 req/hr)
64
+ gh auth login # or: export GITHUB_TOKEN="ghp_..."
65
+ ```
66
+
67
+ ---
68
+
69
+ ## Overview & Methodology
70
+
71
+ PolyglotMiner traverses repository histories without checking out physical file trees or downloading file blobs, achieving mining throughputs exceeding **70+ repositories per minute**.
72
+
73
+ * **$O(1)$ Memory Blobless Tree Diffing**: Traverses commit trees via native `git ls-tree -r` streaming with zero blob network calls.
74
+ * **Native Subprocess Streaming**: Direct `git rev-list` and `git log --name-status` pipelines eliminate commit-object memory allocation overhead.
75
+ * **Dual Execution Modes**: Fast **Empirical Module Mode** (default, file-turnover analysis) or **Physical LOC Mode** (`--full-loc`, batched `git cat-file --batch` streaming).
76
+ * **Noise & False-Positive Elimination**: Excludes C/C++/Objective-C header pairings (`.h` vs `.c`/`.cpp`/`.m`), identical renames/moves, DevOps/CI glue scripts (`.sh`, `.bat`, Dockerfiles), and vendor directories (`node_modules/`, `Pods/`, `vendor/`).
77
+
78
+ ### Empirical Migration Archetypes
79
+
80
+ | Archetype | Defining Empirical Criteria |
81
+ | :--- | :--- |
82
+ | **Direct File Replacement** | Verified 1-to-1 file conversion (e.g. `Parser.java` $\to$ `Parser.kt`) in co-changes |
83
+ | **Incremental Migration** | Macro inverse displacement with Pearson correlation $r \le -0.25$ and source contraction $\ge 25\%$ |
84
+ | **Heterogeneous Interop** | Long-term concurrent multi-language co-existence ($r \approx 0$) |
85
+ | **Polyglot Co-Growth** | Concurrent expansion of both languages across milestones ($r > 0$) |
86
+
87
+ $$\text{Milestone Pearson Correlation: } r = \frac{\sum (s_i - \bar{s})(t_i - \bar{t})}{\sqrt{\sum (s_i - \bar{s})^2 \sum (t_i - \bar{t})^2}} \le -0.25$$
88
+
89
+ ---
90
+
91
+ ## CLI Command Reference
92
+
93
+ Both `polyglotminer` and its short alias `pgminer` are registered executable commands.
94
+
95
+ | Command | Aliases | Description | Quick Example |
96
+ | :--- | :--- | :--- | :--- |
97
+ | **`scan`** | `s` | Point-in-time language distribution & LOC metrics | `polyglotminer scan . --open` |
98
+ | **`history`** | `h`, `evol` | Longitudinal equidistant milestone trajectories | `polyglotminer history repo -k 10 --open` |
99
+ | **`compare`** | `c`, `diff` | Pairwise 2-commit delta & churn inspector | `polyglotminer compare . HEAD~10 HEAD` |
100
+ | **`mine`** | `cochange` | Mined multi-language commits & file replacements | `polyglotminer mine repo --min-loc 10` |
101
+ | **`campaign`** | `harvest` | Multiprocess batch discovery & mining pipeline | `polyglotminer campaign -p "C->C++,JS->TS"` |
102
+ | **`ui`** | `serve` | Launch interactive Web Analytics Studio | `polyglotminer ui --port 8080` |
103
+ | **`saved`** | `bookmarks`| List and export curated benchmark corpus | `polyglotminer saved --export-csv corpus.csv` |
104
+ | **`bookmark`** | `star` | Tag or annotate a repository in SQLite | `polyglotminer bookmark repo -c "Java->Kotlin"` |
105
+ | **`rebuild`** | `b` | Instant HTML dashboard re-compilation | `polyglotminer rebuild --open` |
106
+ | **`list`** | `ls` | List stored snapshots, runs, and repositories | `polyglotminer list` |
107
+ | **`clean`** | `rm` | Delete reports, snapshots, or database records | `polyglotminer clean my-repo --yes` |
108
+ | **`doctor`** | `check` | System diagnostics & dependency health-check | `polyglotminer doctor` |
109
+ | **`completion`**| | Auto-detect shell and install tab-completion | `polyglotminer completion --install` |
110
+
111
+ ### Key Command Examples
112
+
113
+ ```bash
114
+ # 1. Point-in-time profiling (local directory or remote repository)
115
+ polyglotminer scan https://github.com/facebook/react --commit v18.2.0 --open
116
+
117
+ # 2. Longitudinal evolution trajectory across 10 milestone intervals
118
+ polyglotminer history handmux/handmux --samples 10 --open
119
+
120
+ # 3. Directed migration campaign (target specific transitions e.g. C -> C++ or JS -> TS)
121
+ polyglotminer campaign -p "C->C++,JS->TS" --workers 4 --limit 50 --open
122
+
123
+ # 4. Continuous overnight mining loop with stratified star sampling
124
+ polyglotminer campaign --min-stars 50 --max-stars 5000 --seed 42 --workers 2 --loop
125
+
126
+ # 5. Mine multi-language co-changes with minimum LOC delta threshold
127
+ polyglotminer mine carlphilipp/chicago-commutes --source-lang Java --target-lang Kotlin --export-csv pairs.csv
128
+
129
+ # 6. Pairwise 2-commit turnover comparison
130
+ polyglotminer compare vuejs/vue v2.6.0 v3.0.0 --open
131
+ ```
132
+
133
+ <div align="center">
134
+ <img src=".github/assets/cli_history_demo.png" alt="PolyglotMiner CLI Terminal Evolution Output" width="95%">
135
+ </div>
136
+
137
+ ---
138
+
139
+ ## Web Analytics Studio
140
+
141
+ Launch the dedicated read-only analytics suite with `polyglotminer ui` (default: `http://localhost:8080`).
142
+
143
+ <div align="center">
144
+ <img src=".github/assets/web_studio_trajectory.png" alt="PolyglotMiner Interactive Analytics Studio" width="75%">
145
+ </div>
146
+
147
+ * **Graph Navigator & Spotlight Visualizations**: Instant navigation across transition ecosystems, turnover archetypes, star tier distributions, and commit depth brackets.
148
+ * **Live Heuristic Tuning**: Interactive threshold sliders (candidate pairs threshold, minimum confidence score, archetype filtering) with instant client-side recalculation.
149
+ * **Instant SSR & Skeleton Shimmers**: Virtualized tables and lightweight SQLite streaming (<10MB RAM) for lag-free exploration of large datasets.
150
+ * **Academic Export Suite**: 1-click export of publication-ready high-resolution figures (300 DPI PNG), LaTeX `booktabs` tables, and CSV datasets.
151
+ * **REST API**: Integrates `/api/summary`, `/api/corpus`, `/api/repositories`, `/api/repositories/saved`, `/api/cochanges`, and dynamic `/<owner>/<repo>/evolution_latest.html` routing.
152
+
153
+ ---
154
+
155
+ ## Database Architecture
156
+
157
+ Analytical state is stored in an embedded SQLite database (`reports/polyglotminer.db`) with Write-Ahead Logging (`WAL`):
158
+
159
+ | Table | Primary Key | Description |
160
+ | :--- | :--- | :--- |
161
+ | **`repositories`** | `name_with_owner` | Discovered corpus metadata, stars, composition, bookmarks, and status |
162
+ | **`snapshots`** | `id` | Point-in-time scan metrics, file counts, code/comment/blank lines |
163
+ | **`evolution_runs`** | `run_id` | Longitudinal trajectories, milestone sample counts, and migration pairs |
164
+ | **`cochange_commits`** | `id` | Commits modifying $\ge 2$ languages with net directional churn deltas |
165
+ | **`migration_pairs`** | `id` | 1-to-1 file replacements, deleted/added snippets, and similarity confidence |
166
+
167
+ * **Process Recycling**: Campaign workers recycle processes every 25 repositories (`maxtasksperchild=25`) for 100% OS memory reclamation.
168
+ * **Watchdog & Failure Resilience**: 75-second per-repository watchdog timeout. Failed or timeout repositories are permanently tracked in SQLite so resume queues never stall.
169
+
170
+ ---
171
+
172
+ ## Verification & Tests
173
+
174
+ ```bash
175
+ # Run full automated test suite
176
+ python -m unittest discover tests
177
+
178
+ # Run specific verification suites
179
+ python -m unittest tests/test_cli_smoke.py
180
+ python -m unittest tests/test_directed_campaign.py
181
+ python -m unittest tests/test_header_migration_exclusion.py
182
+ ```
183
+
184
+ ---
185
+
186
+ ## Repository Structure
187
+
188
+ ```text
189
+ PolyglotMiner/
190
+ ├── polyglotminer/ # Core research package
191
+ │ ├── __init__.py # Version and package metadata (v1.0.1)
192
+ │ ├── cli.py # Unified CLI parser and router
193
+ │ ├── scanner.py # Single-commit line-counting and tokenizer
194
+ │ ├── evolution.py # Milestone sampler and trajectory engine
195
+ │ ├── cochange.py # Multi-language commit and snippet extractor
196
+ │ ├── campaign.py # Multiprocess mass-mining campaign engine
197
+ │ ├── cache.py # Git clone cache pool (.cache/repos/)
198
+ │ ├── languages.py # Language specifications and color maps
199
+ │ ├── config.py # Vendor and asset exclusion rules
200
+ │ ├── db.py # SQLite persistence layer (WAL mode)
201
+ │ ├── storage.py # Storage orchestration
202
+ │ ├── server.py # Embedded REST API and Web Studio server
203
+ │ ├── doctor.py # System diagnostics & dependency check
204
+ │ ├── completion.py # Shell autocompletion generator
205
+ │ ├── wizard.py # Interactive quick-start terminal wizard (TUI)
206
+ │ ├── models.py # Analytical data structures
207
+ │ ├── reporters/ # CLI and HTML formatters
208
+ │ └── templates/ # Web Studio Jinja2 templates
209
+ ├── tests/ # Automated verification test suite
210
+ ├── main.py # Universal entry point
211
+ ├── requirements.txt # Package dependencies
212
+ ├── setup.py # Setuptools build configuration
213
+ └── README.md # Documentation
214
+ ```
215
+
216
+ ---
217
+
218
+ ## Citation
219
+
220
+ If you use **PolyglotMiner** in academic publications, please cite:
221
+
222
+ ```bibtex
223
+ @software{polyglotminer2026,
224
+ author = {Sari, Yanouche},
225
+ title = {PolyglotMiner: An Empirical Research Platform for Longitudinal Software Evolution and Cross-Language Migration Mining},
226
+ year = {2026},
227
+ version = {1.0.1},
228
+ url = {https://github.com/SariYanouche/PolyglotMiner}
229
+ }
230
+ ```
@@ -0,0 +1,3 @@
1
+ """PolyglotMiner - Git Repository Analysis & Cross-Language Migration Research Tool."""
2
+
3
+ __version__ = "1.0.1"