borges 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. borges-1.1.0/.gitignore +104 -0
  2. borges-1.1.0/CHANGELOG.md +68 -0
  3. borges-1.1.0/CITATION.cff +49 -0
  4. borges-1.1.0/LICENSE +21 -0
  5. borges-1.1.0/PKG-INFO +321 -0
  6. borges-1.1.0/README.md +266 -0
  7. borges-1.1.0/config.yaml +252 -0
  8. borges-1.1.0/data/reference/negative_samples/bootstrap_hexagon_b_variant.png +0 -0
  9. borges-1.1.0/data/reference/negative_samples/bootstrap_purple_square_b.png +0 -0
  10. borges-1.1.0/data/reference/negative_samples/chinese_hosting_text_logo.png +0 -0
  11. borges-1.1.0/data/reference/negative_samples/facebook_blue_f_logo.png +0 -0
  12. borges-1.1.0/data/reference/negative_samples/generic_corporate_logo.png +0 -0
  13. borges-1.1.0/data/reference/negative_samples/generic_globe_web_icon.png +0 -0
  14. borges-1.1.0/data/reference/negative_samples/generic_home_house_icon.png +0 -0
  15. borges-1.1.0/data/reference/negative_samples/generic_hosting_server_icon.png +0 -0
  16. borges-1.1.0/data/reference/negative_samples/generic_isp_branding.png +0 -0
  17. borges-1.1.0/data/reference/negative_samples/generic_platform_icon.png +0 -0
  18. borges-1.1.0/data/reference/negative_samples/generic_provider_logo.png +0 -0
  19. borges-1.1.0/data/reference/negative_samples/generic_warning_triangle.png +0 -0
  20. borges-1.1.0/data/reference/negative_samples/google_play_triangular_button.png +0 -0
  21. borges-1.1.0/data/reference/negative_samples/mega_group_209asn_generic_e99155e1_urls.txt +28 -0
  22. borges-1.1.0/data/reference/negative_samples/ngnix.png +0 -0
  23. borges-1.1.0/data/reference/negative_samples/peeringdb_default_favicon.png +0 -0
  24. borges-1.1.0/data/reference/negative_samples/unknown_generic_favicon.png +0 -0
  25. borges-1.1.0/data/reference/negative_samples/wordpress.png +0 -0
  26. borges-1.1.0/data/reference/negative_samples/wordpress2.png +0 -0
  27. borges-1.1.0/data/reference/negative_samples/wordpress_default_w_logo.png +0 -0
  28. borges-1.1.0/docs/merge-guard-evaluation.md +102 -0
  29. borges-1.1.0/pyproject.toml +162 -0
  30. borges-1.1.0/scripts/check_wheel.py +57 -0
  31. borges-1.1.0/scripts/download_data.py +10 -0
  32. borges-1.1.0/scripts/evaluate_merge_guard.py +185 -0
  33. borges-1.1.0/scripts/migrate_data.py +320 -0
  34. borges-1.1.0/src/borges/__init__.py +20 -0
  35. borges-1.1.0/src/borges/analyzers/__init__.py +14 -0
  36. borges-1.1.0/src/borges/analyzers/llm_analyzer.py +500 -0
  37. borges-1.1.0/src/borges/analyzers/network_consolidator.py +1989 -0
  38. borges-1.1.0/src/borges/analyzers/number_validator.py +311 -0
  39. borges-1.1.0/src/borges/analyzers/redirect_analyzer.py +203 -0
  40. borges-1.1.0/src/borges/analyzers/whois_analyzer.py +184 -0
  41. borges-1.1.0/src/borges/cli.py +634 -0
  42. borges-1.1.0/src/borges/config.py +395 -0
  43. borges-1.1.0/src/borges/data/__init__.py +46 -0
  44. borges-1.1.0/src/borges/data/download.py +330 -0
  45. borges-1.1.0/src/borges/data/exporters.py +319 -0
  46. borges-1.1.0/src/borges/data/loaders.py +510 -0
  47. borges-1.1.0/src/borges/data/processors.py +476 -0
  48. borges-1.1.0/src/borges/models/__init__.py +27 -0
  49. borges-1.1.0/src/borges/models/as_network.py +330 -0
  50. borges-1.1.0/src/borges/models/schemas.py +212 -0
  51. borges-1.1.0/src/borges/pipeline/__init__.py +27 -0
  52. borges-1.1.0/src/borges/pipeline/runner.py +409 -0
  53. borges-1.1.0/src/borges/pipeline/stages.py +1404 -0
  54. borges-1.1.0/src/borges/resources.py +34 -0
  55. borges-1.1.0/src/borges/scrapers/__init__.py +11 -0
  56. borges-1.1.0/src/borges/scrapers/favicon_scraper.py +269 -0
  57. borges-1.1.0/src/borges/scrapers/html_scraper.py +239 -0
  58. borges-1.1.0/src/borges/scrapers/redirect_scraper.py +249 -0
  59. borges-1.1.0/src/borges/utils/__init__.py +19 -0
  60. borges-1.1.0/src/borges/utils/http_client.py +231 -0
  61. borges-1.1.0/src/borges/utils/llm_client.py +231 -0
  62. borges-1.1.0/src/borges/utils/logging.py +171 -0
  63. borges-1.1.0/tests/__init__.py +0 -0
  64. borges-1.1.0/tests/test_config.py +162 -0
  65. borges-1.1.0/tests/test_merge_guard.py +199 -0
  66. borges-1.1.0/tests/test_models.py +171 -0
  67. borges-1.1.0/tests/test_processors.py +220 -0
@@ -0,0 +1,104 @@
1
+ # Environment files
2
+ .env
3
+ .env.local
4
+ .env.*.local
5
+
6
+ # Data directories
7
+ data/
8
+ !src/borges/data/
9
+ !data/reference/
10
+ raw_htmls_2024/
11
+ input_files/
12
+ output_files/
13
+ favicons_2024/
14
+
15
+ # Python
16
+ __pycache__/
17
+ *.py[cod]
18
+ *$py.class
19
+ *.so
20
+ .Python
21
+ build/
22
+ develop-eggs/
23
+ dist/
24
+ downloads/
25
+ eggs/
26
+ .eggs/
27
+ lib/
28
+ lib64/
29
+ parts/
30
+ sdist/
31
+ var/
32
+ wheels/
33
+ *.egg-info/
34
+ .installed.cfg
35
+ *.egg
36
+ MANIFEST
37
+
38
+ # Virtual environments
39
+ venv/
40
+ ENV/
41
+ env/
42
+ .venv
43
+
44
+ # IDE
45
+ .vscode/
46
+ .idea/
47
+ *.swp
48
+ *.swo
49
+ *~
50
+ .DS_Store
51
+
52
+ # Testing
53
+ .coverage
54
+ .pytest_cache/
55
+ htmlcov/
56
+ .tox/
57
+ .nox/
58
+ coverage.xml
59
+ *.cover
60
+ .hypothesis/
61
+
62
+ # Jupyter Notebook
63
+ .ipynb_checkpoints
64
+ *.ipynb
65
+
66
+ # Logs
67
+ logs/
68
+ *.log
69
+
70
+ # Temporary files
71
+ *.tmp
72
+ *.temp
73
+ *.cache
74
+
75
+ # Project specific
76
+ checkpoints/
77
+ *.parquet
78
+ *.feather
79
+ *.hdf5
80
+ *.h5
81
+ *.pickle
82
+ *.pkl
83
+
84
+ # Documentation
85
+ docs/_build/
86
+ site/
87
+
88
+ # Package managers
89
+ pip-log.txt
90
+ pip-delete-this-directory.txt
91
+
92
+ # Type checking
93
+ .mypy_cache/
94
+ .dmypy.json
95
+ dmypy.json
96
+ .pyre/
97
+ .pytype/
98
+
99
+ # Local configuration
100
+ config.local.yaml
101
+ settings.local.json
102
+
103
+ # Forensic analysis directory (for debugging/development)
104
+ /forensic_analysis/
@@ -0,0 +1,68 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning: [SemVer](https://semver.org).
5
+ Changes that alter the inferred mappings are listed separately under *Changed (results)*.
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [1.1.0] - 2026-10-01
10
+
11
+ First release on PyPI (`pip install borges`).
12
+
13
+ ### Added
14
+
15
+ - The default configuration (the paper's prompts, blocklists and PeeringDB exclusions)
16
+ and the reference favicons now ship inside the package. `borges init` writes the full
17
+ configuration, so a pip install behaves like a repository checkout.
18
+ - `borges download`: fetch PeeringDB and AS2Org snapshots from CAIDA (previously only
19
+ `scripts/download_data.py`, which still works).
20
+ - Release workflow: a `vX.Y.Z` tag publishes to PyPI (trusted publishing) and creates the
21
+ GitHub release. CI builds the package and smoke-tests the wheel in a clean environment.
22
+
23
+ - Merge guard for network consolidation (`processing.merge_guard`, **off by default**):
24
+ favicon groups must share a website brand; websites of lookup services (RIR RDAP,
25
+ bgp.tools, …) are dropped; an unverified weak link cannot join two established
26
+ organizations on its own; one extracted relationship cannot join two large
27
+ organizations; and the PeeringDB pass ignores single-ASN ties where WHOIS disagrees
28
+ (the AS4004 pattern, optional and off by default). WHOIS and PeeringDB organizations
29
+ count as corroborating links. Decisions go to `merge_guard_review_*.json`. On the
30
+ 2025-09-29 run, groups combining β‰₯ 20 WHOIS organizations drop from 6 to 0, and
31
+ wrong-merge pairs in the changed groups drop by 96% while 92% of correct pairs are
32
+ kept. On the 2025-08 replays, Level3 and Orange stay apart without hand blocklists
33
+ (see `docs/merge-guard-evaluation.md`).
34
+ - `scripts/evaluate_merge_guard.py`: replay consolidation offline from a finished run
35
+ (no scraping, no LLM calls), compare it with the original output, and sweep guard
36
+ settings with `--set key=value`.
37
+
38
+ - `api.openai.base_url`: run the LLM stages against any OpenAI-compatible server (e.g.
39
+ a local Ollama), so no paid API key is required.
40
+ - `CITATION.cff` (with the paper's DOI), `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`,
41
+ `SECURITY.md`, `AGENTS.md`, issue and pull request templates, `CODEOWNERS` and
42
+ Dependabot.
43
+ - `.git-blame-ignore-revs` for the bulk `black` reformat.
44
+
45
+ ### Changed
46
+
47
+ - Python 3.10+ is required (3.9 is end-of-life). CI tests 3.10–3.13.
48
+ - README restructured. It now documents running without a paid key and the stage list
49
+ (including `network_consolidation`).
50
+ - Code formatted with `black` 26, pinned to `<27` in the dev extras.
51
+
52
+ ### Fixed
53
+
54
+ - `borges init` crashed when no `config.yaml` existed yet.
55
+ - `--config` / `BORGES_CONFIG` was ignored by the LLM client and analyzers, which
56
+ re-read `./config.yaml`.
57
+ - CI had never passed: it called `flake8`, which was not installed; replaced by `ruff`.
58
+
59
+ ### Changed (results)
60
+
61
+ - `DataCleaner.clean_url` now rejects website URLs whose host has no dot (e.g.
62
+ `https://short`). Such URLs cannot resolve, so they could not produce redirects or
63
+ favicons before either.
64
+
65
+ ## [1.0] - 2025-08-20
66
+
67
+ Restructured from research scripts into a Python package with the `borges` CLI
68
+ (tagged `v1.0` on GitHub; `pyproject.toml` said 0.2.0 at the time).
@@ -0,0 +1,49 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use Borges in your research, please cite the IMC 2025 paper below."
3
+ title: "Borges: AS-to-Organization mapping framework"
4
+ type: software
5
+ version: 1.1.0
6
+ date-released: 2026-10-01
7
+ license: MIT
8
+ repository-code: "https://github.com/NU-AquaLab/borges"
9
+ url: "https://nu-aqualab.github.io/borges-website/"
10
+ keywords:
11
+ - autonomous systems
12
+ - AS-to-organization
13
+ - as2org
14
+ - PeeringDB
15
+ - WHOIS
16
+ - internet measurement
17
+ authors:
18
+ - given-names: Carlos
19
+ family-names: Selmo
20
+ - given-names: Esteban
21
+ family-names: Carisimo
22
+ - given-names: FabiΓ‘n E.
23
+ family-names: Bustamante
24
+ - given-names: J. Ignacio
25
+ family-names: Alvarez-Hamelin
26
+ preferred-citation:
27
+ type: conference-paper
28
+ title: "Learning AS-to-Organization Mappings with Borges"
29
+ authors:
30
+ - given-names: Carlos
31
+ family-names: Selmo
32
+ - given-names: Esteban
33
+ family-names: Carisimo
34
+ - given-names: FabiΓ‘n E.
35
+ family-names: Bustamante
36
+ - given-names: J. Ignacio
37
+ family-names: Alvarez-Hamelin
38
+ collection-title: "Proceedings of the 2025 ACM Internet Measurement Conference"
39
+ conference:
40
+ name: "ACM Internet Measurement Conference (IMC '25)"
41
+ city: Madison
42
+ region: WI
43
+ country: US
44
+ publisher:
45
+ name: Association for Computing Machinery
46
+ doi: 10.1145/3730567.3732918
47
+ year: 2025
48
+ month: 10
49
+ url: "https://doi.org/10.1145/3730567.3732918"
borges-1.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Carlos Selmo, Esteban Carisimo, FabiΓ‘n E. Bustamante, J. Ignacio Alvarez-Hamelin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
borges-1.1.0/PKG-INFO ADDED
@@ -0,0 +1,321 @@
1
+ Metadata-Version: 2.5
2
+ Name: borges
3
+ Version: 1.1.0
4
+ Summary: AS-to-Organization mapping: infer sibling Autonomous Systems from WHOIS, PeeringDB and web signals (IMC 2025)
5
+ Project-URL: Homepage, https://nu-aqualab.github.io/borges-website/
6
+ Project-URL: Repository, https://github.com/NU-AquaLab/borges
7
+ Project-URL: Paper, https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf
8
+ Author: Carlos Selmo, Esteban Carisimo, Fabian E. Bustamante, J. Ignacio Alvarez-Hamelin
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: analysis,as-to-organization,as2org,autonomous-systems,network,peeringdb
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Intended Audience :: Telecommunications Industry
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
23
+ Classifier: Topic :: System :: Networking
24
+ Requires-Python: >=3.10
25
+ Requires-Dist: beautifulsoup4>=4.12.0
26
+ Requires-Dist: click>=8.1.0
27
+ Requires-Dist: httpx>=0.25.0
28
+ Requires-Dist: langchain-community>=0.1.0
29
+ Requires-Dist: langchain-openai>=0.1.0
30
+ Requires-Dist: langchain>=0.1.0
31
+ Requires-Dist: openai>=1.0.0
32
+ Requires-Dist: pandas>=2.0.0
33
+ Requires-Dist: pillow>=10.0.0
34
+ Requires-Dist: pyarrow>=14.0.0
35
+ Requires-Dist: pydantic>=2.0.0
36
+ Requires-Dist: python-dotenv>=1.0.0
37
+ Requires-Dist: pyyaml>=6.0
38
+ Requires-Dist: requests>=2.31.0
39
+ Requires-Dist: structlog>=24.0.0
40
+ Requires-Dist: tenacity>=8.2.0
41
+ Requires-Dist: tldextract>=5.0.0
42
+ Requires-Dist: tqdm>=4.66.0
43
+ Provides-Extra: dev
44
+ Requires-Dist: black<27,>=26.0.0; extra == 'dev'
45
+ Requires-Dist: ipython>=8.0.0; extra == 'dev'
46
+ Requires-Dist: mypy>=1.5.0; extra == 'dev'
47
+ Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
48
+ Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
49
+ Requires-Dist: pytest-mock>=3.11.0; extra == 'dev'
50
+ Requires-Dist: pytest>=7.4.0; extra == 'dev'
51
+ Requires-Dist: ruff>=0.1.0; extra == 'dev'
52
+ Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
53
+ Requires-Dist: types-requests>=2.31.0; extra == 'dev'
54
+ Description-Content-Type: text/markdown
55
+
56
+ # πŸ—ΊοΈ Borges
57
+
58
+ **Borges** (*Better ORGanizations Entities mappingS*) maps Autonomous Systems (ASes) to the organizations that operate them. It finds **sibling ASes** run by the same company, even when WHOIS lists them under different organizations. It combines CAIDA's WHOIS-based AS2Org and PeeringDB with two new signals: **LLM extraction** of sibling ASNs from PeeringDB's free-text fields, and **website inference** from redirect chains, domain similarity and favicons. It is the code of the ACM IMC 2025 paper [*Learning AS-to-Organization Mappings with Borges*](https://doi.org/10.1145/3730567.3732918).
59
+
60
+ [![CI](https://github.com/NU-AquaLab/borges/actions/workflows/ci.yml/badge.svg)](https://github.com/NU-AquaLab/borges/actions/workflows/ci.yml)
61
+ [![Website](https://img.shields.io/badge/website-nu--aqualab.github.io%2Fborges--website-blue.svg)](https://nu-aqualab.github.io/borges-website/)
62
+ [![Paper](https://img.shields.io/badge/DOI-10.1145%2F3730567.3732918-informational.svg)](https://doi.org/10.1145/3730567.3732918)
63
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
64
+ [![PyPI](https://img.shields.io/pypi/v/borges.svg)](https://pypi.org/project/borges/)
65
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
66
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
67
+
68
+ > [!NOTE]
69
+ > **Mappings go stale quickly.** Companies merge, rebrand and disappear. Edgio vanished
70
+ > between the paper's submission and its presentation. The [published artifacts](https://nu-aqualab.github.io/borges-website/)
71
+ > are a September 2025 snapshot. To get current mappings, run Borges on fresh PeeringDB
72
+ > and AS2Org data. A full run with `gpt-4o-mini` costs about **US$3**. You can also run it
73
+ > with **no paid API key** (see [below](#-running-without-a-paid-api-key)).
74
+
75
+ ## ✨ Features
76
+
77
+ - 🏒 **Organization keys**: groups ASes by CAIDA AS2Org (WHOIS) organization IDs and PeeringDB organization IDs
78
+ - πŸ€– **LLM sibling extraction**: few-shot prompting reads PeeringDB `notes` and `aka` fields and keeps only real sibling ASNs, not phone numbers, years or prefix limits
79
+ - πŸ”€ **Redirect analysis**: networks whose websites resolve to the same final URL are grouped together
80
+ - πŸ–ΌοΈ **Favicon analysis**: shared favicons plus similar domains reveal common branding; a vision LLM rejects framework defaults (WordPress, Bootstrap, …)
81
+ - 🧩 **Consolidation**: merges every signal into network groups, with blocklists for known false bridges
82
+ - πŸ’Ύ **Robust pipeline**: nine stages with checkpoint/resume, parallel scraping with caching, and Parquet/JSON/CSV exports
83
+ - πŸ’Έ **Free to run**: tests and CI need no key. The LLM stages can be skipped or pointed at a local model
84
+
85
+ ## πŸš€ Quick Start
86
+
87
+ ```bash
88
+ pip install borges # or: uv pip install borges
89
+ borges init # writes config.yaml (prompts, blocklists), .env and data/
90
+ borges download # latest PeeringDB + AS2Org snapshots into data/input/
91
+ borges pipeline run # needs an LLM key, or see below to skip it
92
+ ```
93
+
94
+ From a clone (for development):
95
+
96
+ ```bash
97
+ git clone https://github.com/NU-AquaLab/borges.git
98
+ cd borges
99
+ uv venv && source .venv/bin/activate
100
+ uv pip install -e ".[dev]"
101
+ ```
102
+
103
+ ## πŸ“– Usage
104
+
105
+ ### Pipeline stages
106
+
107
+ | # | Stage | What it does | LLM |
108
+ |---|---|---|---|
109
+ | 1 | `load_data` | Load PeeringDB and WHOIS (AS2Org) data | |
110
+ | 2 | `redirect_scraping` | Follow each PeeringDB website to its final URL (no HTML stored) | |
111
+ | 3 | `as_detection` | Extract sibling ASNs from `notes` and `aka` | πŸ€– |
112
+ | 4 | `redirect_analysis` | Group networks that share a final URL | |
113
+ | 5 | `favicon_download` | Download website favicons | |
114
+ | 6 | `favicon_analysis` | Decide whether shared favicons mean a shared company | πŸ€– |
115
+ | 7 | `whois_processing` | Group ASes by AS2Org organization | |
116
+ | 8 | `network_consolidation` | Merge groups from all sources | |
117
+ | 9 | `export_results` | Write Parquet/JSON/CSV and reports to `data/output/` | |
118
+
119
+ ### Commands
120
+
121
+ ```bash
122
+ borges pipeline list # stages and their dependencies
123
+ borges pipeline run # full run
124
+ borges pipeline run --stage redirect_scraping --stage as_detection # selected stages
125
+ borges pipeline run --skip favicon_analysis # skip stages
126
+ borges pipeline run --resume # continue from checkpoints
127
+ borges pipeline run --dry-run # show what would run
128
+ borges --config my-config.yaml pipeline run # or: export BORGES_CONFIG=...
129
+ borges download --peeringdb-date 2025-07-16 --as2org-date 2025-07-01
130
+ ```
131
+
132
+ `borges --help` lists the rest (`report generate`, `favicon download`, `config`, `version`).
133
+
134
+ ### πŸ’Έ Running without a paid API key
135
+
136
+ Only stages 3 and 6 call an LLM. Pick one option:
137
+
138
+ | Option | How | Trade-off |
139
+ |---|---|---|
140
+ | **Skip the LLM stages** | `borges pipeline run --skip as_detection --skip favicon_analysis` | No sibling extraction from `notes`/`aka`, no favicon grouping |
141
+ | **Free local model** | Any OpenAI-compatible server: [Ollama](https://ollama.com), LM Studio, vLLM. Set `base_url` (below) | Results differ from the paper, which used `gpt-4o-mini` |
142
+ | **OpenAI** | `OPENAI_API_KEY=...` in `.env` | About US$3 per full run |
143
+
144
+ With Ollama (`ollama pull llama3.2` and `ollama pull llama3.2-vision`), set this in `config.yaml`:
145
+
146
+ ```yaml
147
+ api:
148
+ openai:
149
+ api_key: unused # local servers ignore it, but it must be set
150
+ base_url: http://localhost:11434/v1
151
+ model: llama3.2
152
+ vision_model: llama3.2-vision
153
+ ```
154
+
155
+ ### Programmatic use
156
+
157
+ ```python
158
+ from borges.config import load_config, set_config
159
+ from borges.pipeline import Pipeline
160
+
161
+ config = load_config("config.yaml")
162
+ set_config(config)
163
+ pipeline = Pipeline(config)
164
+ results = pipeline.run(skip_stages=["as_detection", "favicon_analysis"])
165
+ ```
166
+
167
+ ## πŸ—ƒοΈ Output
168
+
169
+ `export_results` writes timestamped files to `data/output/`:
170
+
171
+ | File | Contents |
172
+ |---|---|
173
+ | `autonomous_systems_*.parquet` | One row per AS |
174
+ | `organizations_*.parquet` | Organization groupings |
175
+ | `relationships_*.parquet` | Detected sibling relationships and their source |
176
+ | `network_groups_*.parquet` | Consolidated groups of sibling ASes |
177
+ | `redirect_analysis_*.parquet` | Final URL of each network's website |
178
+ | `network_report_*.json` | Full analysis report |
179
+ | `export_summary_*.json` | Export metadata and statistics |
180
+
181
+ The main settings in `config.yaml`:
182
+
183
+ | Key | Default | Purpose |
184
+ |---|---|---|
185
+ | `api.openai.model` / `vision_model` | `gpt-4o-mini` | Text and vision models |
186
+ | `api.openai.base_url` | *(unset: OpenAI)* | Any OpenAI-compatible endpoint |
187
+ | `api.openai.request_delay` / `retry_delay` | `0.5` / `30` s | Rate limiting |
188
+ | `scraping.html.max_workers` / `timeout` | `100` / `30` s | Redirect scraping parallelism |
189
+ | `scraping.favicon.max_workers` | `50` | Favicon download parallelism |
190
+ | `peeringdb_asn_exclusions`, blocklists | see file | Known false organizational bridges |
191
+ | `processing.merge_guard.enabled` | `false` | Guard against false merges (brand check, bridge guard); see [evaluation](docs/merge-guard-evaluation.md) |
192
+
193
+ ## πŸ”¬ Methodology (brief)
194
+
195
+ The full method is in the [paper](https://doi.org/10.1145/3730567.3732918) ([PDF](https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf)). In short:
196
+
197
+ - **Organization keys.** Start from organization IDs in CAIDA's AS2Org (WHOIS) and in PeeringDB.
198
+ - **Information extraction.** Few-shot prompting of `gpt-4o-mini` (temperature 0) recovers sibling ASNs written in PeeringDB's `notes` and `aka` fields. Earlier work used regular expressions, which confuse phone numbers, years and prefix limits with ASNs.
199
+ - **Website inference.**
200
+ - *Redirects*: networks whose PeeringDB websites end at the same final URL are siblings. For example, Limelight (AS22822) and Edgecast (AS15133) both redirect to `www.edg.io`.
201
+ - *Favicons*: websites with the same favicon and the same subdomain are grouped. Remaining shared favicons go to a vision-LLM classifier that separates company logos (Claro Chile and Claro Puerto Rico) from framework defaults (Bootstrap, WordPress).
202
+ - **Consolidation.** All signals are merged into organization groups.
203
+
204
+ Results on the July 2024 PeeringDB and AS2Org snapshots:
205
+
206
+ | Measure | Result |
207
+ |---|---|
208
+ | Sibling extraction from `notes`/`aka` (320 records checked by hand) | accuracy 0.947 Β· precision 0.974 Β· recall 0.94 |
209
+ | Favicon company classifier (449 favicons checked by hand) | accuracy 0.986 Β· precision 0.997 Β· recall 0.984 |
210
+ | Organization Factor (new metric, 0 to 1) | 0.3576: +7% over AS2Org, +3.3% over as2org+ |
211
+ | Users newly attributed to large conglomerates | β‰ˆ192 million (β‰ˆ5% of the Internet population) |
212
+
213
+ ### ⚠️ Known limitations
214
+
215
+ - **No website history.** There is no longitudinal archive of the websites listed in PeeringDB, so past runs cannot be reproduced exactly from the code alone. Keep your inputs and outputs.
216
+ - **PeeringDB coverage is partial.** Registration is voluntary, and entries can be incomplete or outdated. For example, some Microsoft ASNs are missing.
217
+ - **Errors in the input propagate.** If a PeeringDB record lists the wrong sibling, Borges faithfully extracts the wrong sibling (e.g. AS10026 listing AS2706).
218
+ - **Layered ownership is out of scope.** Groups spanning separate brands and regions are not linked, such as AmΓ©rica MΓ³vil's Claro and A1.
219
+ - **False merges through weak signals.** One favicon or website shared by unrelated networks can chain them into one group. The optional [merge guard](docs/merge-guard-evaluation.md) prevents most of these. It is off by default until its results are reviewed.
220
+ - **Known false bridges** are handled by configuration:
221
+ - AS4004 appears under Orange in PeeringDB but under Sprint in WHOIS. It is listed in `peeringdb_asn_exclusions`.
222
+ - Shared PeeringDB listings and default favicons can wrongly join small networks to large operators. They are covered by blocklists in `config.yaml`.
223
+ - **LLM output varies by model.** The published numbers are for `gpt-4o-mini`. Local or newer models need their own validation.
224
+
225
+ ## πŸ—οΈ Architecture
226
+
227
+ ```
228
+ src/borges/
229
+ β”œβ”€β”€ cli.py # Click CLI: init | pipeline | report | favicon | config | version
230
+ β”œβ”€β”€ config.py # Pydantic config, ${ENV} interpolation, global get_config()/set_config()
231
+ β”œβ”€β”€ pipeline/
232
+ β”‚ β”œβ”€β”€ runner.py # Pipeline: stage ordering, dependencies, checkpoints
233
+ β”‚ └── stages.py # the nine stages
234
+ β”œβ”€β”€ analyzers/ # LLM sibling extraction, redirects, WHOIS, number validation, consolidation
235
+ β”œβ”€β”€ scrapers/ # redirect, HTML and favicon scrapers
236
+ β”œβ”€β”€ data/ # loaders, processors, exporters
237
+ β”œβ”€β”€ models/ # AS network model and Pydantic schemas
238
+ └── utils/ # LLM client (OpenAI-compatible), HTTP client, logging
239
+ scripts/ # download_data.py (= borges download), evaluate_merge_guard.py, check_wheel.py
240
+ config.yaml # default configuration, prompts and blocklists (shipped in the wheel)
241
+ data/reference/ # negative favicon samples (framework defaults)
242
+ tests/ # unit tests; no network, no API key
243
+ ```
244
+
245
+ ## πŸ§ͺ Development
246
+
247
+ ```bash
248
+ uv pip install -e ".[dev]"
249
+ ruff check src tests scripts --select E9,F63,F7,F82 # syntax errors and undefined names
250
+ black --check src tests # formatting (black 26)
251
+ mypy src/borges # informational for now
252
+ pytest # no network, no API key
253
+ git config blame.ignoreRevsFile .git-blame-ignore-revs # hide the bulk reformat from blame
254
+ ```
255
+
256
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
257
+
258
+ ## πŸ“Š Example Output
259
+
260
+ ```
261
+ $ borges pipeline run --skip as_detection --skip favicon_analysis --dry-run
262
+ Pipeline dry run - stages that would be executed:
263
+ βœ“ load_data: Load initial data from PeeringDB and WHOIS.
264
+ βœ“ redirect_scraping: Scrape redirect information from websites (no HTML content).
265
+ βœ“ redirect_analysis: Analyze URL redirects.
266
+ βœ“ favicon_download: Scrape favicons from websites.
267
+ βœ“ whois_processing: Process WHOIS data.
268
+ βœ“ network_consolidation: Consolidate network groups from different analysis sources.
269
+ βœ“ export_results: Export final results.
270
+ ```
271
+
272
+ ## 🀝 Contributing
273
+
274
+ Bug fixes, documentation, tests and new signals are welcome. Please open an issue before large changes. See [CONTRIBUTING.md](CONTRIBUTING.md), the [Code of Conduct](CODE_OF_CONDUCT.md), and [SECURITY.md](SECURITY.md) to report a vulnerability privately.
275
+
276
+ 1. Fork the repository
277
+ 2. Create a feature branch (`git checkout -b fix/my-fix`)
278
+ 3. Commit your changes (`git commit -m 'Fix …'`)
279
+ 4. Push to the branch (`git push origin fix/my-fix`)
280
+ 5. Open a Pull Request
281
+
282
+ ## πŸ“„ License
283
+
284
+ [MIT](LICENSE).
285
+
286
+ ## πŸ“ Citation
287
+
288
+ If you use Borges, please cite the paper. It is also in [CITATION.cff](CITATION.cff), which feeds GitHub's **Cite this repository** button:
289
+
290
+ ```bibtex
291
+ @inproceedings{borges:imc,
292
+ author = {Selmo, Carlos and Carisimo, Esteban and Bustamante, Fabi{\'a}n E. and Alvarez-Hamelin, J. Ignacio},
293
+ title = {Learning AS-to-Organization Mappings with Borges},
294
+ booktitle = {Proceedings of the 2025 ACM Internet Measurement Conference},
295
+ series = {IMC '25},
296
+ year = {2025},
297
+ month = {10},
298
+ location = {Madison, WI, USA},
299
+ publisher = {Association for Computing Machinery},
300
+ doi = {10.1145/3730567.3732918},
301
+ url = {https://doi.org/10.1145/3730567.3732918}
302
+ }
303
+ ```
304
+
305
+ ## πŸ”— Related Resources
306
+
307
+ - **Paper**: [ACM Digital Library](https://doi.org/10.1145/3730567.3732918) Β· [PDF](https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf)
308
+ - **Website and artifacts**: [nu-aqualab.github.io/borges-website](https://nu-aqualab.github.io/borges-website/) (September 2025 mappings)
309
+ - **Press**: [Mapping Who Really Runs the Internet: Introducing Borges](https://pulse.internetsociety.org/blog/mapping-who-really-runs-the-internet-introducing-borges) (Internet Society Pulse)
310
+ - **CAIDA AS2Org**: [AS Organizations dataset](https://www.caida.org/catalog/datasets/as-organizations/) and the [PeeringDB archive](https://publicdata.caida.org/datasets/peeringdb/)
311
+ - **as2org+**: Arturi, Carisimo and Bustamante, *as2org+: Enriching AS-to-Organization Mappings with PeeringDB*, PAM 2023
312
+ - **State-Owned ASes**: [estcarisimo/state-owned-ases](https://github.com/estcarisimo/state-owned-ases), a hand-built dataset where Borges-style automation could help
313
+
314
+ ## πŸ™ Acknowledgements
315
+
316
+ - **Paper**: *Learning AS-to-Organization Mappings with Borges*
317
+ - **Authors**: Carlos Selmo, Esteban Carisimo, FabiΓ‘n E. Bustamante, J. Ignacio Alvarez-Hamelin
318
+ - **Conference**: ACM Internet Measurement Conference (IMC) 2025, Madison, WI, USA
319
+ - **Funding**: NSF grant CNS-2107392 and UBACyT 20020220100053BA
320
+
321
+ Thanks to [CAIDA](https://www.caida.org/) for archiving PeeringDB snapshots and publishing AS2Org, to [PeeringDB](https://www.peeringdb.com), and to [@zhiyichenGT](https://github.com/zhiyichenGT) for the consolidator and WHOIS fixes.