borges 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- borges-1.1.0/.gitignore +104 -0
- borges-1.1.0/CHANGELOG.md +68 -0
- borges-1.1.0/CITATION.cff +49 -0
- borges-1.1.0/LICENSE +21 -0
- borges-1.1.0/PKG-INFO +321 -0
- borges-1.1.0/README.md +266 -0
- borges-1.1.0/config.yaml +252 -0
- borges-1.1.0/data/reference/negative_samples/bootstrap_hexagon_b_variant.png +0 -0
- borges-1.1.0/data/reference/negative_samples/bootstrap_purple_square_b.png +0 -0
- borges-1.1.0/data/reference/negative_samples/chinese_hosting_text_logo.png +0 -0
- borges-1.1.0/data/reference/negative_samples/facebook_blue_f_logo.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_corporate_logo.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_globe_web_icon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_home_house_icon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_hosting_server_icon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_isp_branding.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_platform_icon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_provider_logo.png +0 -0
- borges-1.1.0/data/reference/negative_samples/generic_warning_triangle.png +0 -0
- borges-1.1.0/data/reference/negative_samples/google_play_triangular_button.png +0 -0
- borges-1.1.0/data/reference/negative_samples/mega_group_209asn_generic_e99155e1_urls.txt +28 -0
- borges-1.1.0/data/reference/negative_samples/ngnix.png +0 -0
- borges-1.1.0/data/reference/negative_samples/peeringdb_default_favicon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/unknown_generic_favicon.png +0 -0
- borges-1.1.0/data/reference/negative_samples/wordpress.png +0 -0
- borges-1.1.0/data/reference/negative_samples/wordpress2.png +0 -0
- borges-1.1.0/data/reference/negative_samples/wordpress_default_w_logo.png +0 -0
- borges-1.1.0/docs/merge-guard-evaluation.md +102 -0
- borges-1.1.0/pyproject.toml +162 -0
- borges-1.1.0/scripts/check_wheel.py +57 -0
- borges-1.1.0/scripts/download_data.py +10 -0
- borges-1.1.0/scripts/evaluate_merge_guard.py +185 -0
- borges-1.1.0/scripts/migrate_data.py +320 -0
- borges-1.1.0/src/borges/__init__.py +20 -0
- borges-1.1.0/src/borges/analyzers/__init__.py +14 -0
- borges-1.1.0/src/borges/analyzers/llm_analyzer.py +500 -0
- borges-1.1.0/src/borges/analyzers/network_consolidator.py +1989 -0
- borges-1.1.0/src/borges/analyzers/number_validator.py +311 -0
- borges-1.1.0/src/borges/analyzers/redirect_analyzer.py +203 -0
- borges-1.1.0/src/borges/analyzers/whois_analyzer.py +184 -0
- borges-1.1.0/src/borges/cli.py +634 -0
- borges-1.1.0/src/borges/config.py +395 -0
- borges-1.1.0/src/borges/data/__init__.py +46 -0
- borges-1.1.0/src/borges/data/download.py +330 -0
- borges-1.1.0/src/borges/data/exporters.py +319 -0
- borges-1.1.0/src/borges/data/loaders.py +510 -0
- borges-1.1.0/src/borges/data/processors.py +476 -0
- borges-1.1.0/src/borges/models/__init__.py +27 -0
- borges-1.1.0/src/borges/models/as_network.py +330 -0
- borges-1.1.0/src/borges/models/schemas.py +212 -0
- borges-1.1.0/src/borges/pipeline/__init__.py +27 -0
- borges-1.1.0/src/borges/pipeline/runner.py +409 -0
- borges-1.1.0/src/borges/pipeline/stages.py +1404 -0
- borges-1.1.0/src/borges/resources.py +34 -0
- borges-1.1.0/src/borges/scrapers/__init__.py +11 -0
- borges-1.1.0/src/borges/scrapers/favicon_scraper.py +269 -0
- borges-1.1.0/src/borges/scrapers/html_scraper.py +239 -0
- borges-1.1.0/src/borges/scrapers/redirect_scraper.py +249 -0
- borges-1.1.0/src/borges/utils/__init__.py +19 -0
- borges-1.1.0/src/borges/utils/http_client.py +231 -0
- borges-1.1.0/src/borges/utils/llm_client.py +231 -0
- borges-1.1.0/src/borges/utils/logging.py +171 -0
- borges-1.1.0/tests/__init__.py +0 -0
- borges-1.1.0/tests/test_config.py +162 -0
- borges-1.1.0/tests/test_merge_guard.py +199 -0
- borges-1.1.0/tests/test_models.py +171 -0
- borges-1.1.0/tests/test_processors.py +220 -0
borges-1.1.0/.gitignore
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Environment files
|
|
2
|
+
.env
|
|
3
|
+
.env.local
|
|
4
|
+
.env.*.local
|
|
5
|
+
|
|
6
|
+
# Data directories
|
|
7
|
+
data/
|
|
8
|
+
!src/borges/data/
|
|
9
|
+
!data/reference/
|
|
10
|
+
raw_htmls_2024/
|
|
11
|
+
input_files/
|
|
12
|
+
output_files/
|
|
13
|
+
favicons_2024/
|
|
14
|
+
|
|
15
|
+
# Python
|
|
16
|
+
__pycache__/
|
|
17
|
+
*.py[cod]
|
|
18
|
+
*$py.class
|
|
19
|
+
*.so
|
|
20
|
+
.Python
|
|
21
|
+
build/
|
|
22
|
+
develop-eggs/
|
|
23
|
+
dist/
|
|
24
|
+
downloads/
|
|
25
|
+
eggs/
|
|
26
|
+
.eggs/
|
|
27
|
+
lib/
|
|
28
|
+
lib64/
|
|
29
|
+
parts/
|
|
30
|
+
sdist/
|
|
31
|
+
var/
|
|
32
|
+
wheels/
|
|
33
|
+
*.egg-info/
|
|
34
|
+
.installed.cfg
|
|
35
|
+
*.egg
|
|
36
|
+
MANIFEST
|
|
37
|
+
|
|
38
|
+
# Virtual environments
|
|
39
|
+
venv/
|
|
40
|
+
ENV/
|
|
41
|
+
env/
|
|
42
|
+
.venv
|
|
43
|
+
|
|
44
|
+
# IDE
|
|
45
|
+
.vscode/
|
|
46
|
+
.idea/
|
|
47
|
+
*.swp
|
|
48
|
+
*.swo
|
|
49
|
+
*~
|
|
50
|
+
.DS_Store
|
|
51
|
+
|
|
52
|
+
# Testing
|
|
53
|
+
.coverage
|
|
54
|
+
.pytest_cache/
|
|
55
|
+
htmlcov/
|
|
56
|
+
.tox/
|
|
57
|
+
.nox/
|
|
58
|
+
coverage.xml
|
|
59
|
+
*.cover
|
|
60
|
+
.hypothesis/
|
|
61
|
+
|
|
62
|
+
# Jupyter Notebook
|
|
63
|
+
.ipynb_checkpoints
|
|
64
|
+
*.ipynb
|
|
65
|
+
|
|
66
|
+
# Logs
|
|
67
|
+
logs/
|
|
68
|
+
*.log
|
|
69
|
+
|
|
70
|
+
# Temporary files
|
|
71
|
+
*.tmp
|
|
72
|
+
*.temp
|
|
73
|
+
*.cache
|
|
74
|
+
|
|
75
|
+
# Project specific
|
|
76
|
+
checkpoints/
|
|
77
|
+
*.parquet
|
|
78
|
+
*.feather
|
|
79
|
+
*.hdf5
|
|
80
|
+
*.h5
|
|
81
|
+
*.pickle
|
|
82
|
+
*.pkl
|
|
83
|
+
|
|
84
|
+
# Documentation
|
|
85
|
+
docs/_build/
|
|
86
|
+
site/
|
|
87
|
+
|
|
88
|
+
# Package managers
|
|
89
|
+
pip-log.txt
|
|
90
|
+
pip-delete-this-directory.txt
|
|
91
|
+
|
|
92
|
+
# Type checking
|
|
93
|
+
.mypy_cache/
|
|
94
|
+
.dmypy.json
|
|
95
|
+
dmypy.json
|
|
96
|
+
.pyre/
|
|
97
|
+
.pytype/
|
|
98
|
+
|
|
99
|
+
# Local configuration
|
|
100
|
+
config.local.yaml
|
|
101
|
+
settings.local.json
|
|
102
|
+
|
|
103
|
+
# Forensic analysis directory (for debugging/development)
|
|
104
|
+
/forensic_analysis/
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning: [SemVer](https://semver.org).
|
|
5
|
+
Changes that alter the inferred mappings are listed separately under *Changed (results)*.
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [1.1.0] - 2026-10-01
|
|
10
|
+
|
|
11
|
+
First release on PyPI (`pip install borges`).
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- The default configuration (the paper's prompts, blocklists and PeeringDB exclusions)
|
|
16
|
+
and the reference favicons now ship inside the package. `borges init` writes the full
|
|
17
|
+
configuration, so a pip install behaves like a repository checkout.
|
|
18
|
+
- `borges download`: fetch PeeringDB and AS2Org snapshots from CAIDA (previously only
|
|
19
|
+
`scripts/download_data.py`, which still works).
|
|
20
|
+
- Release workflow: a `vX.Y.Z` tag publishes to PyPI (trusted publishing) and creates the
|
|
21
|
+
GitHub release. CI builds the package and smoke-tests the wheel in a clean environment.
|
|
22
|
+
|
|
23
|
+
- Merge guard for network consolidation (`processing.merge_guard`, **off by default**):
|
|
24
|
+
favicon groups must share a website brand; websites of lookup services (RIR RDAP,
|
|
25
|
+
bgp.tools, β¦) are dropped; an unverified weak link cannot join two established
|
|
26
|
+
organizations on its own; one extracted relationship cannot join two large
|
|
27
|
+
organizations; and the PeeringDB pass ignores single-ASN ties where WHOIS disagrees
|
|
28
|
+
(the AS4004 pattern, optional and off by default). WHOIS and PeeringDB organizations
|
|
29
|
+
count as corroborating links. Decisions go to `merge_guard_review_*.json`. On the
|
|
30
|
+
2025-09-29 run, groups combining β₯ 20 WHOIS organizations drop from 6 to 0, and
|
|
31
|
+
wrong-merge pairs in the changed groups drop by 96% while 92% of correct pairs are
|
|
32
|
+
kept. On the 2025-08 replays, Level3 and Orange stay apart without hand blocklists
|
|
33
|
+
(see `docs/merge-guard-evaluation.md`).
|
|
34
|
+
- `scripts/evaluate_merge_guard.py`: replay consolidation offline from a finished run
|
|
35
|
+
(no scraping, no LLM calls), compare it with the original output, and sweep guard
|
|
36
|
+
settings with `--set key=value`.
|
|
37
|
+
|
|
38
|
+
- `api.openai.base_url`: run the LLM stages against any OpenAI-compatible server (e.g.
|
|
39
|
+
a local Ollama), so no paid API key is required.
|
|
40
|
+
- `CITATION.cff` (with the paper's DOI), `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`,
|
|
41
|
+
`SECURITY.md`, `AGENTS.md`, issue and pull request templates, `CODEOWNERS` and
|
|
42
|
+
Dependabot.
|
|
43
|
+
- `.git-blame-ignore-revs` for the bulk `black` reformat.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- Python 3.10+ is required (3.9 is end-of-life). CI tests 3.10β3.13.
|
|
48
|
+
- README restructured. It now documents running without a paid key and the stage list
|
|
49
|
+
(including `network_consolidation`).
|
|
50
|
+
- Code formatted with `black` 26, pinned to `<27` in the dev extras.
|
|
51
|
+
|
|
52
|
+
### Fixed
|
|
53
|
+
|
|
54
|
+
- `borges init` crashed when no `config.yaml` existed yet.
|
|
55
|
+
- `--config` / `BORGES_CONFIG` was ignored by the LLM client and analyzers, which
|
|
56
|
+
re-read `./config.yaml`.
|
|
57
|
+
- CI had never passed: it called `flake8`, which was not installed; replaced by `ruff`.
|
|
58
|
+
|
|
59
|
+
### Changed (results)
|
|
60
|
+
|
|
61
|
+
- `DataCleaner.clean_url` now rejects website URLs whose host has no dot (e.g.
|
|
62
|
+
`https://short`). Such URLs cannot resolve, so they could not produce redirects or
|
|
63
|
+
favicons before either.
|
|
64
|
+
|
|
65
|
+
## [1.0] - 2025-08-20
|
|
66
|
+
|
|
67
|
+
Restructured from research scripts into a Python package with the `borges` CLI
|
|
68
|
+
(tagged `v1.0` on GitHub; `pyproject.toml` said 0.2.0 at the time).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use Borges in your research, please cite the IMC 2025 paper below."
|
|
3
|
+
title: "Borges: AS-to-Organization mapping framework"
|
|
4
|
+
type: software
|
|
5
|
+
version: 1.1.0
|
|
6
|
+
date-released: 2026-10-01
|
|
7
|
+
license: MIT
|
|
8
|
+
repository-code: "https://github.com/NU-AquaLab/borges"
|
|
9
|
+
url: "https://nu-aqualab.github.io/borges-website/"
|
|
10
|
+
keywords:
|
|
11
|
+
- autonomous systems
|
|
12
|
+
- AS-to-organization
|
|
13
|
+
- as2org
|
|
14
|
+
- PeeringDB
|
|
15
|
+
- WHOIS
|
|
16
|
+
- internet measurement
|
|
17
|
+
authors:
|
|
18
|
+
- given-names: Carlos
|
|
19
|
+
family-names: Selmo
|
|
20
|
+
- given-names: Esteban
|
|
21
|
+
family-names: Carisimo
|
|
22
|
+
- given-names: FabiΓ‘n E.
|
|
23
|
+
family-names: Bustamante
|
|
24
|
+
- given-names: J. Ignacio
|
|
25
|
+
family-names: Alvarez-Hamelin
|
|
26
|
+
preferred-citation:
|
|
27
|
+
type: conference-paper
|
|
28
|
+
title: "Learning AS-to-Organization Mappings with Borges"
|
|
29
|
+
authors:
|
|
30
|
+
- given-names: Carlos
|
|
31
|
+
family-names: Selmo
|
|
32
|
+
- given-names: Esteban
|
|
33
|
+
family-names: Carisimo
|
|
34
|
+
- given-names: FabiΓ‘n E.
|
|
35
|
+
family-names: Bustamante
|
|
36
|
+
- given-names: J. Ignacio
|
|
37
|
+
family-names: Alvarez-Hamelin
|
|
38
|
+
collection-title: "Proceedings of the 2025 ACM Internet Measurement Conference"
|
|
39
|
+
conference:
|
|
40
|
+
name: "ACM Internet Measurement Conference (IMC '25)"
|
|
41
|
+
city: Madison
|
|
42
|
+
region: WI
|
|
43
|
+
country: US
|
|
44
|
+
publisher:
|
|
45
|
+
name: Association for Computing Machinery
|
|
46
|
+
doi: 10.1145/3730567.3732918
|
|
47
|
+
year: 2025
|
|
48
|
+
month: 10
|
|
49
|
+
url: "https://doi.org/10.1145/3730567.3732918"
|
borges-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Carlos Selmo, Esteban Carisimo, FabiΓ‘n E. Bustamante, J. Ignacio Alvarez-Hamelin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
borges-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: borges
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: AS-to-Organization mapping: infer sibling Autonomous Systems from WHOIS, PeeringDB and web signals (IMC 2025)
|
|
5
|
+
Project-URL: Homepage, https://nu-aqualab.github.io/borges-website/
|
|
6
|
+
Project-URL: Repository, https://github.com/NU-AquaLab/borges
|
|
7
|
+
Project-URL: Paper, https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf
|
|
8
|
+
Author: Carlos Selmo, Esteban Carisimo, Fabian E. Bustamante, J. Ignacio Alvarez-Hamelin
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: analysis,as-to-organization,as2org,autonomous-systems,network,peeringdb
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Intended Audience :: Telecommunications Industry
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
23
|
+
Classifier: Topic :: System :: Networking
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
26
|
+
Requires-Dist: click>=8.1.0
|
|
27
|
+
Requires-Dist: httpx>=0.25.0
|
|
28
|
+
Requires-Dist: langchain-community>=0.1.0
|
|
29
|
+
Requires-Dist: langchain-openai>=0.1.0
|
|
30
|
+
Requires-Dist: langchain>=0.1.0
|
|
31
|
+
Requires-Dist: openai>=1.0.0
|
|
32
|
+
Requires-Dist: pandas>=2.0.0
|
|
33
|
+
Requires-Dist: pillow>=10.0.0
|
|
34
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
35
|
+
Requires-Dist: pydantic>=2.0.0
|
|
36
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
37
|
+
Requires-Dist: pyyaml>=6.0
|
|
38
|
+
Requires-Dist: requests>=2.31.0
|
|
39
|
+
Requires-Dist: structlog>=24.0.0
|
|
40
|
+
Requires-Dist: tenacity>=8.2.0
|
|
41
|
+
Requires-Dist: tldextract>=5.0.0
|
|
42
|
+
Requires-Dist: tqdm>=4.66.0
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: black<27,>=26.0.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: ipython>=8.0.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: mypy>=1.5.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: pytest-mock>=3.11.0; extra == 'dev'
|
|
50
|
+
Requires-Dist: pytest>=7.4.0; extra == 'dev'
|
|
51
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
52
|
+
Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: types-requests>=2.31.0; extra == 'dev'
|
|
54
|
+
Description-Content-Type: text/markdown
|
|
55
|
+
|
|
56
|
+
# πΊοΈ Borges
|
|
57
|
+
|
|
58
|
+
**Borges** (*Better ORGanizations Entities mappingS*) maps Autonomous Systems (ASes) to the organizations that operate them. It finds **sibling ASes** run by the same company, even when WHOIS lists them under different organizations. It combines CAIDA's WHOIS-based AS2Org and PeeringDB with two new signals: **LLM extraction** of sibling ASNs from PeeringDB's free-text fields, and **website inference** from redirect chains, domain similarity and favicons. It is the code of the ACM IMC 2025 paper [*Learning AS-to-Organization Mappings with Borges*](https://doi.org/10.1145/3730567.3732918).
|
|
59
|
+
|
|
60
|
+
[](https://github.com/NU-AquaLab/borges/actions/workflows/ci.yml)
|
|
61
|
+
[](https://nu-aqualab.github.io/borges-website/)
|
|
62
|
+
[](https://doi.org/10.1145/3730567.3732918)
|
|
63
|
+
[](LICENSE)
|
|
64
|
+
[](https://pypi.org/project/borges/)
|
|
65
|
+
[](https://www.python.org/downloads/)
|
|
66
|
+
[](https://github.com/psf/black)
|
|
67
|
+
|
|
68
|
+
> [!NOTE]
|
|
69
|
+
> **Mappings go stale quickly.** Companies merge, rebrand and disappear. Edgio vanished
|
|
70
|
+
> between the paper's submission and its presentation. The [published artifacts](https://nu-aqualab.github.io/borges-website/)
|
|
71
|
+
> are a September 2025 snapshot. To get current mappings, run Borges on fresh PeeringDB
|
|
72
|
+
> and AS2Org data. A full run with `gpt-4o-mini` costs about **US$3**. You can also run it
|
|
73
|
+
> with **no paid API key** (see [below](#-running-without-a-paid-api-key)).
|
|
74
|
+
|
|
75
|
+
## β¨ Features
|
|
76
|
+
|
|
77
|
+
- π’ **Organization keys**: groups ASes by CAIDA AS2Org (WHOIS) organization IDs and PeeringDB organization IDs
|
|
78
|
+
- π€ **LLM sibling extraction**: few-shot prompting reads PeeringDB `notes` and `aka` fields and keeps only real sibling ASNs, not phone numbers, years or prefix limits
|
|
79
|
+
- π **Redirect analysis**: networks whose websites resolve to the same final URL are grouped together
|
|
80
|
+
- πΌοΈ **Favicon analysis**: shared favicons plus similar domains reveal common branding; a vision LLM rejects framework defaults (WordPress, Bootstrap, β¦)
|
|
81
|
+
- π§© **Consolidation**: merges every signal into network groups, with blocklists for known false bridges
|
|
82
|
+
- πΎ **Robust pipeline**: nine stages with checkpoint/resume, parallel scraping with caching, and Parquet/JSON/CSV exports
|
|
83
|
+
- πΈ **Free to run**: tests and CI need no key. The LLM stages can be skipped or pointed at a local model
|
|
84
|
+
|
|
85
|
+
## π Quick Start
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install borges # or: uv pip install borges
|
|
89
|
+
borges init # writes config.yaml (prompts, blocklists), .env and data/
|
|
90
|
+
borges download # latest PeeringDB + AS2Org snapshots into data/input/
|
|
91
|
+
borges pipeline run # needs an LLM key, or see below to skip it
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
From a clone (for development):
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
git clone https://github.com/NU-AquaLab/borges.git
|
|
98
|
+
cd borges
|
|
99
|
+
uv venv && source .venv/bin/activate
|
|
100
|
+
uv pip install -e ".[dev]"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## π Usage
|
|
104
|
+
|
|
105
|
+
### Pipeline stages
|
|
106
|
+
|
|
107
|
+
| # | Stage | What it does | LLM |
|
|
108
|
+
|---|---|---|---|
|
|
109
|
+
| 1 | `load_data` | Load PeeringDB and WHOIS (AS2Org) data | |
|
|
110
|
+
| 2 | `redirect_scraping` | Follow each PeeringDB website to its final URL (no HTML stored) | |
|
|
111
|
+
| 3 | `as_detection` | Extract sibling ASNs from `notes` and `aka` | π€ |
|
|
112
|
+
| 4 | `redirect_analysis` | Group networks that share a final URL | |
|
|
113
|
+
| 5 | `favicon_download` | Download website favicons | |
|
|
114
|
+
| 6 | `favicon_analysis` | Decide whether shared favicons mean a shared company | π€ |
|
|
115
|
+
| 7 | `whois_processing` | Group ASes by AS2Org organization | |
|
|
116
|
+
| 8 | `network_consolidation` | Merge groups from all sources | |
|
|
117
|
+
| 9 | `export_results` | Write Parquet/JSON/CSV and reports to `data/output/` | |
|
|
118
|
+
|
|
119
|
+
### Commands
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
borges pipeline list # stages and their dependencies
|
|
123
|
+
borges pipeline run # full run
|
|
124
|
+
borges pipeline run --stage redirect_scraping --stage as_detection # selected stages
|
|
125
|
+
borges pipeline run --skip favicon_analysis # skip stages
|
|
126
|
+
borges pipeline run --resume # continue from checkpoints
|
|
127
|
+
borges pipeline run --dry-run # show what would run
|
|
128
|
+
borges --config my-config.yaml pipeline run # or: export BORGES_CONFIG=...
|
|
129
|
+
borges download --peeringdb-date 2025-07-16 --as2org-date 2025-07-01
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`borges --help` lists the rest (`report generate`, `favicon download`, `config`, `version`).
|
|
133
|
+
|
|
134
|
+
### πΈ Running without a paid API key
|
|
135
|
+
|
|
136
|
+
Only stages 3 and 6 call an LLM. Pick one option:
|
|
137
|
+
|
|
138
|
+
| Option | How | Trade-off |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| **Skip the LLM stages** | `borges pipeline run --skip as_detection --skip favicon_analysis` | No sibling extraction from `notes`/`aka`, no favicon grouping |
|
|
141
|
+
| **Free local model** | Any OpenAI-compatible server: [Ollama](https://ollama.com), LM Studio, vLLM. Set `base_url` (below) | Results differ from the paper, which used `gpt-4o-mini` |
|
|
142
|
+
| **OpenAI** | `OPENAI_API_KEY=...` in `.env` | About US$3 per full run |
|
|
143
|
+
|
|
144
|
+
With Ollama (`ollama pull llama3.2` and `ollama pull llama3.2-vision`), set this in `config.yaml`:
|
|
145
|
+
|
|
146
|
+
```yaml
|
|
147
|
+
api:
|
|
148
|
+
openai:
|
|
149
|
+
api_key: unused # local servers ignore it, but it must be set
|
|
150
|
+
base_url: http://localhost:11434/v1
|
|
151
|
+
model: llama3.2
|
|
152
|
+
vision_model: llama3.2-vision
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Programmatic use
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from borges.config import load_config, set_config
|
|
159
|
+
from borges.pipeline import Pipeline
|
|
160
|
+
|
|
161
|
+
config = load_config("config.yaml")
|
|
162
|
+
set_config(config)
|
|
163
|
+
pipeline = Pipeline(config)
|
|
164
|
+
results = pipeline.run(skip_stages=["as_detection", "favicon_analysis"])
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## ποΈ Output
|
|
168
|
+
|
|
169
|
+
`export_results` writes timestamped files to `data/output/`:
|
|
170
|
+
|
|
171
|
+
| File | Contents |
|
|
172
|
+
|---|---|
|
|
173
|
+
| `autonomous_systems_*.parquet` | One row per AS |
|
|
174
|
+
| `organizations_*.parquet` | Organization groupings |
|
|
175
|
+
| `relationships_*.parquet` | Detected sibling relationships and their source |
|
|
176
|
+
| `network_groups_*.parquet` | Consolidated groups of sibling ASes |
|
|
177
|
+
| `redirect_analysis_*.parquet` | Final URL of each network's website |
|
|
178
|
+
| `network_report_*.json` | Full analysis report |
|
|
179
|
+
| `export_summary_*.json` | Export metadata and statistics |
|
|
180
|
+
|
|
181
|
+
The main settings in `config.yaml`:
|
|
182
|
+
|
|
183
|
+
| Key | Default | Purpose |
|
|
184
|
+
|---|---|---|
|
|
185
|
+
| `api.openai.model` / `vision_model` | `gpt-4o-mini` | Text and vision models |
|
|
186
|
+
| `api.openai.base_url` | *(unset: OpenAI)* | Any OpenAI-compatible endpoint |
|
|
187
|
+
| `api.openai.request_delay` / `retry_delay` | `0.5` / `30` s | Rate limiting |
|
|
188
|
+
| `scraping.html.max_workers` / `timeout` | `100` / `30` s | Redirect scraping parallelism |
|
|
189
|
+
| `scraping.favicon.max_workers` | `50` | Favicon download parallelism |
|
|
190
|
+
| `peeringdb_asn_exclusions`, blocklists | see file | Known false organizational bridges |
|
|
191
|
+
| `processing.merge_guard.enabled` | `false` | Guard against false merges (brand check, bridge guard); see [evaluation](docs/merge-guard-evaluation.md) |
|
|
192
|
+
|
|
193
|
+
## π¬ Methodology (brief)
|
|
194
|
+
|
|
195
|
+
The full method is in the [paper](https://doi.org/10.1145/3730567.3732918) ([PDF](https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf)). In short:
|
|
196
|
+
|
|
197
|
+
- **Organization keys.** Start from organization IDs in CAIDA's AS2Org (WHOIS) and in PeeringDB.
|
|
198
|
+
- **Information extraction.** Few-shot prompting of `gpt-4o-mini` (temperature 0) recovers sibling ASNs written in PeeringDB's `notes` and `aka` fields. Earlier work used regular expressions, which confuse phone numbers, years and prefix limits with ASNs.
|
|
199
|
+
- **Website inference.**
|
|
200
|
+
- *Redirects*: networks whose PeeringDB websites end at the same final URL are siblings. For example, Limelight (AS22822) and Edgecast (AS15133) both redirect to `www.edg.io`.
|
|
201
|
+
- *Favicons*: websites with the same favicon and the same subdomain are grouped. Remaining shared favicons go to a vision-LLM classifier that separates company logos (Claro Chile and Claro Puerto Rico) from framework defaults (Bootstrap, WordPress).
|
|
202
|
+
- **Consolidation.** All signals are merged into organization groups.
|
|
203
|
+
|
|
204
|
+
Results on the July 2024 PeeringDB and AS2Org snapshots:
|
|
205
|
+
|
|
206
|
+
| Measure | Result |
|
|
207
|
+
|---|---|
|
|
208
|
+
| Sibling extraction from `notes`/`aka` (320 records checked by hand) | accuracy 0.947 Β· precision 0.974 Β· recall 0.94 |
|
|
209
|
+
| Favicon company classifier (449 favicons checked by hand) | accuracy 0.986 Β· precision 0.997 Β· recall 0.984 |
|
|
210
|
+
| Organization Factor (new metric, 0 to 1) | 0.3576: +7% over AS2Org, +3.3% over as2org+ |
|
|
211
|
+
| Users newly attributed to large conglomerates | β192 million (β5% of the Internet population) |
|
|
212
|
+
|
|
213
|
+
### β οΈ Known limitations
|
|
214
|
+
|
|
215
|
+
- **No website history.** There is no longitudinal archive of the websites listed in PeeringDB, so past runs cannot be reproduced exactly from the code alone. Keep your inputs and outputs.
|
|
216
|
+
- **PeeringDB coverage is partial.** Registration is voluntary, and entries can be incomplete or outdated. For example, some Microsoft ASNs are missing.
|
|
217
|
+
- **Errors in the input propagate.** If a PeeringDB record lists the wrong sibling, Borges faithfully extracts the wrong sibling (e.g. AS10026 listing AS2706).
|
|
218
|
+
- **Layered ownership is out of scope.** Groups spanning separate brands and regions are not linked, such as AmΓ©rica MΓ³vil's Claro and A1.
|
|
219
|
+
- **False merges through weak signals.** One favicon or website shared by unrelated networks can chain them into one group. The optional [merge guard](docs/merge-guard-evaluation.md) prevents most of these. It is off by default until its results are reviewed.
|
|
220
|
+
- **Known false bridges** are handled by configuration:
|
|
221
|
+
- AS4004 appears under Orange in PeeringDB but under Sprint in WHOIS. It is listed in `peeringdb_asn_exclusions`.
|
|
222
|
+
- Shared PeeringDB listings and default favicons can wrongly join small networks to large operators. They are covered by blocklists in `config.yaml`.
|
|
223
|
+
- **LLM output varies by model.** The published numbers are for `gpt-4o-mini`. Local or newer models need their own validation.
|
|
224
|
+
|
|
225
|
+
## ποΈ Architecture
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
src/borges/
|
|
229
|
+
βββ cli.py # Click CLI: init | pipeline | report | favicon | config | version
|
|
230
|
+
βββ config.py # Pydantic config, ${ENV} interpolation, global get_config()/set_config()
|
|
231
|
+
βββ pipeline/
|
|
232
|
+
β βββ runner.py # Pipeline: stage ordering, dependencies, checkpoints
|
|
233
|
+
β βββ stages.py # the nine stages
|
|
234
|
+
βββ analyzers/ # LLM sibling extraction, redirects, WHOIS, number validation, consolidation
|
|
235
|
+
βββ scrapers/ # redirect, HTML and favicon scrapers
|
|
236
|
+
βββ data/ # loaders, processors, exporters
|
|
237
|
+
βββ models/ # AS network model and Pydantic schemas
|
|
238
|
+
βββ utils/ # LLM client (OpenAI-compatible), HTTP client, logging
|
|
239
|
+
scripts/ # download_data.py (= borges download), evaluate_merge_guard.py, check_wheel.py
|
|
240
|
+
config.yaml # default configuration, prompts and blocklists (shipped in the wheel)
|
|
241
|
+
data/reference/ # negative favicon samples (framework defaults)
|
|
242
|
+
tests/ # unit tests; no network, no API key
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
## π§ͺ Development
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
uv pip install -e ".[dev]"
|
|
249
|
+
ruff check src tests scripts --select E9,F63,F7,F82 # syntax errors and undefined names
|
|
250
|
+
black --check src tests # formatting (black 26)
|
|
251
|
+
mypy src/borges # informational for now
|
|
252
|
+
pytest # no network, no API key
|
|
253
|
+
git config blame.ignoreRevsFile .git-blame-ignore-revs # hide the bulk reformat from blame
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
|
|
257
|
+
|
|
258
|
+
## π Example Output
|
|
259
|
+
|
|
260
|
+
```
|
|
261
|
+
$ borges pipeline run --skip as_detection --skip favicon_analysis --dry-run
|
|
262
|
+
Pipeline dry run - stages that would be executed:
|
|
263
|
+
β load_data: Load initial data from PeeringDB and WHOIS.
|
|
264
|
+
β redirect_scraping: Scrape redirect information from websites (no HTML content).
|
|
265
|
+
β redirect_analysis: Analyze URL redirects.
|
|
266
|
+
β favicon_download: Scrape favicons from websites.
|
|
267
|
+
β whois_processing: Process WHOIS data.
|
|
268
|
+
β network_consolidation: Consolidate network groups from different analysis sources.
|
|
269
|
+
β export_results: Export final results.
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## π€ Contributing
|
|
273
|
+
|
|
274
|
+
Bug fixes, documentation, tests and new signals are welcome. Please open an issue before large changes. See [CONTRIBUTING.md](CONTRIBUTING.md), the [Code of Conduct](CODE_OF_CONDUCT.md), and [SECURITY.md](SECURITY.md) to report a vulnerability privately.
|
|
275
|
+
|
|
276
|
+
1. Fork the repository
|
|
277
|
+
2. Create a feature branch (`git checkout -b fix/my-fix`)
|
|
278
|
+
3. Commit your changes (`git commit -m 'Fix β¦'`)
|
|
279
|
+
4. Push to the branch (`git push origin fix/my-fix`)
|
|
280
|
+
5. Open a Pull Request
|
|
281
|
+
|
|
282
|
+
## π License
|
|
283
|
+
|
|
284
|
+
[MIT](LICENSE).
|
|
285
|
+
|
|
286
|
+
## π Citation
|
|
287
|
+
|
|
288
|
+
If you use Borges, please cite the paper. It is also in [CITATION.cff](CITATION.cff), which feeds GitHub's **Cite this repository** button:
|
|
289
|
+
|
|
290
|
+
```bibtex
|
|
291
|
+
@inproceedings{borges:imc,
|
|
292
|
+
author = {Selmo, Carlos and Carisimo, Esteban and Bustamante, Fabi{\'a}n E. and Alvarez-Hamelin, J. Ignacio},
|
|
293
|
+
title = {Learning AS-to-Organization Mappings with Borges},
|
|
294
|
+
booktitle = {Proceedings of the 2025 ACM Internet Measurement Conference},
|
|
295
|
+
series = {IMC '25},
|
|
296
|
+
year = {2025},
|
|
297
|
+
month = {10},
|
|
298
|
+
location = {Madison, WI, USA},
|
|
299
|
+
publisher = {Association for Computing Machinery},
|
|
300
|
+
doi = {10.1145/3730567.3732918},
|
|
301
|
+
url = {https://doi.org/10.1145/3730567.3732918}
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
## π Related Resources
|
|
306
|
+
|
|
307
|
+
- **Paper**: [ACM Digital Library](https://doi.org/10.1145/3730567.3732918) Β· [PDF](https://estcarisimo.github.io/assets/pdf/papers/2025-IMC-borges.pdf)
|
|
308
|
+
- **Website and artifacts**: [nu-aqualab.github.io/borges-website](https://nu-aqualab.github.io/borges-website/) (September 2025 mappings)
|
|
309
|
+
- **Press**: [Mapping Who Really Runs the Internet: Introducing Borges](https://pulse.internetsociety.org/blog/mapping-who-really-runs-the-internet-introducing-borges) (Internet Society Pulse)
|
|
310
|
+
- **CAIDA AS2Org**: [AS Organizations dataset](https://www.caida.org/catalog/datasets/as-organizations/) and the [PeeringDB archive](https://publicdata.caida.org/datasets/peeringdb/)
|
|
311
|
+
- **as2org+**: Arturi, Carisimo and Bustamante, *as2org+: Enriching AS-to-Organization Mappings with PeeringDB*, PAM 2023
|
|
312
|
+
- **State-Owned ASes**: [estcarisimo/state-owned-ases](https://github.com/estcarisimo/state-owned-ases), a hand-built dataset where Borges-style automation could help
|
|
313
|
+
|
|
314
|
+
## π Acknowledgements
|
|
315
|
+
|
|
316
|
+
- **Paper**: *Learning AS-to-Organization Mappings with Borges*
|
|
317
|
+
- **Authors**: Carlos Selmo, Esteban Carisimo, FabiΓ‘n E. Bustamante, J. Ignacio Alvarez-Hamelin
|
|
318
|
+
- **Conference**: ACM Internet Measurement Conference (IMC) 2025, Madison, WI, USA
|
|
319
|
+
- **Funding**: NSF grant CNS-2107392 and UBACyT 20020220100053BA
|
|
320
|
+
|
|
321
|
+
Thanks to [CAIDA](https://www.caida.org/) for archiving PeeringDB snapshots and publishing AS2Org, to [PeeringDB](https://www.peeringdb.com), and to [@zhiyichenGT](https://github.com/zhiyichenGT) for the consolidator and WHOIS fixes.
|