iporigin 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iporigin/__init__.py +51 -0
- iporigin/_data.py +112 -0
- iporigin/cli.py +83 -0
- iporigin/core.py +116 -0
- iporigin/data/ranges.bin +0 -0
- iporigin/online.py +65 -0
- iporigin/py.typed +0 -0
- iporigin-1.0.0.dist-info/METADATA +247 -0
- iporigin-1.0.0.dist-info/RECORD +13 -0
- iporigin-1.0.0.dist-info/WHEEL +5 -0
- iporigin-1.0.0.dist-info/entry_points.txt +2 -0
- iporigin-1.0.0.dist-info/licenses/LICENSE +21 -0
- iporigin-1.0.0.dist-info/top_level.txt +1 -0
iporigin/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Tell a datacenter IP from a home connection, offline.
|
|
2
|
+
|
|
3
|
+
>>> import iporigin
|
|
4
|
+
>>> iporigin.classify("52.95.110.1")
|
|
5
|
+
Origin(ip='52.95.110.1', kind='hosting', provider='Amazon AWS', source='local')
|
|
6
|
+
>>> iporigin.is_datacenter("8.8.8.8")
|
|
7
|
+
True
|
|
8
|
+
|
|
9
|
+
No network calls, no API key: the answer comes from a table compiled from
|
|
10
|
+
the providers' own published feeds. See iporigin.online for the optional
|
|
11
|
+
live lookup that also covers consumer VPN exit nodes.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from ._data import DatasetError
|
|
15
|
+
from .core import (
|
|
16
|
+
ANONYMIZER_KINDS,
|
|
17
|
+
BOT,
|
|
18
|
+
CDN,
|
|
19
|
+
DATACENTER_KINDS,
|
|
20
|
+
HOSTING,
|
|
21
|
+
RESERVED,
|
|
22
|
+
TOR,
|
|
23
|
+
UNKNOWN,
|
|
24
|
+
VPN,
|
|
25
|
+
Origin,
|
|
26
|
+
classify,
|
|
27
|
+
classify_many,
|
|
28
|
+
dataset_info,
|
|
29
|
+
is_datacenter,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__version__ = "1.0.0"
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"ANONYMIZER_KINDS",
|
|
36
|
+
"BOT",
|
|
37
|
+
"CDN",
|
|
38
|
+
"DATACENTER_KINDS",
|
|
39
|
+
"DatasetError",
|
|
40
|
+
"HOSTING",
|
|
41
|
+
"Origin",
|
|
42
|
+
"RESERVED",
|
|
43
|
+
"TOR",
|
|
44
|
+
"UNKNOWN",
|
|
45
|
+
"VPN",
|
|
46
|
+
"classify",
|
|
47
|
+
"classify_many",
|
|
48
|
+
"dataset_info",
|
|
49
|
+
"is_datacenter",
|
|
50
|
+
"__version__",
|
|
51
|
+
]
|
iporigin/_data.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Loading and querying the packed range table.
|
|
2
|
+
|
|
3
|
+
The table is loaded once, lazily, on the first lookup — importing iporigin
|
|
4
|
+
must stay cheap for a program that imports it and never calls it.
|
|
5
|
+
|
|
6
|
+
Lookups are a bisect over a sorted column of range starts. Ranges never
|
|
7
|
+
overlap within the merged table, so the candidate found by bisect is the
|
|
8
|
+
only one that can contain the address: one comparison decides it.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import bisect
|
|
12
|
+
import struct
|
|
13
|
+
from array import array
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
MAGIC = b"IPORIGIN"
|
|
17
|
+
SUPPORTED_VERSION = 1
|
|
18
|
+
|
|
19
|
+
DATA_FILE = Path(__file__).parent / "data" / "ranges.bin"
|
|
20
|
+
|
|
21
|
+
_HEADER = struct.Struct("<8sHQHII")
|
|
22
|
+
_V4_RECORD = struct.Struct("<IIH")
|
|
23
|
+
_V6_RECORD = struct.Struct("<16s16sH")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DatasetError(RuntimeError):
|
|
27
|
+
"""The bundled dataset is missing or unreadable."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _Dataset:
|
|
31
|
+
__slots__ = (
|
|
32
|
+
"built_at", "labels",
|
|
33
|
+
"v4_starts", "v4_ends", "v4_labels",
|
|
34
|
+
"v6_starts", "v6_ends", "v6_labels",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def __init__(self, blob):
|
|
38
|
+
magic, version, built_at, n_labels, n_v4, n_v6 = _HEADER.unpack_from(blob, 0)
|
|
39
|
+
if magic != MAGIC:
|
|
40
|
+
raise DatasetError("not an iporigin dataset")
|
|
41
|
+
if version != SUPPORTED_VERSION:
|
|
42
|
+
raise DatasetError(
|
|
43
|
+
"dataset format v%d, this build understands v%d — upgrade iporigin"
|
|
44
|
+
% (version, SUPPORTED_VERSION)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
self.built_at = built_at
|
|
48
|
+
offset = _HEADER.size
|
|
49
|
+
|
|
50
|
+
self.labels = []
|
|
51
|
+
for _ in range(n_labels):
|
|
52
|
+
(length,) = struct.unpack_from("<H", blob, offset)
|
|
53
|
+
offset += 2
|
|
54
|
+
provider, _, kind = blob[offset:offset + length].decode("utf-8").partition("\t")
|
|
55
|
+
self.labels.append((provider, kind))
|
|
56
|
+
offset += length
|
|
57
|
+
|
|
58
|
+
# Parallel arrays rather than tuples: 21k ranges as Python tuples is
|
|
59
|
+
# several MB of objects, as arrays it is a few hundred KB.
|
|
60
|
+
self.v4_starts = array("L")
|
|
61
|
+
self.v4_ends = array("L")
|
|
62
|
+
self.v4_labels = array("H")
|
|
63
|
+
for _ in range(n_v4):
|
|
64
|
+
start, end, label = _V4_RECORD.unpack_from(blob, offset)
|
|
65
|
+
self.v4_starts.append(start)
|
|
66
|
+
self.v4_ends.append(end)
|
|
67
|
+
self.v4_labels.append(label)
|
|
68
|
+
offset += _V4_RECORD.size
|
|
69
|
+
|
|
70
|
+
# IPv6 addresses do not fit a machine word, so these stay Python ints.
|
|
71
|
+
self.v6_starts = []
|
|
72
|
+
self.v6_ends = []
|
|
73
|
+
self.v6_labels = array("H")
|
|
74
|
+
for _ in range(n_v6):
|
|
75
|
+
start, end, label = _V6_RECORD.unpack_from(blob, offset)
|
|
76
|
+
self.v6_starts.append(int.from_bytes(start, "big"))
|
|
77
|
+
self.v6_ends.append(int.from_bytes(end, "big"))
|
|
78
|
+
self.v6_labels.append(label)
|
|
79
|
+
offset += _V6_RECORD.size
|
|
80
|
+
|
|
81
|
+
def find(self, value, version):
|
|
82
|
+
"""Return (provider, kind) for an address, or None."""
|
|
83
|
+
if version == 4:
|
|
84
|
+
starts, ends, labels = self.v4_starts, self.v4_ends, self.v4_labels
|
|
85
|
+
else:
|
|
86
|
+
starts, ends, labels = self.v6_starts, self.v6_ends, self.v6_labels
|
|
87
|
+
|
|
88
|
+
index = bisect.bisect_right(starts, value) - 1
|
|
89
|
+
if index < 0 or value > ends[index]:
|
|
90
|
+
return None
|
|
91
|
+
return self.labels[labels[index]]
|
|
92
|
+
|
|
93
|
+
def __len__(self):
|
|
94
|
+
return len(self.v4_starts) + len(self.v6_starts)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
_loaded = None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def dataset():
|
|
101
|
+
global _loaded
|
|
102
|
+
if _loaded is None:
|
|
103
|
+
try:
|
|
104
|
+
blob = DATA_FILE.read_bytes()
|
|
105
|
+
except OSError as exc:
|
|
106
|
+
raise DatasetError(
|
|
107
|
+
"bundled dataset missing at %s — reinstall iporigin, or run "
|
|
108
|
+
"tools/build_dataset.py if you are working from a checkout"
|
|
109
|
+
% DATA_FILE
|
|
110
|
+
) from exc
|
|
111
|
+
_loaded = _Dataset(blob)
|
|
112
|
+
return _loaded
|
iporigin/cli.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Command line front end: iporigin 8.8.8.8"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from . import __version__, classify, dataset_info
|
|
8
|
+
from .core import DatasetError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _read_addresses(args):
|
|
12
|
+
if args.ip:
|
|
13
|
+
return args.ip
|
|
14
|
+
# No arguments: read from stdin, so `cut -f1 access.log | iporigin` works.
|
|
15
|
+
return (line.strip() for line in sys.stdin if line.strip())
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main(argv=None):
|
|
19
|
+
parser = argparse.ArgumentParser(
|
|
20
|
+
prog="iporigin",
|
|
21
|
+
description="Classify IP addresses as hosting, CDN, VPN or unknown.",
|
|
22
|
+
)
|
|
23
|
+
parser.add_argument("ip", nargs="*", help="addresses to classify (default: stdin)")
|
|
24
|
+
parser.add_argument("--json", action="store_true", help="output JSON lines")
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"--online",
|
|
27
|
+
action="store_true",
|
|
28
|
+
help="also query the free Unblock Master API (covers consumer VPNs)",
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--datacenter-only",
|
|
32
|
+
action="store_true",
|
|
33
|
+
help="print only addresses that are in a datacenter range",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument("--info", action="store_true", help="show dataset provenance")
|
|
36
|
+
parser.add_argument("--version", action="version", version="iporigin " + __version__)
|
|
37
|
+
args = parser.parse_args(argv)
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
if args.info:
|
|
41
|
+
print(json.dumps(dataset_info(), indent=2, sort_keys=True))
|
|
42
|
+
return 0
|
|
43
|
+
|
|
44
|
+
lookup = classify
|
|
45
|
+
if args.online:
|
|
46
|
+
from .online import classify_online as lookup # noqa: N813
|
|
47
|
+
|
|
48
|
+
exit_code = 0
|
|
49
|
+
for raw in _read_addresses(args):
|
|
50
|
+
try:
|
|
51
|
+
origin = lookup(raw)
|
|
52
|
+
except ValueError as exc:
|
|
53
|
+
print("%s: %s" % (raw, exc), file=sys.stderr)
|
|
54
|
+
exit_code = 2
|
|
55
|
+
continue
|
|
56
|
+
|
|
57
|
+
if args.datacenter_only and not origin.is_datacenter:
|
|
58
|
+
continue
|
|
59
|
+
|
|
60
|
+
if args.json:
|
|
61
|
+
print(json.dumps({
|
|
62
|
+
"ip": origin.ip,
|
|
63
|
+
"kind": origin.kind,
|
|
64
|
+
"provider": origin.provider,
|
|
65
|
+
"source": origin.source,
|
|
66
|
+
"is_datacenter": origin.is_datacenter,
|
|
67
|
+
}))
|
|
68
|
+
else:
|
|
69
|
+
print("%-40s %-9s %s" % (origin.ip, origin.kind, origin.provider))
|
|
70
|
+
return exit_code
|
|
71
|
+
|
|
72
|
+
except DatasetError as exc:
|
|
73
|
+
print("iporigin: %s" % exc, file=sys.stderr)
|
|
74
|
+
return 1
|
|
75
|
+
except BrokenPipeError:
|
|
76
|
+
# `iporigin < big.txt | head` is a normal thing to do.
|
|
77
|
+
return 0
|
|
78
|
+
except KeyboardInterrupt:
|
|
79
|
+
return 130
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
raise SystemExit(main())
|
iporigin/core.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Classify an IP address by where it is hosted."""
|
|
2
|
+
|
|
3
|
+
import ipaddress
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from ._data import DatasetError, dataset # noqa: F401 — re-exported
|
|
7
|
+
|
|
8
|
+
# What a result can be. Deliberately small: a caller should be able to
|
|
9
|
+
# switch on this exhaustively.
|
|
10
|
+
HOSTING = "hosting" # a machine in a cloud or hosting provider
|
|
11
|
+
CDN = "cdn" # edge infrastructure fronting other people's sites
|
|
12
|
+
VPN = "vpn" # a consumer VPN or private relay exit
|
|
13
|
+
TOR = "tor" # a Tor exit node
|
|
14
|
+
BOT = "bot" # a declared crawler (Googlebot, GPTBot, ...)
|
|
15
|
+
RESERVED = "reserved" # private, loopback, link-local, documentation...
|
|
16
|
+
UNKNOWN = "unknown" # in no list we have — most often a residential ISP
|
|
17
|
+
|
|
18
|
+
#: Kinds that mean "this address is a machine, not somebody's home line".
|
|
19
|
+
#: A person browsing through Mullvad is at home, but the address they arrive
|
|
20
|
+
#: from is still a server — this is about the address, not the human.
|
|
21
|
+
DATACENTER_KINDS = frozenset({HOSTING, CDN, VPN, TOR, BOT})
|
|
22
|
+
|
|
23
|
+
#: Kinds that mean "whoever is behind this is deliberately hidden".
|
|
24
|
+
#: A different decision from DATACENTER_KINDS: you might rate-limit a cloud
|
|
25
|
+
#: IP but refuse a payment from an anonymised one, or the exact reverse.
|
|
26
|
+
ANONYMIZER_KINDS = frozenset({VPN, TOR})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Origin:
|
|
31
|
+
"""What we know about an address.
|
|
32
|
+
|
|
33
|
+
``kind`` is always set. ``provider`` is empty when we did not match a
|
|
34
|
+
known range — absence of evidence, so do not read UNKNOWN as
|
|
35
|
+
"residential", only as "not in our data".
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
ip: str
|
|
39
|
+
kind: str
|
|
40
|
+
provider: str = ""
|
|
41
|
+
source: str = "local"
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def is_datacenter(self):
|
|
45
|
+
return self.kind in DATACENTER_KINDS
|
|
46
|
+
|
|
47
|
+
def __bool__(self):
|
|
48
|
+
# Guard against `if classify(ip):` reading as "is a datacenter".
|
|
49
|
+
# An Origin is always a result, so make the truthiness meaningless
|
|
50
|
+
# rather than misleading.
|
|
51
|
+
return True
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse(ip):
|
|
55
|
+
if isinstance(ip, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
|
|
56
|
+
return ip
|
|
57
|
+
try:
|
|
58
|
+
return ipaddress.ip_address(str(ip).strip())
|
|
59
|
+
except ValueError as exc:
|
|
60
|
+
raise ValueError("%r is not an IP address" % (ip,)) from exc
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def classify(ip):
|
|
64
|
+
"""Classify one address against the bundled dataset.
|
|
65
|
+
|
|
66
|
+
>>> classify("1.1.1.1").provider
|
|
67
|
+
'Cloudflare'
|
|
68
|
+
|
|
69
|
+
Raises ValueError if ``ip`` is not an address. Never makes a network
|
|
70
|
+
request — see iporigin.online for that.
|
|
71
|
+
"""
|
|
72
|
+
address = _parse(ip)
|
|
73
|
+
|
|
74
|
+
# A reserved address is not "unknown", and answering "hosting: no" for
|
|
75
|
+
# 127.0.0.1 would be technically true and completely useless.
|
|
76
|
+
if not address.is_global:
|
|
77
|
+
return Origin(str(address), RESERVED)
|
|
78
|
+
|
|
79
|
+
found = dataset().find(int(address), address.version)
|
|
80
|
+
if found is None:
|
|
81
|
+
return Origin(str(address), UNKNOWN)
|
|
82
|
+
|
|
83
|
+
provider, kind = found
|
|
84
|
+
return Origin(str(address), kind, provider)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def is_datacenter(ip):
|
|
88
|
+
"""True when the address belongs to a known hosting, CDN or VPN range.
|
|
89
|
+
|
|
90
|
+
False means "not in our data", which is not the same as "residential" —
|
|
91
|
+
our coverage is the published feeds listed in tools/sources.py.
|
|
92
|
+
"""
|
|
93
|
+
return classify(ip).is_datacenter
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def classify_many(ips):
|
|
97
|
+
"""Classify an iterable of addresses, yielding Origin objects.
|
|
98
|
+
|
|
99
|
+
Loads the dataset once for the whole batch, so this is the right call
|
|
100
|
+
for scanning a log file.
|
|
101
|
+
"""
|
|
102
|
+
dataset()
|
|
103
|
+
for ip in ips:
|
|
104
|
+
yield classify(ip)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def dataset_info():
|
|
108
|
+
"""Provenance of the bundled data: when it was built and how big it is."""
|
|
109
|
+
data = dataset()
|
|
110
|
+
return {
|
|
111
|
+
"built_at": data.built_at,
|
|
112
|
+
"ranges": len(data),
|
|
113
|
+
"ipv4_ranges": len(data.v4_starts),
|
|
114
|
+
"ipv6_ranges": len(data.v6_starts),
|
|
115
|
+
"providers": sorted({provider for provider, _ in data.labels}),
|
|
116
|
+
}
|
iporigin/data/ranges.bin
ADDED
|
Binary file
|
iporigin/online.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Optional live lookup, for what the bundled dataset cannot know.
|
|
2
|
+
|
|
3
|
+
The offline dataset covers published hosting and CDN ranges. It does not
|
|
4
|
+
cover consumer VPN exit nodes, because no provider publishes those. When
|
|
5
|
+
that matters, this asks the free Unblock Master API, which does its own
|
|
6
|
+
detection:
|
|
7
|
+
|
|
8
|
+
from iporigin.online import classify_online
|
|
9
|
+
classify_online("203.0.113.10")
|
|
10
|
+
|
|
11
|
+
Nothing else in iporigin makes a network request; you have to call this
|
|
12
|
+
module explicitly. It needs no API key.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import urllib.error
|
|
17
|
+
import urllib.request
|
|
18
|
+
|
|
19
|
+
from .core import HOSTING, UNKNOWN, VPN, Origin, _parse, classify
|
|
20
|
+
|
|
21
|
+
API_URL = "https://www.unblockmaster.com/api/v1/ip/%s"
|
|
22
|
+
USER_AGENT = "iporigin/1.0 (+https://github.com/Yuix-Networks/iporigin)"
|
|
23
|
+
DEFAULT_TIMEOUT = 5
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LookupError_(RuntimeError):
|
|
27
|
+
"""The online lookup could not be completed."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def classify_online(ip, timeout=DEFAULT_TIMEOUT, fall_back=True):
|
|
31
|
+
"""Classify an address using the live API.
|
|
32
|
+
|
|
33
|
+
``fall_back`` returns the offline answer when the request fails, which
|
|
34
|
+
is usually what you want in a request path. Pass False to get an
|
|
35
|
+
exception instead, when a wrong answer is worse than no answer.
|
|
36
|
+
"""
|
|
37
|
+
address = _parse(ip)
|
|
38
|
+
offline = classify(address)
|
|
39
|
+
if offline.kind == "reserved":
|
|
40
|
+
return offline
|
|
41
|
+
|
|
42
|
+
request = urllib.request.Request(
|
|
43
|
+
API_URL % address, headers={"User-Agent": USER_AGENT}
|
|
44
|
+
)
|
|
45
|
+
try:
|
|
46
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
47
|
+
payload = json.loads(response.read().decode("utf-8"))
|
|
48
|
+
except (urllib.error.URLError, OSError, ValueError, TimeoutError) as exc:
|
|
49
|
+
if fall_back:
|
|
50
|
+
return offline
|
|
51
|
+
raise LookupError_("live lookup failed for %s: %s" % (address, exc)) from exc
|
|
52
|
+
|
|
53
|
+
# The API answers null when it could not determine the address, which is
|
|
54
|
+
# not the same as False. Only overrule the offline answer on a real yes.
|
|
55
|
+
if payload.get("is_vpn") is True:
|
|
56
|
+
# A provider name we already matched locally is more specific than
|
|
57
|
+
# anything the API returns, so keep it.
|
|
58
|
+
provider = offline.provider or payload.get("provider") or ""
|
|
59
|
+
kind = offline.kind if offline.kind in (HOSTING, "cdn") else VPN
|
|
60
|
+
return Origin(str(address), kind, provider, source="api")
|
|
61
|
+
|
|
62
|
+
if payload.get("is_vpn") is False and offline.kind == UNKNOWN:
|
|
63
|
+
return Origin(str(address), UNKNOWN, payload.get("provider") or "", source="api")
|
|
64
|
+
|
|
65
|
+
return offline
|
iporigin/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: iporigin
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Is this IP a datacenter, VPN, Tor exit or a home connection? Offline lookup, no API key.
|
|
5
|
+
Author-email: Yuix Networks <info@yuix.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://www.unblockmaster.com/free-ip-api/
|
|
8
|
+
Project-URL: Source, https://github.com/Yuix-Networks/iporigin
|
|
9
|
+
Project-URL: Issues, https://github.com/Yuix-Networks/iporigin/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Yuix-Networks/iporigin/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: ip,geolocation,datacenter,hosting,vpn,proxy,cloud,cidr,abuse,fraud,bot-detection,tor,hetzner,ovh
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Internet
|
|
25
|
+
Classifier: Topic :: Security
|
|
26
|
+
Classifier: Topic :: System :: Networking
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.8
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# iporigin
|
|
36
|
+
|
|
37
|
+
Tell a datacenter IP from a home connection — offline, with no API key.
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
>>> import iporigin
|
|
41
|
+
>>> iporigin.classify("52.95.110.1")
|
|
42
|
+
Origin(ip='52.95.110.1', kind='hosting', provider='Amazon AWS', source='local')
|
|
43
|
+
>>> iporigin.is_datacenter("8.8.8.8")
|
|
44
|
+
True
|
|
45
|
+
>>> iporigin.is_datacenter("127.0.0.1")
|
|
46
|
+
False
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
No network calls. No signup. No runtime dependencies. The answer comes from
|
|
50
|
+
a bundled table of 33,000 ranges covering 37 hosting providers, CDNs,
|
|
51
|
+
consumer VPNs, Tor and declared crawlers.
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
pip install iporigin
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Why
|
|
60
|
+
|
|
61
|
+
"Is this visitor on a server or on a home connection?" comes up constantly —
|
|
62
|
+
scoring signups, filtering scrapers out of analytics, deciding whether an
|
|
63
|
+
abuse report is worth acting on, flagging logins from hosting ranges. The
|
|
64
|
+
usual answers are a paid API or a hand-maintained list of CIDRs that goes
|
|
65
|
+
stale in a month.
|
|
66
|
+
|
|
67
|
+
This is a third option: the providers publish their own ranges, so the list
|
|
68
|
+
can be compiled from source and rebuilt on a schedule. You get a local
|
|
69
|
+
lookup in microseconds, and you can verify every byte of the data by
|
|
70
|
+
re-running the build script.
|
|
71
|
+
|
|
72
|
+
## Use
|
|
73
|
+
|
|
74
|
+
### Classify one address
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import iporigin
|
|
78
|
+
|
|
79
|
+
origin = iporigin.classify("140.82.121.4")
|
|
80
|
+
origin.kind # 'hosting'
|
|
81
|
+
origin.provider # 'GitHub'
|
|
82
|
+
origin.is_datacenter # True
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`kind` is one of:
|
|
86
|
+
|
|
87
|
+
| kind | meaning | examples |
|
|
88
|
+
| --- | --- | --- |
|
|
89
|
+
| `hosting` | a machine in a cloud or hosting provider | AWS, Hetzner, OVHcloud |
|
|
90
|
+
| `cdn` | edge infrastructure fronting other people's sites | Cloudflare, Akamai |
|
|
91
|
+
| `vpn` | a consumer VPN or private relay exit | Mullvad, ProtonVPN, Apple Private Relay |
|
|
92
|
+
| `tor` | a Tor exit node | |
|
|
93
|
+
| `bot` | a declared crawler | Googlebot, GPTBot, ClaudeBot |
|
|
94
|
+
| `reserved` | private, loopback, link-local, documentation | `127.0.0.1`, `10.0.0.0/8` |
|
|
95
|
+
| `unknown` | in none of our lists — most often a residential ISP | |
|
|
96
|
+
|
|
97
|
+
Two sets are exported for the common decisions:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
origin.kind in iporigin.DATACENTER_KINDS # a machine, not a home line
|
|
101
|
+
origin.kind in iporigin.ANONYMIZER_KINDS # vpn or tor
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
They are deliberately different questions. Someone arriving through Mullvad
|
|
105
|
+
is a person at home, but the address they arrive *from* is a server — so it
|
|
106
|
+
is in both sets, and you may well want to rate-limit one and refuse the
|
|
107
|
+
other.
|
|
108
|
+
|
|
109
|
+
`unknown` means *absence of evidence*. It is not a positive claim that the
|
|
110
|
+
address is residential, and the difference matters if you are going to block
|
|
111
|
+
someone over it.
|
|
112
|
+
|
|
113
|
+
### Scan a lot of them
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
for origin in iporigin.classify_many(ip_list):
|
|
117
|
+
if origin.is_datacenter:
|
|
118
|
+
print(origin.ip, origin.provider)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`classify_many` loads the table once for the whole batch.
|
|
122
|
+
|
|
123
|
+
### From the shell
|
|
124
|
+
|
|
125
|
+
```console
|
|
126
|
+
$ iporigin 8.8.8.8 140.82.121.4 192.168.1.1
|
|
127
|
+
8.8.8.8 hosting Google
|
|
128
|
+
140.82.121.4 hosting GitHub
|
|
129
|
+
192.168.1.1 reserved
|
|
130
|
+
|
|
131
|
+
$ cut -d' ' -f1 access.log | iporigin --datacenter-only --json
|
|
132
|
+
{"ip": "34.82.1.5", "kind": "hosting", "provider": "Google Cloud", ...}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
`iporigin --info` prints what is in the bundled dataset and when it was built.
|
|
136
|
+
|
|
137
|
+
### Going beyond the bundled table
|
|
138
|
+
|
|
139
|
+
The table covers the VPN providers whose exits are publicly tracked —
|
|
140
|
+
Mullvad, ProtonVPN, Apple Private Relay. Most VPN companies are not in that
|
|
141
|
+
set. When you need broader coverage, `iporigin.online` asks the free
|
|
142
|
+
[Unblock Master IP API](https://www.unblockmaster.com/free-ip-api/), which
|
|
143
|
+
does its own detection:
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
from iporigin.online import classify_online
|
|
147
|
+
|
|
148
|
+
classify_online("203.0.113.10") # may return kind='vpn'
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
It falls back to the offline answer if the request fails, so it is safe in a
|
|
152
|
+
request path. Nothing else in the library touches the network — you have to
|
|
153
|
+
import this module on purpose. No key required.
|
|
154
|
+
|
|
155
|
+
## What is in the dataset
|
|
156
|
+
|
|
157
|
+
37 providers, in two tiers.
|
|
158
|
+
|
|
159
|
+
**Published by the provider.** The authoritative tier — each of these is the
|
|
160
|
+
company's own feed, fetched at build time:
|
|
161
|
+
|
|
162
|
+
| Provider | Feed |
|
|
163
|
+
| --- | --- |
|
|
164
|
+
| Amazon AWS | `ip-ranges.amazonaws.com/ip-ranges.json` |
|
|
165
|
+
| Google, Google Cloud | `gstatic.com/ipranges/goog.json`, `cloud.json` |
|
|
166
|
+
| Microsoft Azure | Service Tags JSON |
|
|
167
|
+
| DigitalOcean | `digitalocean.com/geo/google.csv` |
|
|
168
|
+
| Linode | RFC 8805 geofeed |
|
|
169
|
+
| Vultr | `geofeed.constant.com` |
|
|
170
|
+
| Oracle Cloud | `public_ip_ranges.json` |
|
|
171
|
+
| GitHub | `api.github.com/meta` |
|
|
172
|
+
| Cloudflare | `cloudflare.com/ips-v4`, `ips-v6` |
|
|
173
|
+
| Fastly | `api.fastly.com/public-ip-list` |
|
|
174
|
+
|
|
175
|
+
**Community-maintained lists.** Some providers publish nothing
|
|
176
|
+
machine-readable — Hetzner and OVH being the two that matter most, since a
|
|
177
|
+
large share of abusive traffic comes from them. Consumer VPN exits, Tor and
|
|
178
|
+
crawler ranges have the same problem for a different reason: nobody with the
|
|
179
|
+
data has an interest in publishing it. Those come from two community repos,
|
|
180
|
+
and are second-hand by definition:
|
|
181
|
+
|
|
182
|
+
- [`rezmoss/cloud-provider-ip-addresses`](https://github.com/rezmoss/cloud-provider-ip-addresses) (CC0)
|
|
183
|
+
- [`lord-alfred/ipranges`](https://github.com/lord-alfred/ipranges) (CC0)
|
|
184
|
+
|
|
185
|
+
Covering Hetzner, OVHcloud, Scaleway, Alibaba Cloud, Leaseweb, UpCloud, IBM
|
|
186
|
+
Cloud, Huawei Cloud, Tencent Cloud, Rackspace, Akamai, Gcore, Mullvad,
|
|
187
|
+
ProtonVPN, Apple Private Relay, Tor, and ten declared crawlers.
|
|
188
|
+
|
|
189
|
+
Both are CC0, which is why these two and not the half-dozen other repos
|
|
190
|
+
covering the same ground. Redistributing an unlicensed list inside an MIT
|
|
191
|
+
package is not something a dependency should ask of the people who install
|
|
192
|
+
it.
|
|
193
|
+
|
|
194
|
+
About 449,000 published prefixes collapse into 33,647 disjoint ranges
|
|
195
|
+
(17,169 IPv4, 16,478 IPv6). Rebuild it yourself at any time:
|
|
196
|
+
|
|
197
|
+
```
|
|
198
|
+
python tools/build_dataset.py
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
A GitHub Action re-runs that weekly and opens a PR when the ranges move.
|
|
202
|
+
|
|
203
|
+
### Known gaps
|
|
204
|
+
|
|
205
|
+
Being explicit about these is more useful than pretending they are not there:
|
|
206
|
+
|
|
207
|
+
- **Most consumer VPNs.** Only the ones whose exits are publicly tracked are
|
|
208
|
+
in the table. Use `iporigin.online` for the rest.
|
|
209
|
+
- **Some provider-owned addresses** sit outside the ranges the provider
|
|
210
|
+
publishes. `1.1.1.1` is Cloudflare's resolver but is not in Cloudflare's
|
|
211
|
+
published edge list, so it comes back `unknown`.
|
|
212
|
+
- **Second-hand data is second-hand.** The community tier is as good as
|
|
213
|
+
those repos are, and they are not the provider speaking.
|
|
214
|
+
- The data is **as accurate as the feeds**. A range reassigned yesterday is
|
|
215
|
+
wrong until the next rebuild.
|
|
216
|
+
|
|
217
|
+
## How the lookup works
|
|
218
|
+
|
|
219
|
+
Ranges are stored as inclusive integer start/end pairs in sorted, *disjoint*
|
|
220
|
+
order, so a lookup is one `bisect` plus one comparison.
|
|
221
|
+
|
|
222
|
+
Making them disjoint is the part that matters. Feeds overlap each other —
|
|
223
|
+
GitHub runs on Azure and AWS, so its prefixes sit inside theirs. A bisect
|
|
224
|
+
inspects exactly one candidate, and with overlapping ranges that candidate
|
|
225
|
+
can be a narrow range that ends before the address while a wider range still
|
|
226
|
+
contains it, which returns `unknown` for an address plainly in the table. The
|
|
227
|
+
build script therefore sweeps the ranges into a disjoint partition, and where
|
|
228
|
+
they overlap the narrowest one wins — GitHub inside Azure answers GitHub,
|
|
229
|
+
which is the more specific truth.
|
|
230
|
+
|
|
231
|
+
The table loads lazily on the first lookup, so importing the library and
|
|
232
|
+
never calling it costs nothing.
|
|
233
|
+
|
|
234
|
+
## Compatibility
|
|
235
|
+
|
|
236
|
+
Python 3.8+. No dependencies.
|
|
237
|
+
|
|
238
|
+
## License
|
|
239
|
+
|
|
240
|
+
MIT. The compiled dataset is derived from the providers' own public feeds,
|
|
241
|
+
each published for exactly this purpose.
|
|
242
|
+
|
|
243
|
+
---
|
|
244
|
+
|
|
245
|
+
Built by [Yuix Networks](https://yuix.org), who also run
|
|
246
|
+
[Unblock Master](https://www.unblockmaster.com/) and its
|
|
247
|
+
[free IP lookup API](https://www.unblockmaster.com/free-ip-api/).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
iporigin/__init__.py,sha256=ND1IexQpPFE0UoRuOk4GAcfN7dtL3BYyt6q8x4Me-AE,1028
|
|
2
|
+
iporigin/_data.py,sha256=yrCJZ8QoKbyb0v6MzYZgMqFP3tQ-8mXv0HukMg0K5Bw,3737
|
|
3
|
+
iporigin/cli.py,sha256=JmBzTgIjc5M8N3PLTwpqLLjlxa-qMljTf9fv385XCSY,2617
|
|
4
|
+
iporigin/core.py,sha256=bN-MXIIPBdCW7_2Z6Rr0XXiWy38zmZX1EtAftiq8zUo,3861
|
|
5
|
+
iporigin/online.py,sha256=MOgnpGutymSGPCpg5SsJ4pRYzWHDF3xcz5kL0Og97QU,2481
|
|
6
|
+
iporigin/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
iporigin/data/ranges.bin,sha256=N9R-kv4Ig8ZbZf7AFR7nb7oHW3XeQHTxAnmtNZwfbCo,732611
|
|
8
|
+
iporigin-1.0.0.dist-info/licenses/LICENSE,sha256=3UvYE0zJFWjo8fFvty8TzwzFPp3y4WS5_QiQ_J4d5Ww,1074
|
|
9
|
+
iporigin-1.0.0.dist-info/METADATA,sha256=U2JugzKmdoUe6Dt5O_aYHrvIs_62RAGXZgY07w9ahkw,9174
|
|
10
|
+
iporigin-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
11
|
+
iporigin-1.0.0.dist-info/entry_points.txt,sha256=P1lx3Q9qckkC1l5O5-3GhBb0l_aQDOprPmMw9MFave8,47
|
|
12
|
+
iporigin-1.0.0.dist-info/top_level.txt,sha256=7xuiCflcKCO8xd-XWVej18POAUNee7sT24IoS7la3nE,9
|
|
13
|
+
iporigin-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yuix Networks Inc
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
iporigin
|