plevin 0.1.0__py3-none-any.whl

plevin/__init__.py

@@ -0,0 +1,399 @@
+"""Location, network and abuse information for any IP address in one offline file."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from datetime import datetime
+from importlib import import_module
+from importlib.metadata import PackageNotFoundError, version
+from os import PathLike
+from pathlib import Path
+from time import time as now
+from typing import Any
+
+from . import address, derive
+from .address import (
+    BENCHMARK,
+    DOCUMENTATION,
+    LINK_LOCAL,
+    LOOPBACK,
+    MAPPED,
+    MULTICAST,
+    PRIVATE,
+    RESERVED,
+    SHARED,
+    SIXTOFOUR,
+    TEREDO,
+    UNIQUE_LOCAL,
+    Value,
+    parse,
+    purpose,
+    span,
+    spelled,
+    tunnel,
+)
+from .extra import clock, country
+from .models import (
+    Abuse,
+    Carrier,
+    City,
+    Country,
+    District,
+    Metro,
+    Network,
+    Operator,
+    Place,
+    Region,
+    Result,
+    Time,
+)
+from .reader import CACHED, Cache, File, Found
+
+__all__ = [
+    "Abuse",
+    "Carrier",
+    "City",
+    "Country",
+    "District",
+    "Metro",
+    "Network",
+    "Operator",
+    "Place",
+    "Plevin",
+    "Region",
+    "Result",
+    "Time",
+    "address",
+    "database",
+    "lookup",
+    "use",
+]
+
+try:
+    __version__ = version("plevin")
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0"
+
+ENVIRONMENT = "PLEVIN_DB"
+PACKAGES = ("plevin_db", "plevin_db_place", "plevin_db_network", "plevin_db_country")
+MISSING = (
+    "no database found: install one of "
+    f"{', '.join(name.replace('_', '-') for name in PACKAGES)}, set {ENVIRONMENT},"
+    " or pass a path"
+)
+
+WIDE = 1 << 128
+ANSWERED = 1 << 10
+
+Rows = dict[str, Any]
+Ground = tuple[Any, Any, Any, Any, str | None, City | None, Country | None]
+Wires = tuple[int | None, str | None, str | None, Any, Operator | None, Carrier | None]
+Stored = tuple[tuple[Ground, str] | None, tuple[Wires, int | None] | None,
+               Abuse | None]
+
+
+def _text(value: Any) -> str | None:
+    return str(value) if value else None
+
+
+def _count(value: Any) -> int | None:
+    return int(value) if value else None
+
+
+def _metro(rows: Rows | None) -> Metro | None:
+    if rows is None:
+        return None
+    return Metro(code=_count(rows.get("code")), label=_text(rows.get("label")))
+
+
+def _district(rows: Rows | None) -> District | None:
+    if rows is None:
+        return None
+    return District(id=_count(rows.get("id")), code=_text(rows.get("code")),
+                    name=_text(rows.get("name")))
+
+
+def _region(rows: Rows | None) -> Region | None:
+    if rows is None:
+        return None
+    return Region(id=_count(rows.get("id")), code=_text(rows.get("code")),
+                  iso=_text(rows.get("iso")), name=_text(rows.get("name")),
+                  type=_text(rows.get("type")))
+
+
+class Shaped:
+    """One model per row object the file hands back, and the file hands back one."""
+
+    def __init__(self, build: Callable[..., Any]) -> None:
+        self.build = build
+        self.held: dict[Any, tuple[Rows, Any]] = {}
+
+    def __call__(self, rows: Rows | None, *rest: Any) -> Any:
+        if rows is None:
+            return None
+        key = (id(rows), rest)
+        found = self.held.get(key)
+        if found is not None and found[0] is rows:
+            return found[1]
+        if len(self.held) >= CACHED:
+            self.held.clear()
+        model = self.held[key] = (rows, self.build(rows, *rest))
+        return model[1]
+
+
+def _built_city(rows: Rows) -> City:
+    kind = str(rows.get("type", ""))
+    return City(
+        id=_count(rows.get("id")),
+        name=_text(rows.get("name")),
+        ascii=_text(rows.get("ascii")),
+        country=_text(rows.get("country")),
+        population=_count(rows.get("population")),
+        elevation=_count(rows.get("elevation")),
+        postal=_text(rows.get("postal")),
+        postal_partial=_text(rows.get("postal_partial")),
+        timezone=_text(rows.get("timezone")),
+        type=_text(kind),
+        capital=_text(derive.capital(kind)),
+        region=_region(rows.get("region")),
+        district=_district(rows.get("district")),
+        metro=_metro(rows.get("metro")),
+    )
+
+
+_city = Shaped(_built_city)
+
+
+def _place(rows: Rows | None) -> tuple[Ground, str] | None:
+    """The place without its clock, which is the one part an address does not fix."""
+    if rows is None:
+        return None
+    city = _city(rows.get("city"))
+    code = "" if city is None or city.country is None else city.country
+    zone = "" if city is None or city.timezone is None else city.timezone
+    ground = (rows.get("lat"), rows.get("lon"), rows.get("accuracy"),
+              rows.get("confidence"), _text(rows.get("granularity")), city,
+              country(code))
+    return ground, zone
+
+
+def _built_operator(rows: Rows, handle: str) -> Operator:
+    company = str(rows.get("company", ""))
+    website = str(rows.get("website", ""))
+    mailbox = str(rows.get("abuse_email", ""))
+    return Operator(
+        company=_text(company),
+        brand=_text(derive.brand(handle, company)),
+        domain=_text(derive.domain(website, mailbox)),
+        website=_text(website),
+        category=_text(rows.get("category")),
+        tier=_count(rows.get("tier")),
+        peering=_count(rows.get("peering")),
+        scope=_text(rows.get("scope")),
+        rir=_text(rows.get("rir")),
+        since=_count(rows.get("since")),
+        street=_text(rows.get("street")),
+        state=_text(rows.get("state")),
+        postal=_text(rows.get("postal")),
+        country=_text(rows.get("country")),
+        abuse_email=_text(mailbox),
+        city=_city(rows.get("city")),
+    )
+
+
+_operator = Shaped(_built_operator)
+
+
+def _carrier(rows: Rows | None, user_type: str) -> Carrier | None:
+    if rows is None and not user_type:
+        return None
+    held = rows or {}
+    return Carrier(
+        user_type=_text(user_type),
+        user_count=_count(held.get("user_count")),
+        mcc=_count(held.get("mcc")),
+        mnc=_count(held.get("mnc")),
+        is_mobile=user_type == "cellular",
+    )
+
+
+def _abuse(record: Rows | None, system: Rows | None, user_type: str) -> Abuse | None:
+    if record is None and system is None:
+        return None
+    held = record or {}
+    named, inferred = derive.service(str(held.get("service", "")), user_type)
+    evidence = str(held.get("evidence", "")) or inferred
+    return Abuse(
+        name=_text(held.get("name")),
+        service=_text(named),
+        evidence=_text(evidence),
+        risk=held.get("risk"),
+        network_risk=None if system is None else system.get("risk"),
+        last_seen_days=_count(held.get("last_seen_days")),
+        is_anycast=bool(held.get("is_anycast")),
+        is_satellite=bool(held.get("is_satellite")),
+        is_hosting_provider=user_type in derive.SERVERS,
+        is_proxy=named in derive.PROXIES,
+        is_public_proxy=named == "public_proxy",
+        is_residential_proxy=named == "residential_proxy",
+        is_anonymous_vpn=named == "anonymous_vpn",
+        is_tor_exit_node=named == "tor_exit_node",
+        is_private_relay=named == "private_relay",
+        is_anonymous=bool(named),
+    )
+
+
+def _network(rows: Rows | None, user_type: str) -> tuple[Wires, int | None] | None:
+    """The network without its span, which the address the lookup asked about fixes."""
+    if rows is None:
+        return None
+    handle = str(rows.get("handle", ""))
+    wires = (_count(rows.get("asn")), _text(handle), _text(rows.get("rpki")),
+             rows.get("roas"), _operator(rows.get("operator"), handle),
+             _carrier(rows.get("carrier"), user_type))
+    return wires, _count(rows.get("prefix"))
+
+
+def _user_type(record: Rows | None, system: Rows | None) -> str:
+    """The ASN's type sits on the system row; the record carries only an override."""
+    held, below = record or {}, system or {}
+    return str(held.get("user_type") or below.get("user_type") or "")
+
+
+def _stored(rows: Rows) -> Stored:
+    """Everything a boundary answers that no address of it changes, built once."""
+    network = rows.get("network")
+    system = None if network is None else network.get("abuse")
+    user_type = _user_type(rows.get("abuse"), system)
+    return (_place(rows.get("place")), _network(network, user_type),
+            _abuse(rows.get("abuse"), system, user_type))
+
+
+def _result(value: int, wide: bool, stored: Stored | None,
+            moment: datetime | None) -> Result:
+    text, expanded, arpa = spelled(value, wide)
+    marks = purpose(value, wide)
+    through, embedded = tunnel(value, wide)
+    ground, wires, abuse = stored or (None, None, None)
+    return Result(
+        ip=text,
+        version=6 if wide else 4,
+        number=value,
+        compressed=text,
+        expanded=expanded,
+        arpa=arpa,
+        is_global=marks == 0,
+        is_bogon=marks != 0,
+        is_private=marks & PRIVATE != 0,
+        is_loopback=marks & LOOPBACK != 0,
+        is_multicast=marks & MULTICAST != 0,
+        is_reserved=marks & RESERVED != 0,
+        is_link_local=marks & LINK_LOCAL != 0,
+        is_unique_local=marks & UNIQUE_LOCAL != 0,
+        is_documentation=marks & DOCUMENTATION != 0,
+        is_shared=marks & SHARED != 0,
+        is_benchmark=marks & BENCHMARK != 0,
+        is_ipv4_mapped=through == MAPPED,
+        is_6to4=through == SIXTOFOUR,
+        is_teredo=through == TEREDO,
+        tunnel=through,
+        embedded_ipv4=embedded,
+        found=stored is not None,
+        place=None if ground is None else Place(*ground[0], clock(ground[1], moment)),
+        network=None if wires is None else _spanned(*wires, value, wide),
+        abuse=abuse,
+    )
+
+
+def _spanned(wires: Wires, prefix: int | None, value: int, wide: bool) -> Network:
+    asn, handle, rpki, roas, operator, carrier = wires
+    cidr, start, end = (None, None, None) if prefix is None else span(
+        value, wide, prefix)
+    return Network(asn, handle, prefix, cidr, start, end, rpki, roas, operator, carrier)
+
+
+def _found() -> Path:
+    """The database a reader that was given no path should open."""
+    named = os.environ.get(ENVIRONMENT)
+    if named:
+        return Path(named)
+    for package in PACKAGES:
+        try:
+            module = import_module(package)
+        except ModuleNotFoundError:
+            continue
+        return Path(module.PATH)
+    raise LookupError(MISSING)
+
+
+class Plevin:
+    """One database, opened once and asked as often as a log has addresses."""
+
+    def __init__(self, path: str | PathLike[str] | None = None) -> None:
+        self.file = File(_found() if path is None else path)
+        self.stored = Cache(self._stored)
+        self.results = Cache(self._result, ANSWERED)
+        self.second = 0
+
+    @property
+    def path(self) -> str:
+        return self.file.path
+
+    @property
+    def built(self) -> str:
+        return self.file.built
+
+    @property
+    def selection(self) -> str:
+        return self.file.selection
+
+    @property
+    def fields(self) -> list[str]:
+        return self.file.fields
+
+    def _stored(self, found: Found) -> Stored:
+        return _stored(self.file.answers[found])
+
+    def _result(self, key: int) -> Result:
+        wide = key >= WIDE
+        value = key - WIDE if wide else key
+        found = self.file.locate(value, wide)
+        return _result(value, wide, None if found is None else self.stored[found], None)
+
+    def lookup(self, value: Value, moment: datetime | None = None) -> Result:
+        """One address, however it is written, as everything the file answers."""
+        number, wide = parse(value)
+        if moment is not None:
+            found = self.file.locate(number, wide)
+            held = None if found is None else self.stored[found]
+            return _result(number, wide, held, moment)
+        second = int(now())
+        if second != self.second:
+            self.second = second
+            self.results.clear()
+        answer: Result = self.results[number + WIDE if wide else number]
+        return answer
+
+
+_opened: Plevin | None = None
+
+
+def database() -> Plevin:
+    """The database the module reads, opened on the first address asked of it."""
+    global _opened
+    if _opened is None:
+        _opened = Plevin()
+    return _opened
+
+
+def use(path: str | PathLike[str] | None) -> Plevin:
+    """Read a database of your own from here on, or forget the one in use."""
+    global _opened
+    _opened = None if path is None else Plevin(path)
+    return database()
+
+
+def lookup(value: Value, moment: datetime | None = None) -> Result:
+    """One address, however it is written, as everything the file answers."""
+    return database().lookup(value, moment)

plevin/address.py

@@ -0,0 +1,133 @@
+"""What an address is before any database is opened."""
+
+from __future__ import annotations
+
+from bisect import bisect_right
+from ipaddress import IPv4Address, IPv6Address, ip_address, ip_network
+from socket import AF_INET, AF_INET6, inet_pton
+
+Value = str | int | bytes | bytearray | memoryview | IPv4Address | IPv6Address
+
+PRIVATE, LOOPBACK, MULTICAST, RESERVED = 1, 2, 4, 8
+LINK_LOCAL, UNIQUE_LOCAL, DOCUMENTATION, SHARED, BENCHMARK = 16, 32, 64, 128, 256
+
+SPECIAL_V4 = (
+    ("0.0.0.0/8", RESERVED),
+    ("10.0.0.0/8", PRIVATE),
+    ("100.64.0.0/10", PRIVATE | SHARED),
+    ("127.0.0.0/8", PRIVATE | LOOPBACK),
+    ("169.254.0.0/16", PRIVATE | LINK_LOCAL),
+    ("172.16.0.0/12", PRIVATE),
+    ("192.0.0.0/24", RESERVED),
+    ("192.0.2.0/24", RESERVED | DOCUMENTATION),
+    ("192.88.99.0/24", RESERVED),
+    ("192.168.0.0/16", PRIVATE),
+    ("198.18.0.0/15", RESERVED | BENCHMARK),
+    ("198.51.100.0/24", RESERVED | DOCUMENTATION),
+    ("203.0.113.0/24", RESERVED | DOCUMENTATION),
+    ("224.0.0.0/4", MULTICAST),
+    ("240.0.0.0/4", RESERVED),
+)
+SPECIAL_V6 = (
+    ("::/128", RESERVED),
+    ("::1/128", PRIVATE | LOOPBACK),
+    ("::ffff:0:0/96", RESERVED),
+    ("64:ff9b::/96", RESERVED),
+    ("64:ff9b:1::/48", RESERVED),
+    ("100::/64", RESERVED),
+    ("2001::/23", RESERVED),
+    ("2001:db8::/32", RESERVED | DOCUMENTATION),
+    ("2002::/16", RESERVED),
+    ("3fff::/20", RESERVED | DOCUMENTATION),
+    ("fc00::/7", PRIVATE | UNIQUE_LOCAL),
+    ("fe80::/10", PRIVATE | LINK_LOCAL),
+    ("ff00::/8", MULTICAST),
+)
+
+MAPPED = "ipv4-mapped"
+SIXTOFOUR = "6to4"
+TEREDO = "teredo"
+NAT64 = "nat64"
+NAT64_PREFIX = 0x0064FF9B << 96
+
+
+def _table(rows: tuple[tuple[str, int], ...]) -> tuple[list[int], list[tuple[int, int]]]:
+    """The rows as first addresses to bisect, and what each range ends at and marks."""
+    spans = []
+    for text, marks in rows:
+        span = ip_network(text)
+        spans.append((int(span.network_address), int(span.broadcast_address), marks))
+    spans.sort()
+    return [first for first, _, _ in spans], [(last, marks) for _, last, marks in spans]
+
+
+SPECIAL = (_table(SPECIAL_V4), _table(SPECIAL_V6))
+
+
+def parse(value: Value) -> tuple[int, bool]:
+    """An address however it is written, as its number and whether it is a v6 one."""
+    if isinstance(value, str):
+        wide = ":" in value
+        try:
+            packed = inet_pton(AF_INET6 if wide else AF_INET, value)
+        except OSError:
+            return parse(ip_address(value))
+        return int.from_bytes(packed, "big"), wide
+    if isinstance(value, bytes | bytearray | memoryview):
+        return parse(ip_address(bytes(value)))
+    if isinstance(value, IPv4Address | IPv6Address):
+        return int(value), value.version == 6
+    if value < 0 or value > 0xFFFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF:
+        raise ValueError(f"{value} is not an address")
+    return value, value > 0xFFFFFFFF
+
+
+def written(value: int, wide: bool) -> str:
+    """An address as text: v4 from its octets, v6 through the shortening rules."""
+    if wide:
+        return str(IPv6Address(value))
+    return f"{value >> 24}.{value >> 16 & 255}.{value >> 8 & 255}.{value & 255}"
+
+
+def spelled(value: int, wide: bool) -> tuple[str, str, str]:
+    """How the address reads short and in full, and the name a resolver asks by."""
+    if wide:
+        address = IPv6Address(value)
+        return str(address), address.exploded, address.reverse_pointer
+    octets = [str(value >> shift & 255) for shift in (24, 16, 8, 0)]
+    text = ".".join(octets)
+    return text, text, ".".join(reversed(octets)) + ".in-addr.arpa"
+
+
+def tunnel(value: int, wide: bool) -> tuple[str | None, str | None]:
+    """The v4 address a v6 one carries, and the tunnel that puts it there."""
+    if not wide:
+        return None, None
+    address = IPv6Address(value)
+    if address.ipv4_mapped is not None:
+        return MAPPED, str(address.ipv4_mapped)
+    if address.sixtofour is not None:
+        return SIXTOFOUR, str(address.sixtofour)
+    if address.teredo is not None:
+        return TEREDO, str(address.teredo[1])
+    if value >> 32 == NAT64_PREFIX >> 32:
+        return NAT64, written(value & 0xFFFFFFFF, False)
+    return None, None
+
+
+def purpose(value: int, wide: bool) -> int:
+    """What an address is where it is not the internet, bisected out of the table."""
+    starts, ranges = SPECIAL[wide]
+    at = bisect_right(starts, value) - 1
+    if at >= 0:
+        last, marks = ranges[at]
+        if value <= last:
+            return marks
+    return RESERVED if wide and value >> 125 != 1 else 0
+
+
+def span(value: int, wide: bool, prefix: int) -> tuple[str, str, str]:
+    """The announcement the address falls in, masked out of the address itself."""
+    spare = (128 if wide else 32) - prefix
+    start = written(value >> spare << spare, wide)
+    return f"{start}/{prefix}", start, written(value | (1 << spare) - 1, wide)

plevin/derive.py

@@ -0,0 +1,111 @@
+"""The answers the file does not store, read off the ones it does."""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+
+FORMS = frozenset(
+    "inc incorporated llc ltd ltda limited gmbh mbh ag kgaa ohg ev eg sa sab saa sau"
+    " sal saog sac sas sarl srl spa nv bv cv asa aps oyj kft zrt nyrt doo sro ooo zao"
+    " pao pjsc jsc ojsc llp plc pte pteltd pty corp corporation company holding"
+    " holdings group uab sia tov oao pt sdn bhd coltd coltda eireli ead ood eood sti"
+    " ltdsti spzoo".split()
+)
+TAILS = frozenset(
+    "de me epp co as ab ad dd bt lc lp se sl slu sp z oo zoo oy ao esp kg network"
+    " networks net telecom telecoms telecommunication telecommunications"
+    " telecomunicaciones comunicaciones communication communications hosting solutions"
+    " services service technologies technology tech systems system data datacenter"
+    " datacentre cloud internet online isp international global enterprises enterprise"
+    " backbone provider providers of and".split()
+)
+LEAD = frozenset(
+    "the llc ltd gmbh sarl ooo zao pao ao oao jsc ojsc pjsc uab sia tov pt pp ps ip"
+    " spolka".split()
+)
+TAIL = FORMS | TAILS | frozenset({""})
+TLDS = (".com", ".net", ".org", ".io")
+
+TRADING = re.compile(r"(?i).*\b(?:trading as|d/b/a|dba)\b\s*")
+ALIAS = re.compile(r"\(.*?\)|,.*|\s+-\s+.*")
+BARE = re.compile(r"[^0-9a-z]")
+NUMBERED = re.compile(r"AS\d+", re.I)
+HANDLE_TAIL = re.compile(r"-(AS|AP|US|UK|DE|FR|IN|CN|JP|EU|NET|COM|ORG)$")
+NETWORK_TAIL = re.compile(r"(NET|COM|TEL|WEB|LINE)$")
+AUTHORITY = re.compile(r"[/?#]")
+
+SERVERS = frozenset({"hosting", "cdn", "content"})
+ACCESS = frozenset({"residential", "cellular"})
+PROXIES = frozenset({"public_proxy", "residential_proxy"})
+NAMES = 1 << 13
+CAPITALS = {"national capital": "country", "regional capital": "region",
+            "district capital": "district"}
+
+
+def _cased(text: str) -> str:
+    """Shouted words stop shouting, `GOOGLE` reading as `Google` and `IBM` as `IBM`."""
+    return " ".join(
+        word.title() if word.isalpha() and word.isupper() and len(word) > 4 else word
+        for word in text.split(" ")
+    )
+
+
+def _from_company(company: str) -> str:
+    """Aliases, legal forms and the words every network shares, all stripped."""
+    words = ALIAS.sub("", TRADING.sub("", company)).split()
+    tokens = [token for word in words if (token := word.strip("\"'"))]
+    while len(tokens) > 1 and BARE.sub("", tokens[-1].lower()) in TAIL:
+        tokens.pop()
+    while tokens and BARE.sub("", tokens[0].lower()) in LEAD:
+        tokens.pop(0)
+    name = " ".join(tokens)
+    cut = next((len(tld) for tld in TLDS if name.lower().endswith(tld)), 0)
+    return _cased(name[: len(name) - cut])
+
+
+def _from_handle(handle: str) -> str:
+    """The first word of a registry handle, its country and network tails gone."""
+    words = handle.split()
+    head = HANDLE_TAIL.sub("", words[0]) if words else ""
+    if NUMBERED.fullmatch(head):
+        return ""
+    if len(head) > 4 and head.isupper():
+        head = NETWORK_TAIL.sub("", head)
+    return _cased(head)
+
+
+@lru_cache(maxsize=NAMES)
+def brand(handle: str, company: str) -> str:
+    """The name a network goes by: its handle where the company only spells it out."""
+    legal, short = _from_company(company), _from_handle(handle)
+    if not legal or not short:
+        return legal or short
+    if legal.lower() == short.lower():
+        return legal if short.isupper() else short
+    return short if legal.lower().startswith(f"{short.lower()} ") else legal
+
+
+@lru_cache(maxsize=NAMES)
+def domain(website: str, mailbox: str) -> str:
+    """The bare host the website names, else the one the abuse mailbox does."""
+    authority = AUTHORITY.split(website.rpartition("//")[2], maxsplit=1)[0]
+    host = authority.rpartition("@")[2].partition(":")[0]
+    site = (host or mailbox.partition("@")[2]).lower().removeprefix("www.")
+    box = mailbox.partition("@")[2].lower()
+    top = site.rpartition(".")[2]
+    if len(top) > 2 and box.partition(".")[0] == top and site != box:
+        return box
+    return site
+
+
+def service(named: str, user_type: str) -> tuple[str, str]:
+    """A public proxy on an access network is someone's home line, resold."""
+    if named == "public_proxy" and user_type in ACCESS:
+        return "residential_proxy", "inferred"
+    return named, ""
+
+
+def capital(city_type: str) -> str:
+    """The city type already says which capital it is, so nothing stores it twice."""
+    return CAPITALS.get(city_type, "")