ctidb-bin 0.0.1__py3-none-any.whl

ctidb_bin/__init__.py

@@ -0,0 +1,12 @@
+"""Packed Binary — IP intelligence binary format (CTIDB v2, read-only)."""
+from .format import HotRecord, RangeEntry, License, Flags, Plan
+from .reader import PackedReader, Metadata
+from .name_table import NameTableReader
+
+__version__ = "0.0.1"
+__all__ = [
+    "PackedReader",
+    "HotRecord", "RangeEntry", "License", "Flags", "Plan",
+    "Metadata",
+    "NameTableReader",
+]

ctidb_bin/format.py

@@ -0,0 +1,213 @@
+"""Packed Binary 포맷 — 구조체/상수 정의.
+
+docs/FORMAT.md 와 1:1 매칭. 모든 struct fmt 가 little-endian (`<`).
+"""
+from __future__ import annotations
+import struct
+from dataclasses import dataclass
+from enum import IntFlag
+
+# ─────────────────────── 시그니처 / 상수 ───────────────────────
+MAGIC_HEADER = b"CTIDBV2\x00"
+MAGIC_FOOTER = b"\x00PACKED\x00"
+FORMAT_VERSION_MAJOR = 1
+FORMAT_VERSION_MINOR = 2        # v1.2: cold_offset uint64 (Plan 4 4GB+ cold section 지원)
+
+# Section offsets / fixed sizes
+HEADER_SIZE        = 16
+LICENSE_BLOB_SIZE  = 256
+LICENSE_SIG_SIZE   = 256
+LICENSE_SECT_SIZE  = LICENSE_BLOB_SIZE + LICENSE_SIG_SIZE   # 512
+RANGE_TABLE_OFFSET = HEADER_SIZE + LICENSE_SECT_SIZE        # 528
+RANGE_ENTRY_SIZE     = 16               # v1.0/1.1 legacy (uint32 x 4)
+RANGE_ENTRY_SIZE_V12 = 20               # v1.2: uint32 x 3 + uint64 (cold_offset)
+HOT_RECORD_SIZE    = 8
+FOOTER_SIZE        = 16                 # v1.0 (legacy)
+FOOTER_SIZE_V11    = 24                 # v1.1/1.2: +8 byte name_table_offset
+
+
+# ─────────────────────── struct format strings ───────────────────────
+# Header
+HEADER_FMT = "<8sHHI"   # magic(8) + ver_major(2) + ver_minor(2) + range_count(4) = 16
+assert struct.calcsize(HEADER_FMT) == HEADER_SIZE
+
+# Range entry
+RANGE_FMT      = "<IIII"    # v1.0/1.1: start(4) + end(4) + hot_off(4) + cold_off_u32(4) = 16
+RANGE_FMT_V12  = "<IIIQ"    # v1.2:     start(4) + end(4) + hot_off(4) + cold_off_u64(8) = 20
+assert struct.calcsize(RANGE_FMT) == RANGE_ENTRY_SIZE
+assert struct.calcsize(RANGE_FMT_V12) == RANGE_ENTRY_SIZE_V12
+
+# Hot record — 24-bit asn_id 는 4 byte 단위 align 위해 3B + 1B padding 안 쓰고
+# 그냥 1+3+1+1+2 = 8 byte 명시. asn_id 는 uint32 의 하위 24비트만 사용.
+HOT_FMT = "<BBBB BBH"
+# = country(1) + asn_lo(1) + asn_mid(1) + asn_hi(1) + score(1) + score_inbound(1) + flags(2)
+assert struct.calcsize(HOT_FMT) == HOT_RECORD_SIZE
+
+# License blob (256B)
+LICENSE_FMT = "<32s16s B Q Q I 187s"
+# license_key(32) + customer_id(16) + plan(1) + issued(8) + expire(8) + permissions(4) + reserved(187) = 256
+assert struct.calcsize(LICENSE_FMT) == LICENSE_BLOB_SIZE
+
+# Footer — v1.0 legacy (magic + crc + fv = 16B)
+FOOTER_FMT = "<8sII"    # magic(8) + crc32(4) + format_ver(4) = 16
+assert struct.calcsize(FOOTER_FMT) == FOOTER_SIZE
+
+# Footer — v1.1 (magic + crc + fv + name_table_offset = 24B)
+FOOTER_FMT_V11 = "<8sIIQ"
+assert struct.calcsize(FOOTER_FMT_V11) == FOOTER_SIZE_V11
+
+
+# ─────────────────────── Flags bitfield ───────────────────────
+class Flags(IntFlag):
+    IS_VPN          = 1 << 0
+    IS_TOR          = 1 << 1
+    IS_CDN          = 1 << 2
+    IS_PROXY        = 1 << 3
+    IS_HOSTING      = 1 << 4
+    IS_CLOUD        = 1 << 5
+    IS_C2           = 1 << 6
+    IS_HONEYPOT     = 1 << 7
+    HAS_ABUSE       = 1 << 8
+    HAS_CVE         = 1 << 9
+    IS_MOBILE       = 1 << 10
+
+
+# ─────────────────────── Plan IDs ───────────────────────
+class Plan(IntFlag):
+    GEOLOCATION    = 1
+    TI_C2          = 2
+    FRAUD          = 3
+    FULL           = 4
+    PRIVACY        = 5
+
+
+# ─────────────────────── Dataclasses (in-memory representation) ───────────────────────
+@dataclass(frozen=True)
+class HotRecord:
+    country_code_id: int    # uint8
+    asn_id:          int    # uint24 — 16M 까지
+    score:           int    # uint8 (0-100)
+    score_inbound:   int    # uint8 (0-100)
+    flags:           int    # uint16 bitfield
+
+    def pack(self) -> bytes:
+        a = self.asn_id & 0xFFFFFF
+        return struct.pack(
+            HOT_FMT,
+            self.country_code_id & 0xFF,
+            a & 0xFF,
+            (a >> 8) & 0xFF,
+            (a >> 16) & 0xFF,
+            self.score & 0xFF,
+            self.score_inbound & 0xFF,
+            self.flags & 0xFFFF,
+        )
+
+    @classmethod
+    def unpack_from(cls, buf, offset: int) -> "HotRecord":
+        cc, a_lo, a_mid, a_hi, sc, sc_in, fl = struct.unpack_from(HOT_FMT, buf, offset)
+        return cls(
+            country_code_id=cc,
+            asn_id=(a_hi << 16) | (a_mid << 8) | a_lo,
+            score=sc,
+            score_inbound=sc_in,
+            flags=fl,
+        )
+
+
+@dataclass(frozen=True)
+class RangeEntry:
+    start_ip:   int     # uint32 (host endian, little-endian on x86)
+    end_ip:     int
+    hot_offset: int     # uint32 (offset within Hot Records section)
+    cold_idx:   int     # uint32 (0 = no cold record)
+
+    def pack(self) -> bytes:
+        return struct.pack(RANGE_FMT, self.start_ip, self.end_ip, self.hot_offset, self.cold_idx)
+
+    @classmethod
+    def unpack_from(cls, buf, offset: int) -> "RangeEntry":
+        s, e, h, c = struct.unpack_from(RANGE_FMT, buf, offset)
+        return cls(start_ip=s, end_ip=e, hot_offset=h, cold_idx=c)
+
+
+@dataclass(frozen=True)
+class License:
+    license_key:   str
+    customer_id:   str
+    plan:          int
+    issued_at:     int      # unix ts
+    expire_at:     int
+    permissions:   int      # bitfield
+
+    def pack(self) -> bytes:
+        return struct.pack(
+            LICENSE_FMT,
+            self.license_key.encode("ascii")[:32],
+            self.customer_id.encode("ascii")[:16],
+            self.plan & 0xFF,
+            self.issued_at & 0xFFFFFFFFFFFFFFFF,
+            self.expire_at & 0xFFFFFFFFFFFFFFFF,
+            self.permissions & 0xFFFFFFFF,
+            b"",                     # reserved (zero-padded by struct)
+        )
+
+    @classmethod
+    def unpack(cls, blob: bytes) -> "License":
+        k, c, p, iss, exp, perm, _ = struct.unpack(LICENSE_FMT, blob)
+        return cls(
+            license_key=k.rstrip(b"\x00").decode("ascii"),
+            customer_id=c.rstrip(b"\x00").decode("ascii"),
+            plan=p,
+            issued_at=iss,
+            expire_at=exp,
+            permissions=perm,
+        )
+
+
+# ─────────────────────── Header / Footer helpers ───────────────────────
+def pack_header(range_count: int) -> bytes:
+    return struct.pack(
+        HEADER_FMT,
+        MAGIC_HEADER,
+        FORMAT_VERSION_MAJOR,
+        FORMAT_VERSION_MINOR,
+        range_count,
+    )
+
+
+def unpack_header(buf: bytes) -> tuple[int, int, int]:
+    """returns (ver_major, ver_minor, range_count)"""
+    magic, vmaj, vmin, rcount = struct.unpack(HEADER_FMT, buf[:HEADER_SIZE])
+    if magic != MAGIC_HEADER:
+        raise ValueError(f"bad magic: {magic!r}")
+    return vmaj, vmin, rcount
+
+
+def pack_footer(crc32: int, name_table_offset: int | None = None) -> bytes:
+    """v1.1: name_table_offset 를 명시적으로 저장 (24 byte).
+    v1.0 legacy: name_table_offset=None 이면 16 byte footer.
+    """
+    fv = (FORMAT_VERSION_MAJOR << 16) | FORMAT_VERSION_MINOR
+    if name_table_offset is None:
+        return struct.pack(FOOTER_FMT, MAGIC_FOOTER, crc32 & 0xFFFFFFFF, fv)
+    return struct.pack(
+        FOOTER_FMT_V11, MAGIC_FOOTER, crc32 & 0xFFFFFFFF, fv,
+        name_table_offset & 0xFFFFFFFFFFFFFFFF,
+    )
+
+
+def unpack_footer(buf: bytes) -> tuple[int, int, int | None]:
+    """dual-mode: buf 는 file 끝의 24 byte 이상.
+    반환: (crc32, format_version, name_table_offset_or_None)
+    - v1.1 (24B): name_table_offset explicit
+    - v1.0 (16B): None (호출부에서 heuristic 계산)
+    """
+    tail = bytes(buf)
+    if len(tail) >= FOOTER_SIZE_V11 and tail[-FOOTER_SIZE_V11:-FOOTER_SIZE_V11+8] == MAGIC_FOOTER:
+        magic, crc, fv, name_off = struct.unpack(FOOTER_FMT_V11, tail[-FOOTER_SIZE_V11:])
+        return crc, fv, name_off
+    magic, crc, fv = struct.unpack(FOOTER_FMT, tail[-FOOTER_SIZE:])
+    if magic != MAGIC_FOOTER:
+        raise ValueError(f"bad footer magic: {magic!r}")
+    return crc, fv, None

ctidb_bin/name_table.py

@@ -0,0 +1,84 @@
+"""Name Table — string ↔ uint32 ID 양방향 매핑.
+
+writer 단계: string add → ID 할당 (dedup)
+reader 단계: mmap 위에서 ID → string lookup (zero-copy 가능한 곳은)
+"""
+from __future__ import annotations
+import array
+import struct
+from typing import Iterator
+
+
+# ─────────────────────── Writer 측 ───────────────────────
+class NameTableBuilder:
+    """writer 가 record 만들면서 string 마다 ID 할당 + dedup."""
+
+    def __init__(self) -> None:
+        self._str_to_id: dict[str, int] = {}
+        self._id_to_str: list[str] = []
+        # ID 0 = empty string (sentinel)
+        self._add_internal("")
+
+    def _add_internal(self, s: str) -> int:
+        i = len(self._id_to_str)
+        self._str_to_id[s] = i
+        self._id_to_str.append(s)
+        return i
+
+    def add(self, s: str | None) -> int:
+        """string 추가 → ID 반환. 같은 string 은 동일 ID."""
+        if not s:
+            return 0
+        i = self._str_to_id.get(s)
+        if i is not None:
+            return i
+        return self._add_internal(s)
+
+    def __len__(self) -> int:
+        return len(self._id_to_str)
+
+    def serialize(self) -> bytes:
+        """디스크 포맷: [count:uint32] [len:uint16][bytes] × N"""
+        parts: list[bytes] = [struct.pack("<I", len(self._id_to_str))]
+        for s in self._id_to_str:
+            b = s.encode("utf-8")
+            if len(b) > 65535:
+                raise ValueError(f"string too long ({len(b)} bytes, max 65535)")
+            parts.append(struct.pack("<H", len(b)))
+            parts.append(b)
+        return b"".join(parts)
+
+
+# ─────────────────────── Reader 측 ───────────────────────
+class NameTableReader:
+    """mmap buffer 의 name_table section 을 ID → string 조회."""
+
+    def __init__(self, buf, offset: int) -> None:
+        self._buf = buf
+        self._base = offset
+        (count,) = struct.unpack_from("<I", buf, offset)
+        self._count = count
+        # 각 string 의 file offset. list[int](Python int 28B/개) 대신 array.array('Q')(8B/개)
+        # → 40M names 기준 ~1.1GB → 320MB (mmap 리더 메모리 폭식 완화).
+        offsets = array.array('Q', bytes(8 * count))  # 0으로 preallocate
+        cur = offset + 4
+        for i in range(count):
+            (l,) = struct.unpack_from("<H", buf, cur)
+            offsets[i] = cur + 2
+            cur += 2 + l
+        self._offsets = offsets
+
+    def get(self, str_id: int) -> str:
+        if str_id == 0 or str_id >= self._count:
+            return ""
+        off = self._offsets[str_id]
+        # length prefix 는 off-2 위치
+        (l,) = struct.unpack_from("<H", self._buf, off - 2)
+        return bytes(self._buf[off:off + l]).decode("utf-8", errors="replace")
+
+    def __len__(self) -> int:
+        return self._count
+
+    def __iter__(self) -> Iterator[str]:
+        for i in range(self._count):
+            yield self.get(i)

ctidb_bin/reader.py

@@ -0,0 +1,343 @@
+"""Packed Binary Reader — mmap + binary search + zero-copy decode.
+
+Example:
+    with PackedReader.open("/path/cti.bin") as r:
+        rec = r.get("8.8.8.8")
+        print(r.metadata.license_key, rec)
+"""
+from __future__ import annotations
+import sys
+import array
+import mmap
+import struct
+import ipaddress
+import bisect
+from dataclasses import dataclass
+from typing import Any
+
+from .format import (
+    HEADER_SIZE, LICENSE_BLOB_SIZE, LICENSE_SECT_SIZE, RANGE_TABLE_OFFSET,
+    RANGE_ENTRY_SIZE, RANGE_ENTRY_SIZE_V12, HOT_RECORD_SIZE,
+    FOOTER_SIZE, FOOTER_SIZE_V11,
+    HotRecord, RangeEntry, License, Flags,
+    unpack_header, unpack_footer,
+)
+from .name_table import NameTableReader
+
+
+def _ip_to_int(ip: str) -> int:
+    return int(ipaddress.IPv4Address(ip))
+
+
+@dataclass(frozen=True)
+class Metadata:
+    version: tuple[int, int]
+    range_count: int
+    license: License
+    file_size: int
+
+
+class PackedReader:
+    """zero-copy mmap reader. close() 또는 with 로 닫기."""
+
+    def __init__(self, mm: mmap.mmap, file_size: int, with_cold: bool = True) -> None:
+        self._mm = mm
+        self._file_size = file_size
+        self._with_cold = with_cold
+
+        # Header
+        vmaj, vmin, range_count = unpack_header(bytes(mm[:HEADER_SIZE]))
+        self._range_count = range_count
+
+        # ver_major=2 → 컬럼형(col) 레이아웃. 별도 파서로 분기 (range table 아님).
+        self._col = (vmaj == 2)
+        if self._col:
+            self._init_col(mm, file_size, vmin)
+            return
+
+        # License
+        license_blob = bytes(mm[HEADER_SIZE:HEADER_SIZE + LICENSE_BLOB_SIZE])
+        self._license = License.unpack(license_blob)
+        # signature 는 다음 256B (verify 는 license.py 의 verify_signature() 별도)
+
+        # v1.2: RANGE_ENTRY_SIZE 20 (cold_offset uint64). v1.0/1.1: 16.
+        # Header 의 minor version 으로 결정.
+        if vmin >= 2:
+            self._range_entry_size = RANGE_ENTRY_SIZE_V12
+            self._range_fmt = "<IIIQ"
+        else:
+            self._range_entry_size = RANGE_ENTRY_SIZE
+            self._range_fmt = "<IIII"
+
+        # Range table — start_ip 배열만 별도로 두면 binary search 빠름
+        self._range_offset = RANGE_TABLE_OFFSET
+        self._hot_offset = self._range_offset + range_count * self._range_entry_size
+
+        # start_ip 만 모은 정렬 배열 + max(hot_off) 추적
+        # v1.0.3: Python list[int] (28 byte/elem) → array.array('I') (4 byte/elem) — 7배 memory 절감
+        self._start_ips = array.array('I', [0] * range_count)
+        max_hot_off = 0
+        for i in range(range_count):
+            off = self._range_offset + i * self._range_entry_size
+            s, _e, hoff, _c = struct.unpack_from(self._range_fmt, mm, off)
+            self._start_ips[i] = s
+            if hoff > max_hot_off:
+                max_hot_off = hoff
+        # hot section 의 byte size = max_hot_off + 1 record
+        self._hot_size = max_hot_off + HOT_RECORD_SIZE
+
+        # Footer 파싱 — v1.0 legacy 16B, v1.1/1.2 확장 24B (name_table_offset explicit).
+        tail = bytes(mm[-FOOTER_SIZE_V11:]) if file_size >= FOOTER_SIZE_V11 else bytes(mm[-FOOTER_SIZE:])
+        crc, fv, name_off_explicit = unpack_footer(tail)
+        self._crc = crc
+        self._format_ver_packed = fv
+        if name_off_explicit is not None:
+            # v1.1: writer 가 저장한 정확한 offset
+            self._name_table_offset = name_off_explicit
+            self._footer_offset = file_size - FOOTER_SIZE_V11
+        else:
+            # v1.0 legacy: cold 없다는 가정 (cold 있으면 offset 어긋남 — 옛 spec 한계)
+            self._name_table_offset = self._hot_offset + self._hot_size
+            self._footer_offset = file_size - FOOTER_SIZE
+
+        try:
+            self._names = NameTableReader(mm, self._name_table_offset)
+        except Exception:
+            # v1.0 파일 + cold 있어서 offset 어긋남 — name lookup 은 empty 로 처리
+            self._names = None  # type: ignore
+
+        self._metadata = Metadata(
+            version=(vmaj, vmin),
+            range_count=range_count,
+            license=self._license,
+            file_size=file_size,
+        )
+
+    # ─────────────────── public API ───────────────────
+    @classmethod
+    def open(cls, path: str, with_cold: bool = True) -> "PackedReader":
+        f = open(path, "rb")
+        size = f.seek(0, 2)
+        f.seek(0)
+        mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+        f.close()
+        return cls(mm, file_size=size, with_cold=with_cold)
+
+    def close(self) -> None:
+        self._mm.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self.close()
+
+    @property
+    def metadata(self) -> Metadata:
+        return self._metadata
+
+    def get(self, ip: str) -> dict[str, Any] | None:
+        ip_int = _ip_to_int(ip)
+        if self._col:
+            return self._col_get(ip_int)
+        # binary search — bisect_right - 1 은 start_ip <= ip_int 인 가장 큰 idx
+        idx = bisect.bisect_right(self._start_ips, ip_int) - 1
+        if idx < 0:
+            return None
+        # v1.0.9: K=1024 (parquet /32 밀집 대역에서 GeoIP2 base range 놓침 방지).
+        # 넓은 base range 를 찾으면 즉시 break (더 뒤로 갈수록 base 발견 확률 감소).
+        best_hot = None
+        best_width = -1
+        LOOKBACK_LIMIT = 1024
+        for _ in range(LOOKBACK_LIMIT):
+            if idx < 0:
+                break
+            off = self._range_offset + idx * self._range_entry_size
+            s, e, hot_off, cold_off = struct.unpack_from(self._range_fmt, self._mm, off)
+            if s <= ip_int <= e:
+                w = e - s
+                if best_hot is None or w < best_width:
+                    best_hot = hot_off
+                    best_width = w
+                    if w == 0:
+                        break  # /32 최소 — 더 좁을 수 없음
+            idx -= 1
+        if best_hot is None:
+            return None
+        return self._decode_hot(best_hot)
+
+    def get_with_cold(self, ip: str) -> dict[str, Any] | None:
+        """hot + cold 둘 다. cold 는 lazy mmap (fault-in)."""
+        # 현재 v1 은 cold 위치 정보 없음 — hot only
+        return self.get(ip)
+
+    # ─────────────────── internal ───────────────────
+    def _decode_hot(self, hot_off: int) -> dict[str, Any]:
+        abs_off = self._hot_offset + hot_off
+        hr = HotRecord.unpack_from(self._mm, abs_off)
+        out: dict[str, Any] = {
+            "country_code": self._name(hr.country_code_id),
+            "as_name":      self._name(hr.asn_id),
+            "score":        hr.score,
+            "score_inbound": hr.score_inbound,
+        }
+        # flags 분해
+        f = hr.flags
+        for key, mask in [
+            ("is_vpn",      Flags.IS_VPN),
+            ("is_tor",      Flags.IS_TOR),
+            ("is_cdn",      Flags.IS_CDN),
+            ("is_proxy",    Flags.IS_PROXY),
+            ("is_hosting",  Flags.IS_HOSTING),
+            ("is_cloud",    Flags.IS_CLOUD),
+            ("is_c2",       Flags.IS_C2),
+            ("is_honeypot", Flags.IS_HONEYPOT),
+            ("is_mobile",   Flags.IS_MOBILE),
+        ]:
+            if f & mask:
+                out[key] = True
+        return out
+
+    def _name(self, str_id: int) -> str:
+        if self._names is None or str_id == 0:
+            return ""
+        return self._names.get(str_id)
+
+    # ─────────────────── columnar (ver_major=2) ───────────────────
+    # L1 range(country·asn·city) / L2 flags(비겹침 range 리스트) / L3 out·in score(0~5, C·D)
+    _COL_FLAG_KEY = {
+        0: "is_vpn", 1: "is_tor", 2: "is_cdn", 3: "is_proxy", 4: "is_hosting",
+        5: "is_cloud", 6: "is_c2", 7: "is_honeypot", 10: "is_mobile",
+    }
+
+    @staticmethod
+    def _le(arr: "array.array") -> "array.array":
+        # frombytes 는 native 바이트오더 — 디스크는 LE. big-endian 호스트에서만 swap.
+        if sys.byteorder != "little":
+            arr.byteswap()
+        return arr
+
+    def _init_col(self, mm: mmap.mmap, file_size: int, vmin: int) -> None:
+        self._names = None
+        self._mm = mm
+        product_id, section_count = struct.unpack_from("<HH", mm, 12)
+        self._col_product = product_id
+        secs = {}
+        for i in range(section_count):
+            typ, _codec, _rsv, off, ln = struct.unpack_from("<BBHQQ", mm, HEADER_SIZE + i * 20)
+            secs[typ] = (off, ln)
+        self._col_sections = secs
+        self._col_names: list[str] = []
+        # 메모리: list[tuple] 대신 array('I') (4byte/elem) — 761MB bin 이 8Gi OOM 나던 것 해소.
+        self._col_l1_flat = array.array("I")     # [s,e,cid,aid,cityid] * cnt
+        self._col_l1_starts = array.array("I")   # bisect 용 start 컬럼
+        self._col_l1_cnt = 0
+        self._col_l2: dict[int, "array.array"] = {}        # fid -> [s,e]*rc
+        self._col_l2_starts: dict[int, "array.array"] = {}  # fid -> start 컬럼
+        # L3 는 수천만 range(56M+) — materialize 하면 startup 15s+/560MB. mmap 위 key-bisect 로 무적재.
+        self._col_l3_off = 0
+        self._col_l3_cnt = 0
+        self._parse_col_l1(mm)
+        self._parse_col_l2(mm)
+        self._parse_col_l3(mm)
+        lic_off = secs.get(9, (0, 0))[0]
+        try:
+            self._license = License.unpack(bytes(mm[lic_off:lic_off + LICENSE_BLOB_SIZE])) if lic_off else None
+        except Exception:
+            self._license = None
+        self._metadata = Metadata(version=(2, vmin), range_count=self._col_l1_cnt, license=self._license, file_size=file_size)
+
+    def _parse_col_l1(self, mm: mmap.mmap) -> None:
+        if 1 not in self._col_sections:
+            return
+        off = self._col_sections[1][0]
+        cnt = struct.unpack_from("<I", mm, off)[0]
+        p = off + 4
+        flat = array.array("I")
+        flat.frombytes(memoryview(mm)[p:p + cnt * 20])   # 제로카피 벌크 로드
+        self._le(flat)
+        self._col_l1_flat = flat
+        self._col_l1_starts = flat[0::5]                 # compact start 컬럼(array 복사)
+        self._col_l1_cnt = cnt
+        p += cnt * 20
+        ncnt = struct.unpack_from("<I", mm, p)[0]
+        p += 4
+        for _ in range(ncnt):
+            ln = struct.unpack_from("<H", mm, p)[0]
+            p += 2
+            self._col_names.append(bytes(mm[p:p + ln]).decode("utf-8", "replace"))
+            p += ln
+
+    def _parse_col_l2(self, mm: mmap.mmap) -> None:
+        if 2 not in self._col_sections:
+            return
+        off = self._col_sections[2][0]
+        fc = struct.unpack_from("<I", mm, off)[0]
+        dir_off = off + 4
+        blob_off = dir_off + fc * 9
+        for i in range(fc):
+            fid, rel, rc = struct.unpack_from("<BII", mm, dir_off + i * 9)
+            rp = blob_off + rel
+            flat = array.array("I")
+            flat.frombytes(memoryview(mm)[rp:rp + rc * 8])   # [s,e]*rc
+            self._le(flat)
+            self._col_l2[fid] = flat
+            self._col_l2_starts[fid] = flat[0::2]
+
+    def _parse_col_l3(self, mm: mmap.mmap) -> None:
+        # 무적재: offset/cnt 만. 조회 시 mmap 위에서 key-bisect (아래 _col_get).
+        if 3 not in self._col_sections:
+            return
+        off = self._col_sections[3][0]
+        self._col_l3_cnt = struct.unpack_from("<I", mm, off)[0]
+        self._col_l3_off = off + 4   # entry i: <IIBB> at off+4 + i*10
+
+    def _col_name(self, i: int) -> str:
+        return self._col_names[i] if 0 <= i < len(self._col_names) else ""
+
+    def _col_flag(self, fid: int, n: int) -> bool:
+        starts = self._col_l2_starts.get(fid)
+        if not starts:
+            return False
+        i = bisect.bisect_right(starts, n) - 1
+        if i < 0:
+            return False
+        return n <= self._col_l2[fid][i * 2 + 1]   # end
+
+    def _col_get(self, n: int) -> dict[str, Any] | None:
+        i = bisect.bisect_right(self._col_l1_starts, n) - 1
+        if i < 0:
+            return None
+        flat = self._col_l1_flat
+        base = i * 5
+        if n > flat[base + 1]:   # end
+            return None
+        out: dict[str, Any] = {
+            "country_code":  self._col_name(flat[base + 2]),
+            "as_name":       self._col_name(flat[base + 3]),
+            "score":         0,
+            "score_inbound": 0,
+        }
+        for fid, key in self._COL_FLAG_KEY.items():
+            if self._col_flag(fid, n):
+                out[key] = True
+        if self._col_l3_cnt:
+            j = self._col_l3_bisect(n)   # start <= n 인 마지막 range (mmap 직접, 무적재)
+            if j >= 0:
+                s, e, so, si = struct.unpack_from("<IIBB", self._mm, self._col_l3_off + j * 10)
+                if n <= e:
+                    out["score"] = so
+                    out["score_inbound"] = si
+        return out
+
+    def _col_l3_bisect(self, n: int) -> int:
+        """L3 start 컬럼(mmap, 10byte stride) 위 bisect_right-1. 버전 무관 수동 이진탐색(~26회)."""
+        mm, base = self._mm, self._col_l3_off
+        lo, hi = 0, self._col_l3_cnt
+        while lo < hi:
+            mid = (lo + hi) >> 1
+            if n < struct.unpack_from("<I", mm, base + mid * 10)[0]:
+                hi = mid
+            else:
+                lo = mid + 1
+        return lo - 1

ctidb_bin-0.0.1.dist-info/METADATA

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: ctidb-bin
+Version: 0.0.1
+Summary: CriminalIP packed binary CTI DB reader (v2/v3 format)
+Project-URL: Homepage, https://github.com/aispera/ctidb-bin
+Author-email: AI Spera <infra@aispera.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: criminalip,ctidb,packed-binary,threat-intelligence
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Security
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: fast
+Requires-Dist: pyroaring>=1.0; extra == 'fast'
+Requires-Dist: zstandard>=0.22; extra == 'fast'
+Description-Content-Type: text/markdown
+
+# ctidb-bin
+
+**CriminalIP packed binary CTI DB reader** — packed BIN v2/v3 포맷 전용 리더 (read-only, zero-copy mmap).
+
+> 기존 MMDB 포맷 리더가 필요하면 [`ctidb`](https://pypi.org/project/ctidb/) 패키지를 사용하세요.
+> `ctidb-bin`은 신규 packed binary 포맷(v2/v3) 전용입니다.
+
+## 설치
+
+```bash
+pip install ctidb-bin
+```
+
+코어는 순수 파이썬(무의존)입니다. v3 가속 기능(L2 bitmap / cold record 압축)은 선택 설치:
+
+```bash
+pip install "ctidb-bin[fast]"   # pyroaring, zstandard
+```
+
+## 사용법
+
+```python
+from ctidb_bin import PackedReader
+
+# with 구문 (권장 — 자동 close)
+with PackedReader.open("criminalip.ctidb.c.bin") as reader:
+    rec = reader.get("8.8.8.8")
+    if rec:
+        print(rec["country_code"], rec["as_name"])
+        print("hosting:", rec.get("is_hosting"), "cloud:", rec.get("is_cloud"))
+        print("score in/out:", rec.get("score_inbound"), rec.get("score"))
+    else:
+        print("not found")   # DB에 없는 IP
+```
+
+`get(ip)`는 IP 레코드를 `dict`로 반환하며, DB에 없으면 `None`입니다.
+
+## 반환 필드 (예시)
+
+| 필드 | 설명 |
+|---|---|
+| `country_code`, `as_name` | 지리/AS 정보 |
+| `is_vpn`·`is_tor`·`is_proxy`·`is_hosting`·`is_cloud`·`is_mobile` | etcs 태그 |
+| `score`·`score_inbound` | outbound/inbound 위협 점수 (0–5) |
+| `is_c2`, `abuse_record`, `ssl_cn` | 부가 신호 |
+
+## 지원 포맷
+
+- **v2 (columnar)** — 현행 col 레이아웃
+- **v3** — L2 bitmap / cold record (`[fast]` extras 권장)
+- v1.x range-table 레이아웃도 하위호환 읽기 지원
+
+## 라이선스
+
+Apache-2.0 · © AI Spera

ctidb_bin-0.0.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+ctidb_bin/__init__.py,sha256=aQh2n00Pl8TIDt6C18sKfa801kJbHntXF9oa5wPRh64,379
+ctidb_bin/format.py,sha256=q0F8KuBab1OtNnsELUp_CsW2V07ZArlmqZJ7uTx0JMI,8130
+ctidb_bin/name_table.py,sha256=NTFT8PjPv8wvbQGlE1YryViM7bnoCK0IK0QZPFsIMF4,3085
+ctidb_bin/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ctidb_bin/reader.py,sha256=PJgyJmk6n2Fw_q6vd0EjOWJZjYMDuurkPQeXVw1XRV8,13542
+ctidb_bin-0.0.1.dist-info/METADATA,sha256=By10ySYLomVF44WPNnu9yJL5l_Hwieufx6t-O97plmc,2453
+ctidb_bin-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ctidb_bin-0.0.1.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+ctidb_bin-0.0.1.dist-info/RECORD,,

ctidb_bin-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

ctidb_bin-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.