symbolicator-server 0.0.1__tar.gz

symbolicator_server-0.0.1/.github/workflows/python-app.yml

@@ -0,0 +1,35 @@
+name: Python App
+
+on:
+  push:
+    branches: [ '**' ]
+  pull_request:
+    branches: [ 'main' ]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: [ '3.14' ]
+        os: [ macos-latest, ubuntu-latest ]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install pre-commit
+      - name: Run pre-commit hooks
+        run: |
+          pre-commit run --all-files
+      - name: Test install
+        run: |
+          python -m pip install -U .
+      - name: Test show usage
+        run: symbolicator-server --help

symbolicator_server-0.0.1/.github/workflows/python-publish.yml

@@ -0,0 +1,33 @@
+# These workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [ created ]
+
+jobs:
+  deploy:
+
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.14'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U build setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python -m build
+        twine upload dist/*

symbolicator_server-0.0.1/.gitignore

@@ -0,0 +1,50 @@
+# Object files
+
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+*.o
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Caches
+__pycache__
+_CodeSignature
+*.egg-info
+
+# Debug files
+*.dSYM/
+
+# macOS shit
+.DS_Store
+
+# ide files
+.vscode
+.idea
+
+# Build
+**/_version.py
+/tests/.pytest_cache/
+dist/

symbolicator_server-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+# .pre-commit-config.yaml
+repos:
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.9.3
+    hooks:
+      - id: isort
+        args: [ '-m', 'HANGING_INDENT', '-l', '120','--check-only' ]
+        files: .
+
+  - repo: https://github.com/pycqa/flake8
+    rev: "7.0.0"
+    hooks:
+      - id: flake8
+        args: [ '--max-line-length=127' ]
+        files: \.py$

symbolicator_server-0.0.1/PKG-INFO

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: symbolicator-server
+Version: 0.0.1
+Summary: A Python stand-in for ipswd's symbol server
+Author-email: netanelc305 <netanelc305@protonmail.com>
+Maintainer-email: netanelc305 <netanelc305@protonmail.com>
+Project-URL: Homepage, https://github.com/netanelc305/symbolicator-server
+Project-URL: Bug Reports, https://github.com/netanelc305/symbolicator-server/issues
+Keywords: ipsw,ipswd,symbolication,symbols,dyld_shared_cache,server
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.162.2
+Requires-Dist: uvicorn>=0.54.0
+
+# symbolicator-server
+
+A Python stand-in for ipswd's symbol server. It scans IPSWs into a SQLite
+database and serves the same `/v1/...` API, so `ipsw symbolicate --server`
+works against it.
+
+
+## How it differs from ipswd
+
+The API and the answers are the same: a lookup returns the same symbol ipswd
+would, including which one wins when several cover an address (the first one
+inserted). What changes is how symbols are stored and how scans are fed in.
+
+**Symbol storage**
+
+| | ipswd | symbolicator-server |
+|---|---|---|
+| Rows per symbol | 2: `symbols (id, name_id, start, end)` plus `macho_syms (macho_uuid, symbol_id)` | 1: `symbols (macho_id, off, seq, size, name_code)`, `WITHOUT ROWID` |
+| Mach-O reference | The 36-character UUID text, once per symbol | A small integer id |
+| Addresses | Absolute `start` and `end` | Offset from the Mach-O's `__TEXT` start, plus a size. Most values fit in 2-4 bytes |
+| Unnamed functions (`func_<addr>`) | A row in `names` each | No name row. Rebuilt from the address when read |
+| Same Mach-O seen again (the same shared cache in another device's IPSW, or a rescan) | All of its symbols are inserted again with new ids | Its symbol list is fingerprinted. A list already stored is skipped, and a list with new symbols only appends the new ones |
+
+## Requirements
+
+- Python 3.14
+- The [`ipsw`](https://github.com/blacktop/ipsw) CLI on your `PATH`, which does the scanning (`ipsw symbols --json`)
+
+## Install
+
+```shell
+python3 -m pip install -U .
+```
+
+## Run the server
+
+```shell
+symbolicator-server --db syms.db
+```
+
+or `python3 -m symbolicator_server --db syms.db`.
+
+| Option | Default | |
+|---|---|---|
+| `--db` | (required) | SQLite database file, created if missing |
+| `--host` / `--port` | `localhost` / `3993` | Where to listen |
+| `--sigs-dir` | | Default kernel signatures dir for scans |
+| `--pem-db` | | Default AEA pem DB for scans |
+| `--ipsw-bin` | `ipsw` on `PATH` | The ipsw CLI used to scan |
+| `-v`, `--verbose` | | Log every image of a scan |
+
+## Scan an IPSW
+
+```shell
+curl -X POST "http://localhost:3993/v1/syms/scan?path=/PATH/TO/IPSW.ipsw"
+```
+
+The scan runs as a single transaction, and lookups keep working while it runs.
+Scanning an IPSW that is already in the database returns `409`. To scan it
+again and merge in anything new, use `PUT /v1/syms/rescan?path=...`. Both
+accept optional `sig_dir` and `pem_db` query parameters.
+
+## Symbolicate a crash
+
+```shell
+ipsw symbolicate --server http://localhost:3993 CRASH-REPORT.ips --proc "SpringBoard"
+```

symbolicator_server-0.0.1/README.md

@@ -0,0 +1,67 @@
+# symbolicator-server
+
+A Python stand-in for ipswd's symbol server. It scans IPSWs into a SQLite
+database and serves the same `/v1/...` API, so `ipsw symbolicate --server`
+works against it.
+
+
+## How it differs from ipswd
+
+The API and the answers are the same: a lookup returns the same symbol ipswd
+would, including which one wins when several cover an address (the first one
+inserted). What changes is how symbols are stored and how scans are fed in.
+
+**Symbol storage**
+
+| | ipswd | symbolicator-server |
+|---|---|---|
+| Rows per symbol | 2: `symbols (id, name_id, start, end)` plus `macho_syms (macho_uuid, symbol_id)` | 1: `symbols (macho_id, off, seq, size, name_code)`, `WITHOUT ROWID` |
+| Mach-O reference | The 36-character UUID text, once per symbol | A small integer id |
+| Addresses | Absolute `start` and `end` | Offset from the Mach-O's `__TEXT` start, plus a size. Most values fit in 2-4 bytes |
+| Unnamed functions (`func_<addr>`) | A row in `names` each | No name row. Rebuilt from the address when read |
+| Same Mach-O seen again (the same shared cache in another device's IPSW, or a rescan) | All of its symbols are inserted again with new ids | Its symbol list is fingerprinted. A list already stored is skipped, and a list with new symbols only appends the new ones |
+
+## Requirements
+
+- Python 3.14
+- The [`ipsw`](https://github.com/blacktop/ipsw) CLI on your `PATH`, which does the scanning (`ipsw symbols --json`)
+
+## Install
+
+```shell
+python3 -m pip install -U .
+```
+
+## Run the server
+
+```shell
+symbolicator-server --db syms.db
+```
+
+or `python3 -m symbolicator_server --db syms.db`.
+
+| Option | Default | |
+|---|---|---|
+| `--db` | (required) | SQLite database file, created if missing |
+| `--host` / `--port` | `localhost` / `3993` | Where to listen |
+| `--sigs-dir` | | Default kernel signatures dir for scans |
+| `--pem-db` | | Default AEA pem DB for scans |
+| `--ipsw-bin` | `ipsw` on `PATH` | The ipsw CLI used to scan |
+| `-v`, `--verbose` | | Log every image of a scan |
+
+## Scan an IPSW
+
+```shell
+curl -X POST "http://localhost:3993/v1/syms/scan?path=/PATH/TO/IPSW.ipsw"
+```
+
+The scan runs as a single transaction, and lookups keep working while it runs.
+Scanning an IPSW that is already in the database returns `409`. To scan it
+again and merge in anything new, use `PUT /v1/syms/rescan?path=...`. Both
+accept optional `sig_dir` and `pem_db` query parameters.
+
+## Symbolicate a crash
+
+```shell
+ipsw symbolicate --server http://localhost:3993 CRASH-REPORT.ips --proc "SpringBoard"
+```

symbolicator_server-0.0.1/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "symbolicator-server"
+description = "A Python stand-in for ipswd's symbol server"
+readme = "README.md"
+requires-python = ">=3.14"
+keywords = ["ipsw", "ipswd", "symbolication", "symbols", "dyld_shared_cache", "server"]
+authors = [
+    { name = "netanelc305", email = "netanelc305@protonmail.com" }
+]
+maintainers = [
+    { name = "netanelc305", email = "netanelc305@protonmail.com" }
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3 :: Only",
+]
+dynamic = ["dependencies", "version"]
+
+[project.urls]
+"Homepage" = "https://github.com/netanelc305/symbolicator-server"
+"Bug Reports" = "https://github.com/netanelc305/symbolicator-server/issues"
+
+[project.scripts]
+symbolicator-server = "symbolicator_server.__main__:main"
+
+[tool.setuptools.packages.find]
+exclude = ["docs*", "tests*"]
+
+[tool.setuptools.dynamic]
+dependencies = { file = ["requirements.txt"] }
+version = { attr = "symbolicator_server._version.__version__" }
+
+[tool.setuptools_scm]
+version_file = "symbolicator_server/_version.py"
+
+[build-system]
+requires = ["setuptools>=43.0.0", "setuptools_scm>=8", "wheel"]
+build-backend = "setuptools.build_meta"

symbolicator_server-0.0.1/requirements.txt

@@ -0,0 +1,2 @@
+fastapi>=0.162.2
+uvicorn>=0.54.0

symbolicator_server-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

symbolicator_server-0.0.1/symbolicator_server/__main__.py

@@ -0,0 +1,4 @@
+from symbolicator_server.server import main
+
+if __name__ == "__main__":
+    main()

symbolicator_server-0.0.1/symbolicator_server/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = 'g026bd4807'

symbolicator_server-0.0.1/symbolicator_server/scanner.py

@@ -0,0 +1,99 @@
+"""
+Scanning: run the same scanner ipswd uses and feed its output into a store.
+
+Taking an IPSW apart (decrypting the AEA cryptexes, mounting the APFS volumes,
+parsing the dyld_shared_cache and kernelcache, finding every function) is done
+by `ipsw symbols --json`, the scan code inside ipswd, exposed as a command.
+It streams one JSON record per line, grouped per Mach-O:
+
+    {"type":"ipsw",   "id":<sha1>, "name", "version", "build", "platform", "devices":[...]}
+    {"type":"dsc",    "uuid", "shared_region_start"}                   once per shared cache
+    {"type":"image",  "uuid", "kind", "path", "text_start", "text_end", "dsc_uuid"?}
+    {"type":"symbol", "image_uuid", "name", "start", "end"}            ...that image's symbols
+
+Each image is handed to the store as soon as its symbols have arrived (memory
+stays bounded), and the whole scan is one transaction, like ipswd's.
+"""
+
+import json
+import logging
+import subprocess
+import tempfile
+import time
+
+log = logging.getLogger("scanner")
+
+
+class AlreadyScanned(Exception):
+    pass
+
+
+class ScanFailed(Exception):
+    pass
+
+
+def scan(store, ipsw_path, *, ipsw_bin="ipsw", sigs_dir=None, pem_db=None, rescan=False):
+    cmd = [ipsw_bin, "symbols", "--json", ipsw_path]
+    if sigs_dir:
+        cmd += ["--signatures", sigs_dir]
+    if pem_db:
+        cmd += ["--pem-db", pem_db]
+    log.info("running %s", " ".join(cmd))
+
+    started = time.time()
+    counts = {"images": 0, "symbols": 0, "skipped": 0}
+    conn = store.connect()
+    with tempfile.TemporaryFile("w+") as stderr:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, text=True, bufsize=1 << 20)
+        try:
+            ipsw_id = None
+            image, symbols = None, []
+
+            def flush():
+                # Hand the finished image and its symbols to the store.
+                if image is None:
+                    return
+                note = store.save_image(conn, ipsw_id, image, symbols)
+                counts["images"] += 1
+                counts["symbols"] += len(symbols)
+                counts["skipped"] += note.startswith("already stored")
+                log.debug("%s %s: %s", image["kind"], image["path"], note)
+
+            for line in proc.stdout:
+                rec = json.loads(line)
+                kind = rec["type"]
+                if kind == "ipsw":
+                    ipsw_id = rec["id"]
+                    if not rescan and store.has_ipsw(conn, ipsw_id):
+                        raise AlreadyScanned(f"IPSW {rec['name']} ({ipsw_id}) is already in the database")
+                    store.save_ipsw(conn, rec)
+                    log.info("scanning %s %s (%s) for %s",
+                             rec["name"], rec["version"], rec["build"], ", ".join(rec.get("devices", [])))
+                elif kind == "dsc":
+                    store.save_dsc(conn, ipsw_id, rec)
+                elif kind == "image":
+                    flush()
+                    image, symbols = rec, []
+                elif kind == "symbol":
+                    symbols.append((rec["start"], rec["end"], rec["name"]))
+                # other record types (comparison facts) are not part of the symbol database
+            flush()
+
+            if proc.wait() != 0:
+                stderr.seek(0)
+                raise ScanFailed(stderr.read().strip()[-2000:] or f"ipsw exited with {proc.returncode}")
+            conn.commit()
+        except BaseException:
+            conn.rollback()
+            proc.kill()
+            proc.wait()
+            raise
+        finally:
+            conn.close()
+
+    counts["seconds"] = round(time.time() - started, 1)
+    log.info(
+        "scan done in %ss: %d images, %d symbols, %d images already stored",
+        counts["seconds"], counts["images"], counts["symbols"], counts["skipped"],
+    )
+    return counts

symbolicator_server-0.0.1/symbolicator_server/server.py

@@ -0,0 +1,170 @@
+"""
+A Python stand-in for ipswd's symbol server, storing one all-integer row per
+symbol (see store.py).
+
+    symbolicator-server --db syms.db
+
+Same endpoints as ipswd (/v1/...), so `ipsw symbolicate --server` works against it:
+
+    POST /v1/syms/scan?path=<ipsw>[&sig_dir=][&pem_db=]   scan an IPSW into the database
+    PUT  /v1/syms/rescan?path=<ipsw>                      scan again, replacing what changed
+    GET  /v1/_ping
+    GET  /v1/syms/ipsw?version=&build=&device=            is this IPSW scanned?
+    GET  /v1/syms/macho/{uuid}                            a Mach-O (path, __TEXT range)
+    GET  /v1/syms/dsc/{uuid}                              a dyld_shared_cache
+    GET  /v1/syms/dsc/{uuid}/{addr}                       the dylib of a shared cache containing addr
+    GET  /v1/syms/{uuid}/{addr}                           the symbol at addr
+    GET  /v1/syms/{uuid}                                  all symbols of a Mach-O
+
+Plus one that ipswd does not have:
+
+    GET  /v1/stats                                        database size per table
+"""
+
+import argparse
+import logging
+import os
+import shutil
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, PlainTextResponse
+
+from symbolicator_server import scanner
+from symbolicator_server.store import Store, symbol_json
+
+app = FastAPI(title="symbolicator-server")
+STORE = None  # set in main()
+CONFIG = {}  # ipsw binary, default signatures dir and pem DB
+
+
+def error(status, message):
+    return JSONResponse(status_code=status, content={"error": message})
+
+
+def parse_addr(addr):
+    """ipswd parses the address with cast.ToUint64: decimal or 0x-hex, anything else is 0.
+
+    SQLite integers are signed 64-bit and stored addresses are below 2**63, so a
+    larger address can match nothing: it becomes -1 rather than failing to bind."""
+    try:
+        n = int(addr, 0)
+    except ValueError:
+        return 0
+    return n if 0 <= n < 1 << 63 else -1
+
+
+@app.get("/v1/_ping", response_class=PlainTextResponse)
+def ping():
+    return "OK"
+
+
+# --------------------------------------------------------------------- scan
+
+
+def run_scan(request: Request, rescan: bool):
+    q = request.query_params
+    path = q.get("path")
+    if not path:
+        return error(400, "missing path query parameter")
+    if q.get("device"):
+        return error(400, "device-scoped database ingestion is not supported; use device-scoped JSONL export instead")
+    try:
+        scanner.scan(
+            STORE,
+            os.path.normpath(path),
+            ipsw_bin=CONFIG["ipsw_bin"],
+            sigs_dir=q.get("sig_dir") or CONFIG["sigs_dir"],
+            pem_db=q.get("pem_db") or CONFIG["pem_db"],
+            rescan=rescan,
+        )
+    except scanner.AlreadyScanned as e:
+        return error(409, str(e))
+    except Exception as e:  # noqa: BLE001 - ipswd answers 500 with the error text
+        logging.exception("scan failed")
+        return error(500, str(e))
+    return JSONResponse(status_code=201, content={"created": True}) if rescan else {"success": True}
+
+
+# Plain `def` routes run in FastAPI's thread pool, so lookups keep working during a scan.
+@app.post("/v1/syms/scan")
+def scan(request: Request):
+    return run_scan(request, rescan=False)
+
+
+@app.put("/v1/syms/rescan")
+def rescan(request: Request):
+    return run_scan(request, rescan=True)
+
+
+# ------------------------------------------------------------------- lookups
+# Order matters: the fixed paths (ipsw, macho, dsc) must come before /syms/{uuid}.
+
+
+@app.get("/v1/syms/ipsw")
+def get_ipsw(version: str = "", build: str = "", device: str = ""):
+    if not (version and build and device):
+        return error(400, "version, build and device are required")
+    ipsw = STORE.get_ipsw(STORE.reader(), version, build, device)
+    return ipsw if ipsw else error(404, "not found")
+
+
+@app.get("/v1/syms/macho/{uuid}")
+def get_macho(uuid: str):
+    macho = STORE.get_macho(STORE.reader(), uuid)
+    return macho if macho else error(404, "not found")
+
+
+@app.get("/v1/syms/dsc/{uuid}")
+def get_dsc(uuid: str):
+    dsc = STORE.get_dsc(STORE.reader(), uuid)
+    return dsc if dsc else error(404, "not found")
+
+
+@app.get("/v1/syms/dsc/{uuid}/{addr}")
+def get_dsc_image(uuid: str, addr: str):
+    dylib = STORE.get_dsc_image(STORE.reader(), uuid, parse_addr(addr))
+    return dylib if dylib else error(404, "not found")
+
+
+@app.get("/v1/stats")
+def stats():
+    return STORE.stats(STORE.reader())
+
+
+@app.get("/v1/syms/{uuid}/{addr}")
+def get_symbol(uuid: str, addr: str):
+    sym = STORE.get_symbol(STORE.reader(), uuid, parse_addr(addr))
+    return symbol_json(*sym) if sym else error(404, "not found")
+
+
+@app.get("/v1/syms/{uuid}")
+def get_symbols(uuid: str):
+    syms = STORE.get_symbols(STORE.reader(), uuid)
+    return [symbol_json(*s) for s in syms]
+
+
+# ---------------------------------------------------------------------- main
+
+
+def main():
+    global STORE
+    p = argparse.ArgumentParser(description="ipswd-compatible symbol server")
+    p.add_argument("--db", required=True, help="sqlite database file")
+    p.add_argument("--host", default="localhost")
+    p.add_argument("--port", type=int, default=3994)
+    p.add_argument("--sigs-dir", help="default kernel signatures dir (ipswd: daemon.sigs-dir)")
+    p.add_argument("--pem-db", help="default AEA pem DB (ipswd: daemon.pem-db)")
+    p.add_argument("--ipsw-bin", default=shutil.which("ipsw") or "ipsw", help="the ipsw CLI used to scan")
+    p.add_argument("-v", "--verbose", action="store_true", help="log every image of a scan")
+    args = p.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(name)s: %(message)s")
+    STORE = Store(args.db)
+    CONFIG.update(ipsw_bin=args.ipsw_bin, sigs_dir=args.sigs_dir, pem_db=args.pem_db)
+    logging.info("database %s, scanning with %s", args.db, args.ipsw_bin)
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

symbolicator_server-0.0.1/symbolicator_server/store.py

@@ -0,0 +1,385 @@
+"""
+Symbol storage: normalized tables, one all-integer row per symbol.
+
+    ipsws, ipsw_devices, dyld_shared_caches, ipsw_dscs,
+    machos, ipsw_machos, dsc_images                               what was scanned
+    names     (id, name)                                          name -> id
+    macho_ids (id, uuid, base, max_size, next_seq)                uuid -> small integer id
+    symbols   (macho_id, off, seq, size, name_code)  WITHOUT ROWID one row per symbol
+    symbol_lists (macho_id, list_hash)                            fingerprints of merged lists
+
+Why it is small:
+  * The Mach-O is referenced by a small integer instead of its 36-character UUID.
+  * The address is stored as an offset from the Mach-O's __TEXT start and the
+    end as a size, so most values fit in 2-4 bytes (SQLite stores integers in
+    1-8 bytes as needed).
+  * WITHOUT ROWID: the rows live directly in the primary-key B-tree, so there
+    is no second index holding the same columns again.
+  * A list a scan delivers is fingerprinted first; a list already merged
+    (another device's IPSW, a rescan) writes nothing.
+  * Placeholder names (func_<address>) are not stored (name_code 0 / 1).
+
+Same answers as ipswd: a Mach-O's symbols only grow (new lists are merged in),
+and when several symbols cover an address the first inserted (lowest seq)
+wins, like ipswd's ORDER BY id.
+
+The schema is a simplified version of ipswd's. The kernelcache -> kext and
+ipsw -> file-system links are folded into one ipsw_machos table, because they
+do not affect how symbols are stored.
+"""
+
+import hashlib
+import json
+import os
+import sqlite3
+import threading
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS ipsws (
+    id       TEXT PRIMARY KEY,          -- sha1 of the .ipsw file
+    name     TEXT,
+    version  TEXT,
+    build    TEXT,
+    platform TEXT
+);
+CREATE TABLE IF NOT EXISTS ipsw_devices (
+    ipsw_id TEXT,
+    device  TEXT,
+    PRIMARY KEY (ipsw_id, device)
+);
+CREATE TABLE IF NOT EXISTS dyld_shared_caches (
+    uuid                TEXT PRIMARY KEY,
+    shared_region_start INTEGER
+);
+CREATE TABLE IF NOT EXISTS ipsw_dscs (
+    ipsw_id  TEXT,
+    dsc_uuid TEXT,
+    PRIMARY KEY (ipsw_id, dsc_uuid)
+);
+CREATE TABLE IF NOT EXISTS machos (
+    uuid       TEXT PRIMARY KEY,
+    kind       TEXT,                    -- kernel / kext / dylib / macho
+    path       TEXT,
+    text_start INTEGER,
+    text_end   INTEGER
+);
+CREATE TABLE IF NOT EXISTS ipsw_machos (
+    ipsw_id    TEXT,
+    macho_uuid TEXT,
+    PRIMARY KEY (ipsw_id, macho_uuid)
+);
+CREATE TABLE IF NOT EXISTS dsc_images (
+    dsc_uuid   TEXT,
+    macho_uuid TEXT,
+    PRIMARY KEY (dsc_uuid, macho_uuid)
+);
+CREATE TABLE IF NOT EXISTS names (
+    id   INTEGER PRIMARY KEY,
+    name TEXT UNIQUE
+);
+CREATE TABLE IF NOT EXISTS macho_ids (
+    id       INTEGER PRIMARY KEY,
+    uuid     TEXT UNIQUE,
+    base     INTEGER,      -- the Mach-O's __TEXT start; symbol addresses are stored relative to it
+    max_size INTEGER,      -- its largest symbol; bounds how far back a lookup has to look
+    next_seq INTEGER       -- insertion counter: keeps "first inserted wins" like the old ids
+);
+CREATE TABLE IF NOT EXISTS symbols (
+    macho_id  INTEGER,
+    off       INTEGER,     -- start - base
+    seq       INTEGER,     -- insertion order within the Mach-O
+    size      INTEGER,     -- end - start
+    name_code INTEGER,     -- 0/1 = func_ placeholder, otherwise names.id + 2
+    PRIMARY KEY (macho_id, off, seq)
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS symbol_lists (
+    macho_id  INTEGER,
+    list_hash BLOB,
+    PRIMARY KEY (macho_id, list_hash)
+) WITHOUT ROWID;
+"""
+
+HIGH_BIT = 1 << 63  # cleared from kernel addresses, but still part of their func_ names
+
+# name_code: 0 and 1 are placeholders, real names are names.id + 2.
+NAME_FUNC = 0  # func_<start>                (dylibs, file-system Mach-Os)
+NAME_FUNC_KERNEL = 1  # func_<start | bit 63>       (kernel and kexts)
+FIRST_REAL_NAME = 2
+
+LIST_HASH_VERSION = b"\x02"  # part of every stored fingerprint; changing it re-merges every list
+
+
+def placeholder_code(name, start):
+    """NAME_FUNC / NAME_FUNC_KERNEL if name is the func_ placeholder for start, else None."""
+    if name.startswith("func_"):
+        if name == f"func_{start:x}":
+            return NAME_FUNC
+        if name == f"func_{start | HIGH_BIT:x}":
+            return NAME_FUNC_KERNEL
+    return None
+
+
+def decode_name(code, start, name):
+    """(name_id, name) from a stored name_code and the name joined from the names table."""
+    if code == NAME_FUNC:
+        return None, f"func_{start:x}"
+    if code == NAME_FUNC_KERNEL:
+        return None, f"func_{start | HIGH_BIT:x}"
+    return code - FIRST_REAL_NAME, name
+
+
+def list_hash(symbols):
+    """Fingerprint of one symbol list as a scan delivered it (addresses + name text, in order)."""
+    text = "".join(f"{start}:{end}:{name}\n" for start, end, name in symbols)
+    return hashlib.sha256(LIST_HASH_VERSION + text.encode()).digest()
+
+
+# Symbols of one Mach-O as (name_id, name, start, end), by address then insertion order.
+SYMBOLS_SQL = """
+    SELECT m.base + s.off, s.size, s.name_code, n.name
+    FROM macho_ids m
+    JOIN symbols s     ON s.macho_id = m.id
+    LEFT JOIN names n  ON n.id = s.name_code - 2
+    WHERE m.uuid = ?
+    ORDER BY s.off, s.seq
+"""
+
+# Only symbols starting within max_size before addr can cover it: a short range
+# scan of the primary key, then the first inserted among the matches.
+SYMBOL_AT_SQL = """
+    SELECT m.base + s.off, s.size, s.name_code, n.name
+    FROM macho_ids m
+    JOIN symbols s     ON s.macho_id = m.id
+                      AND s.off BETWEEN :addr - m.base - m.max_size AND :addr - m.base
+    LEFT JOIN names n  ON n.id = s.name_code - 2
+    WHERE m.uuid = :uuid AND :addr - m.base < s.off + s.size
+    ORDER BY s.seq LIMIT 1
+"""
+
+
+class Store:
+    def __init__(self, path):
+        self.path = path
+        self._local = threading.local()
+        with self.connect() as conn:
+            conn.execute("PRAGMA journal_mode = WAL")  # persistent: lookups keep working while a scan writes
+            conn.executescript(SCHEMA)
+
+    def connect(self):
+        conn = sqlite3.connect(self.path, timeout=60)
+        conn.execute("PRAGMA synchronous = NORMAL")  # durable enough under WAL, much faster commits
+        conn.execute("PRAGMA cache_size = -65536")  # 64 MiB page cache
+        conn.execute("PRAGMA mmap_size = 1073741824")  # read pages straight from the mapped file
+        return conn
+
+    def reader(self):
+        """A connection for lookups, one per thread and kept open (FastAPI reuses its pool threads)."""
+        conn = getattr(self._local, "conn", None)
+        if conn is None:
+            conn = self._local.conn = self.connect()
+        return conn
+
+    # ------------------------------------------------------------------ saving
+
+    def has_ipsw(self, conn, ipsw_id):
+        return conn.execute("SELECT 1 FROM ipsws WHERE id = ?", (ipsw_id,)).fetchone() is not None
+
+    def save_ipsw(self, conn, rec):
+        conn.execute(
+            "INSERT OR REPLACE INTO ipsws (id, name, version, build, platform) VALUES (?, ?, ?, ?, ?)",
+            (rec["id"], rec["name"], rec["version"], rec["build"], rec.get("platform")),
+        )
+        conn.executemany(
+            "INSERT OR IGNORE INTO ipsw_devices (ipsw_id, device) VALUES (?, ?)",
+            [(rec["id"], dev) for dev in rec.get("devices", [])],
+        )
+
+    def save_dsc(self, conn, ipsw_id, rec):
+        conn.execute(
+            "INSERT OR IGNORE INTO dyld_shared_caches (uuid, shared_region_start) VALUES (?, ?)",
+            (rec["uuid"], rec["shared_region_start"]),
+        )
+        conn.execute("INSERT OR IGNORE INTO ipsw_dscs VALUES (?, ?)", (ipsw_id, rec["uuid"]))
+
+    def save_image(self, conn, ipsw_id, image, symbols):
+        """Save one Mach-O and its symbols. symbols: [(start, end, name), ...]. Returns a note for the scan log."""
+        uuid = image["uuid"]
+        conn.execute(
+            "INSERT OR IGNORE INTO machos (uuid, kind, path, text_start, text_end) VALUES (?, ?, ?, ?, ?)",
+            (uuid, image["kind"], image["path"], image["text_start"], image["text_end"]),
+        )
+        conn.execute("INSERT OR IGNORE INTO ipsw_machos VALUES (?, ?)", (ipsw_id, uuid))
+        if image.get("dsc_uuid"):
+            conn.execute("INSERT OR IGNORE INTO dsc_images VALUES (?, ?)", (image["dsc_uuid"], uuid))
+        return self.save_symbols(conn, uuid, symbols)
+
+    def save_symbols(self, conn, uuid, symbols):
+        macho_id, base, max_size, next_seq = self._macho_id(conn, uuid)
+        fingerprint = list_hash(symbols)
+        seen = conn.execute(
+            "SELECT 1 FROM symbol_lists WHERE macho_id = ? AND list_hash = ?", (macho_id, fingerprint)
+        ).fetchone()
+        if seen:
+            return "already stored, skipped"  # this exact list was merged before: nothing to write
+        conn.execute("INSERT INTO symbol_lists (macho_id, list_hash) VALUES (?, ?)", (macho_id, fingerprint))
+
+        # Encode first, then merge in encoded form: stored rows stay, rows not stored yet are
+        # appended after them. Comparing (off, size, name_code) needs no join with names.
+        codes = [placeholder_code(name, start) for start, _, name in symbols]
+        ids = self.name_ids(conn, {sym[2] for sym, code in zip(symbols, codes) if code is None})
+        known = (
+            set(conn.execute("SELECT off, size, name_code FROM symbols WHERE macho_id = ?", (macho_id,)))
+            if next_seq
+            else set()
+        )
+        rows = []
+        for (start, end, name), code in zip(symbols, codes):
+            # an inverted range is stored with size 0 and never matches, as in ipswd
+            key = (start - base, max(end - start, 0), ids[name] + FIRST_REAL_NAME if code is None else code)
+            if key not in known:
+                known.add(key)
+                rows.append((macho_id, key[0], next_seq + len(rows), key[1], key[2]))
+                max_size = max(max_size, key[1])
+        if not rows:
+            return "already stored, skipped" if next_seq else "no symbols"
+
+        conn.executemany("INSERT INTO symbols (macho_id, off, seq, size, name_code) VALUES (?, ?, ?, ?, ?)", rows)
+        conn.execute(
+            "UPDATE macho_ids SET max_size = ?, next_seq = ? WHERE id = ?", (max_size, next_seq + len(rows), macho_id)
+        )
+        if next_seq:
+            return f"merged {len(rows)} new symbols into the {next_seq} stored"
+        return f"inserted {len(rows)} symbol rows"
+
+    def _macho_id(self, conn, uuid):
+        """(id, base, max_size, next_seq) for uuid, creating the id on first sight."""
+        row = conn.execute("SELECT id, base, max_size, next_seq FROM macho_ids WHERE uuid = ?", (uuid,)).fetchone()
+        if row:
+            return row
+        cur = conn.execute(
+            """INSERT INTO macho_ids (uuid, base, max_size, next_seq)
+               SELECT uuid, COALESCE(text_start, 0), 0, 0 FROM machos WHERE uuid = ?
+               RETURNING id, base, max_size, next_seq""",
+            (uuid,),
+        )
+        return cur.fetchone()
+
+    def name_ids(self, conn, names):
+        """Insert the names that are missing and return {name: id}."""
+        if not names:
+            return {}
+        names = json.dumps(list(names))  # one bound parameter, however many names
+        conn.execute("INSERT OR IGNORE INTO names (name) SELECT value FROM json_each(?)", (names,))
+        return {
+            name: nid
+            for nid, name in conn.execute(
+                "SELECT id, name FROM names WHERE name IN (SELECT value FROM json_each(?))", (names,)
+            )
+        }
+
+    # ----------------------------------------------------------------- lookups
+
+    def get_symbol(self, conn, uuid, addr):
+        """(name_id, name, start, end) of the symbol covering addr, or None."""
+        row = conn.execute(SYMBOL_AT_SQL, {"uuid": uuid, "addr": addr}).fetchone()
+        if row is None:
+            return None
+        start, size, code, name = row
+        return *decode_name(code, start, name), start, start + size
+
+    def get_symbols(self, conn, uuid):
+        """[(name_id, name, start, end), ...] sorted by start."""
+        return [
+            (*decode_name(code, start, name), start, start + size)
+            for start, size, code, name in conn.execute(SYMBOLS_SQL, (uuid,))
+        ]
+
+    def get_ipsw(self, conn, version, build, device):
+        row = conn.execute(
+            """SELECT i.id, i.name, i.version, i.build, i.platform FROM ipsws i
+               JOIN ipsw_devices d ON d.ipsw_id = i.id
+               WHERE i.version = ? AND i.build = ? AND d.device = ?""",
+            (version, build, device),
+        ).fetchone()
+        if row is None:
+            return None
+        devices = [d for (d,) in conn.execute("SELECT device FROM ipsw_devices WHERE ipsw_id = ?", (row[0],))]
+        return ipsw_json(row, devices)
+
+    def get_macho(self, conn, uuid):
+        row = conn.execute("SELECT uuid, path, text_start, text_end FROM machos WHERE uuid = ?", (uuid,)).fetchone()
+        return None if row is None else macho_json(row)
+
+    def get_dsc(self, conn, uuid):
+        row = conn.execute("SELECT uuid, shared_region_start FROM dyld_shared_caches WHERE uuid = ?", (uuid,)).fetchone()
+        return None if row is None else {"uuid": row[0], "shared_region_start": row[1]}
+
+    def get_dsc_image(self, conn, dsc_uuid, addr):
+        """The dylib of a shared cache whose __TEXT contains addr."""
+        row = conn.execute(
+            """SELECT m.uuid, m.path, m.text_start, m.text_end FROM machos m
+               JOIN dsc_images di ON di.macho_uuid = m.uuid
+               WHERE di.dsc_uuid = ? AND m.text_start <= ? AND ? < m.text_end LIMIT 1""",
+            (dsc_uuid, addr, addr),
+        ).fetchone()
+        return None if row is None else macho_json(row)
+
+    def stats(self, conn):
+        files = [self.path, self.path + "-wal"]
+        out = {
+            "db": self.path,
+            "size_bytes": sum(os.path.getsize(f) for f in files if os.path.exists(f)),
+            "tables": {},
+        }
+        tables = [t for (t,) in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")]
+        try:  # per-table bytes (tables + their indexes), when sqlite has dbstat compiled in
+            sizes = dict(conn.execute(
+                """SELECT COALESCE(m.tbl_name, s.name), SUM(s.pgsize) FROM dbstat s
+                   LEFT JOIN sqlite_master m ON m.name = s.name GROUP BY 1"""
+            ))
+        except sqlite3.OperationalError:
+            sizes = {}
+        for t in tables:
+            (rows,) = conn.execute(f'SELECT COUNT(*) FROM "{t}"').fetchone()
+            out["tables"][t] = {"rows": rows, "bytes": sizes.get(t)}
+        return out
+
+
+# --------------------------------------------------------------------------
+# JSON shapes, matching how ipswd serializes its Go models
+# (field names are what the `ipsw symbolicate --server` client decodes).
+# --------------------------------------------------------------------------
+
+
+def ipsw_json(row, devices):
+    ipsw_id, name, version, build, platform = row
+    return {
+        "id": ipsw_id,
+        "name": name,
+        "version": version,
+        "buildid": build,
+        "platform": platform,
+        "devices": [{"name": d} for d in devices],
+    }
+
+
+def macho_json(row):
+    uuid, path, text_start, text_end = row
+    return {
+        "uuid": uuid,
+        "PathID": 0,
+        "Path": {"id": 0, "path": path},
+        "text_start": text_start,
+        "text_end": text_end,
+        "Symbols": None,
+    }
+
+
+def symbol_json(name_id, name, start, end):
+    return {
+        "ID": 0,
+        "NameID": name_id,
+        "Name": {"ID": name_id, "name": name},
+        "start": start,
+        "end": end,
+    }

symbolicator_server-0.0.1/symbolicator_server.egg-info/PKG-INFO

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: symbolicator-server
+Version: 0.0.1
+Summary: A Python stand-in for ipswd's symbol server
+Author-email: netanelc305 <netanelc305@protonmail.com>
+Maintainer-email: netanelc305 <netanelc305@protonmail.com>
+Project-URL: Homepage, https://github.com/netanelc305/symbolicator-server
+Project-URL: Bug Reports, https://github.com/netanelc305/symbolicator-server/issues
+Keywords: ipsw,ipswd,symbolication,symbols,dyld_shared_cache,server
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.162.2
+Requires-Dist: uvicorn>=0.54.0
+
+# symbolicator-server
+
+A Python stand-in for ipswd's symbol server. It scans IPSWs into a SQLite
+database and serves the same `/v1/...` API, so `ipsw symbolicate --server`
+works against it.
+
+
+## How it differs from ipswd
+
+The API and the answers are the same: a lookup returns the same symbol ipswd
+would, including which one wins when several cover an address (the first one
+inserted). What changes is how symbols are stored and how scans are fed in.
+
+**Symbol storage**
+
+| | ipswd | symbolicator-server |
+|---|---|---|
+| Rows per symbol | 2: `symbols (id, name_id, start, end)` plus `macho_syms (macho_uuid, symbol_id)` | 1: `symbols (macho_id, off, seq, size, name_code)`, `WITHOUT ROWID` |
+| Mach-O reference | The 36-character UUID text, once per symbol | A small integer id |
+| Addresses | Absolute `start` and `end` | Offset from the Mach-O's `__TEXT` start, plus a size. Most values fit in 2-4 bytes |
+| Unnamed functions (`func_<addr>`) | A row in `names` each | No name row. Rebuilt from the address when read |
+| Same Mach-O seen again (the same shared cache in another device's IPSW, or a rescan) | All of its symbols are inserted again with new ids | Its symbol list is fingerprinted. A list already stored is skipped, and a list with new symbols only appends the new ones |
+
+## Requirements
+
+- Python 3.14
+- The [`ipsw`](https://github.com/blacktop/ipsw) CLI on your `PATH`, which does the scanning (`ipsw symbols --json`)
+
+## Install
+
+```shell
+python3 -m pip install -U .
+```
+
+## Run the server
+
+```shell
+symbolicator-server --db syms.db
+```
+
+or `python3 -m symbolicator_server --db syms.db`.
+
+| Option | Default | |
+|---|---|---|
+| `--db` | (required) | SQLite database file, created if missing |
+| `--host` / `--port` | `localhost` / `3993` | Where to listen |
+| `--sigs-dir` | | Default kernel signatures dir for scans |
+| `--pem-db` | | Default AEA pem DB for scans |
+| `--ipsw-bin` | `ipsw` on `PATH` | The ipsw CLI used to scan |
+| `-v`, `--verbose` | | Log every image of a scan |
+
+## Scan an IPSW
+
+```shell
+curl -X POST "http://localhost:3993/v1/syms/scan?path=/PATH/TO/IPSW.ipsw"
+```
+
+The scan runs as a single transaction, and lookups keep working while it runs.
+Scanning an IPSW that is already in the database returns `409`. To scan it
+again and merge in anything new, use `PUT /v1/syms/rescan?path=...`. Both
+accept optional `sig_dir` and `pem_db` query parameters.
+
+## Symbolicate a crash
+
+```shell
+ipsw symbolicate --server http://localhost:3993 CRASH-REPORT.ips --proc "SpringBoard"
+```

symbolicator_server-0.0.1/symbolicator_server.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+.gitignore
+.pre-commit-config.yaml
+README.md
+pyproject.toml
+requirements.txt
+.github/workflows/python-app.yml
+.github/workflows/python-publish.yml
+symbolicator_server/__init__.py
+symbolicator_server/__main__.py
+symbolicator_server/_version.py
+symbolicator_server/scanner.py
+symbolicator_server/server.py
+symbolicator_server/store.py
+symbolicator_server.egg-info/PKG-INFO
+symbolicator_server.egg-info/SOURCES.txt
+symbolicator_server.egg-info/dependency_links.txt
+symbolicator_server.egg-info/entry_points.txt
+symbolicator_server.egg-info/requires.txt
+symbolicator_server.egg-info/scm_file_list.json
+symbolicator_server.egg-info/scm_version.json
+symbolicator_server.egg-info/top_level.txt

symbolicator_server-0.0.1/symbolicator_server.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

symbolicator_server-0.0.1/symbolicator_server.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+symbolicator-server = symbolicator_server.__main__:main

symbolicator_server-0.0.1/symbolicator_server.egg-info/requires.txt

@@ -0,0 +1,2 @@
+fastapi>=0.162.2
+uvicorn>=0.54.0

symbolicator_server-0.0.1/symbolicator_server.egg-info/scm_file_list.json

@@ -0,0 +1,16 @@
+{
+  "files": [
+    ".github/workflows/python-app.yml",
+    ".github/workflows/python-publish.yml",
+    ".gitignore",
+    ".pre-commit-config.yaml",
+    "README.md",
+    "pyproject.toml",
+    "requirements.txt",
+    "symbolicator_server/__init__.py",
+    "symbolicator_server/__main__.py",
+    "symbolicator_server/scanner.py",
+    "symbolicator_server/server.py",
+    "symbolicator_server/store.py"
+  ]
+}

symbolicator_server-0.0.1/symbolicator_server.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.0.1",
+  "distance": 0,
+  "node": "g026bd4807427934e641aa6574dd4e013eb7870bc",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-11-15"
+}

symbolicator_server-0.0.1/symbolicator_server.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+dist
+symbolicator_server