pypproxy 0.1.0__py3-none-any.whl

pypproxy/store/db.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import UTC
+from pathlib import Path
+
+import aiosqlite
+
+from .models import Entry, Filter
+
+logger = logging.getLogger(__name__)
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS entries (
+    id          INTEGER PRIMARY KEY,
+    created_at  TEXT NOT NULL,
+    method      TEXT NOT NULL DEFAULT '',
+    scheme      TEXT NOT NULL DEFAULT '',
+    host        TEXT NOT NULL DEFAULT '',
+    path        TEXT NOT NULL DEFAULT '/',
+    query       TEXT NOT NULL DEFAULT '',
+    req_headers TEXT NOT NULL DEFAULT '{}',
+    req_body    BLOB NOT NULL DEFAULT '',
+    status_code INTEGER NOT NULL DEFAULT 0,
+    resp_headers TEXT NOT NULL DEFAULT '{}',
+    resp_body   BLOB NOT NULL DEFAULT '',
+    duration_ms INTEGER NOT NULL DEFAULT 0,
+    protocol    TEXT NOT NULL DEFAULT 'http',
+    tags        TEXT NOT NULL DEFAULT '[]',
+    modified    INTEGER NOT NULL DEFAULT 0,
+    color       TEXT NOT NULL DEFAULT ''
+)
+"""
+
+_CREATE_INDEX = """
+CREATE INDEX IF NOT EXISTS idx_entries_host ON entries (host);
+CREATE INDEX IF NOT EXISTS idx_entries_method ON entries (method);
+CREATE INDEX IF NOT EXISTS idx_entries_protocol ON entries (protocol);
+"""
+
+
+class Database:
+    def __init__(self, path: str) -> None:
+        self._path = path
+        self._db: aiosqlite.Connection | None = None
+        self._lock = asyncio.Lock()
+
+    async def open(self) -> None:
+        Path(self._path).parent.mkdir(parents=True, exist_ok=True)
+        self._db = await aiosqlite.connect(self._path)
+        self._db.row_factory = aiosqlite.Row
+        await self._db.execute(_CREATE_TABLE)
+        for stmt in _CREATE_INDEX.strip().split("\n"):
+            if stmt.strip():
+                await self._db.execute(stmt)
+        await self._db.commit()
+        # Initialize FTS index
+        from pypproxy.store.fts import setup_fts
+
+        await setup_fts(self._db)
+        logger.info("database opened: %s", self._path)
+
+    async def search(self, query: str, limit: int = 50) -> list:
+        from pypproxy.store.fts import search
+
+        if not self._db:
+            return []
+        return await search(self._db, query, limit)
+
+    async def close(self) -> None:
+        if self._db:
+            await self._db.close()
+
+    async def insert(self, entry: Entry) -> None:
+        if not self._db:
+            return
+        async with self._lock:
+            await self._db.execute(
+                """INSERT OR REPLACE INTO entries
+                   (id, created_at, method, scheme, host, path, query,
+                    req_headers, req_body, status_code, resp_headers, resp_body,
+                    duration_ms, protocol, tags, modified, color)
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                _entry_to_row(entry),
+            )
+            await self._db.commit()
+
+    async def update(self, entry: Entry) -> None:
+        await self.insert(entry)
+
+    async def get(self, entry_id: int) -> Entry | None:
+        if not self._db:
+            return None
+        async with self._db.execute("SELECT * FROM entries WHERE id = ?", (entry_id,)) as cur:
+            row = await cur.fetchone()
+        return _row_to_entry(row) if row else None
+
+    async def list(self, f: Filter, offset: int = 0, limit: int = 100) -> tuple[list[Entry], int]:
+        if not self._db:
+            return [], 0
+        where, params = _build_where(f)
+        async with self._db.execute(f"SELECT COUNT(*) FROM entries {where}", params) as cur:
+            total = (await cur.fetchone())[0]
+
+        order = "ORDER BY id DESC"
+        q = f"SELECT * FROM entries {where} {order} LIMIT ? OFFSET ?"
+        async with self._db.execute(q, [*params, limit, offset]) as cur:
+            rows = await cur.fetchall()
+        return [_row_to_entry(r) for r in rows], total
+
+    async def clear(self) -> None:
+        if not self._db:
+            return
+        async with self._lock:
+            await self._db.execute("DELETE FROM entries")
+            await self._db.commit()
+
+    async def max_id(self) -> int:
+        if not self._db:
+            return 0
+        async with self._db.execute("SELECT MAX(id) FROM entries") as cur:
+            row = await cur.fetchone()
+        return row[0] or 0
+
+
+def _build_where(f: Filter) -> tuple[str, list]:
+    clauses, params = [], []
+    if f.method:
+        clauses.append("method = ?")
+        params.append(f.method)
+    if f.host:
+        clauses.append("host = ?")
+        params.append(f.host)
+    if f.protocol:
+        clauses.append("protocol = ?")
+        params.append(f.protocol)
+    if f.search:
+        clauses.append("(host || path) LIKE ?")
+        params.append(f"%{f.search}%")
+    where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
+    return where, params
+
+
+def _entry_to_row(e: Entry) -> tuple:
+    return (
+        e.id,
+        e.created_at.isoformat(),
+        e.method,
+        e.scheme,
+        e.host,
+        e.path,
+        e.query,
+        json.dumps(e.req_headers),
+        e.req_body,
+        e.status_code,
+        json.dumps(e.resp_headers),
+        e.resp_body,
+        e.duration_ms,
+        e.protocol,
+        json.dumps(e.tags),
+        int(e.modified),
+        getattr(e, "color", ""),
+    )
+
+
+def _row_to_entry(row: aiosqlite.Row) -> Entry:
+    from datetime import datetime
+
+    e = Entry()
+    e.id = row["id"]
+    e.created_at = datetime.fromisoformat(row["created_at"]).replace(tzinfo=UTC)
+    e.method = row["method"]
+    e.scheme = row["scheme"]
+    e.host = row["host"]
+    e.path = row["path"]
+    e.query = row["query"]
+    e.req_headers = json.loads(row["req_headers"])
+    e.req_body = bytes(row["req_body"]) if row["req_body"] else b""
+    e.status_code = row["status_code"]
+    e.resp_headers = json.loads(row["resp_headers"])
+    e.resp_body = bytes(row["resp_body"]) if row["resp_body"] else b""
+    e.duration_ms = row["duration_ms"]
+    e.protocol = row["protocol"]
+    e.tags = json.loads(row["tags"])
+    e.modified = bool(row["modified"])
+    e.color = row["color"]
+    return e

pypproxy/store/filter_parser.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .models import Entry
+
+
+@dataclass
+class Condition:
+    field: str
+    op: str  # ==, !=, contains, ~  (regex)
+    value: str
+
+    def matches(self, entry: Entry) -> bool:
+        val = _extract(entry, self.field)
+        if self.op == "==":
+            result = val == self.value
+        elif self.op == "!=":
+            result = val != self.value
+        elif self.op == "contains":
+            result = self.value.lower() in val.lower()
+        elif self.op == "~":
+            try:
+                result = bool(re.search(self.value, val))
+            except re.error:
+                result = False
+        else:
+            result = self.value.lower() in val.lower()
+        return result
+
+
+def _extract(entry: Entry, field: str) -> str:
+    field = field.lower()
+    if field in ("host", "server"):
+        return entry.host
+    if field in ("path", "url"):
+        return entry.path
+    if field in ("method",):
+        return entry.method
+    if field in ("status", "status_code"):
+        return str(entry.status_code)
+    if field in ("protocol", "type"):
+        return entry.protocol
+    if field == "request":
+        hdrs = " ".join(f"{k}: {','.join(v)}" for k, v in entry.req_headers.items())
+        return f"{entry.method} {entry.path} {hdrs} {entry.req_body.decode(errors='replace')}"
+    if field == "response":
+        hdrs = " ".join(f"{k}: {','.join(v)}" for k, v in entry.resp_headers.items())
+        return f"{entry.status_code} {hdrs} {entry.resp_body.decode(errors='replace')}"
+    if field in ("full_text", "all"):
+        req = _extract(entry, "request")
+        resp = _extract(entry, "response")
+        return req + " " + resp
+    return ""
+
+
+class FilterExpression:
+    """
+    Parses PacketProxy-style filter expressions.
+
+    Syntax:
+        field == value
+        field != value
+        field contains value
+        field ~ regex
+        expr && expr
+        expr || expr
+
+    Example:
+        host == example.com && method == POST
+        request contains token || response contains error
+    """
+
+    def __init__(self, expr: str) -> None:
+        self._expr = expr.strip()
+        self._compiled: list | None = None
+        if self._expr:
+            try:
+                self._compiled = _parse(self._expr)
+            except Exception:
+                self._compiled = None
+
+    @property
+    def is_empty(self) -> bool:
+        return not self._expr
+
+    def matches(self, entry: Entry) -> bool:
+        if not self._compiled:
+            return True  # empty or invalid expression → match all
+        return _eval(self._compiled, entry)
+
+
+# --- parser ---
+
+_TOKEN_RE = re.compile(
+    r"(\|\||&&|\(|\)|"
+    r"(?:host|path|url|method|status|status_code|protocol|type|request|response|full_text|all)"
+    r"\s*(?:==|!=|contains|~)\s*[^\s()]+)",
+    re.IGNORECASE,
+)
+
+_COND_RE = re.compile(
+    r"^([\w_]+)\s*(==|!=|contains|~)\s*(.+)$",
+    re.IGNORECASE,
+)
+
+
+def _parse(expr: str) -> list:
+    """Tokenize into an AST-like list: ['cond', Condition], ['and'/'or'], ['('], [')']"""
+    tokens = []
+    i = 0
+    expr = expr.strip()
+
+    while i < len(expr):
+        if expr[i : i + 2] == "&&":
+            tokens.append(("and",))
+            i += 2
+        elif expr[i : i + 2] == "||":
+            tokens.append(("or",))
+            i += 2
+        elif expr[i] == "(":
+            tokens.append(("lparen",))
+            i += 1
+        elif expr[i] == ")":
+            tokens.append(("rparen",))
+            i += 1
+        elif expr[i] == " ":
+            i += 1
+        else:
+            # find end of condition token (up to next operator or paren)
+            end = len(expr)
+            for marker in ["&&", "||", ")", "("]:
+                pos = expr.find(marker, i)
+                if pos != -1 and pos < end:
+                    end = pos
+            token = expr[i:end].strip()
+            if token:
+                m = _COND_RE.match(token)
+                if m:
+                    tokens.append(
+                        (
+                            "cond",
+                            Condition(
+                                field=m.group(1),
+                                op=m.group(2).lower(),
+                                value=m.group(3).strip(),
+                            ),
+                        )
+                    )
+            i = end
+
+    return tokens
+
+
+def _eval(tokens: list, entry: Entry) -> bool:
+    """Evaluate token list with short-circuit && / ||."""
+    if not tokens:
+        return True
+
+    results = []
+    ops = []
+
+    for tok in tokens:
+        kind = tok[0]
+        if kind == "cond":
+            results.append(tok[1].matches(entry))
+        elif kind == "and":
+            ops.append("and")
+        elif kind == "or":
+            ops.append("or")
+
+    if not results:
+        return True
+
+    result = results[0]
+    for i, op in enumerate(ops):
+        if i + 1 >= len(results):
+            break
+        result = result and results[i + 1] if op == "and" else result or results[i + 1]
+    return result

pypproxy/store/fts.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+import aiosqlite
+
+logger = logging.getLogger(__name__)
+
+_CREATE_FTS = """
+CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts
+USING fts5(
+    host,
+    path,
+    req_body,
+    resp_body,
+    req_headers,
+    resp_headers,
+    content='entries',
+    content_rowid='id'
+)
+"""
+
+_TRIGGER_INSERT = """
+CREATE TRIGGER IF NOT EXISTS entries_fts_insert
+AFTER INSERT ON entries BEGIN
+    INSERT INTO entries_fts(rowid, host, path, req_body, resp_body, req_headers, resp_headers)
+    VALUES (new.id, new.host, new.path, new.req_body, new.resp_body, new.req_headers, new.resp_headers);
+END
+"""
+
+_TRIGGER_UPDATE = """
+CREATE TRIGGER IF NOT EXISTS entries_fts_update
+AFTER UPDATE ON entries BEGIN
+    INSERT INTO entries_fts(entries_fts, rowid, host, path, req_body, resp_body, req_headers, resp_headers)
+    VALUES ('delete', old.id, old.host, old.path, old.req_body, old.resp_body, old.req_headers, old.resp_headers);
+    INSERT INTO entries_fts(rowid, host, path, req_body, resp_body, req_headers, resp_headers)
+    VALUES (new.id, new.host, new.path, new.req_body, new.resp_body, new.req_headers, new.resp_headers);
+END
+"""
+
+_TRIGGER_DELETE = """
+CREATE TRIGGER IF NOT EXISTS entries_fts_delete
+BEFORE DELETE ON entries BEGIN
+    INSERT INTO entries_fts(entries_fts, rowid, host, path, req_body, resp_body, req_headers, resp_headers)
+    VALUES ('delete', old.id, old.host, old.path, old.req_body, old.resp_body, old.req_headers, old.resp_headers);
+END
+"""
+
+
+@dataclass
+class SearchResult:
+    entry_id: int
+    rank: float
+    snippet: str
+
+    def to_dict(self) -> dict:
+        return {
+            "entry_id": self.entry_id,
+            "rank": self.rank,
+            "snippet": self.snippet,
+        }
+
+
+async def setup_fts(db: aiosqlite.Connection) -> None:
+    """Create FTS5 virtual table and triggers if not already present."""
+    await db.execute(_CREATE_FTS)
+    await db.execute(_TRIGGER_INSERT)
+    await db.execute(_TRIGGER_UPDATE)
+    await db.execute(_TRIGGER_DELETE)
+    await db.commit()
+    logger.info("FTS5 full-text search index ready")
+
+
+async def search(
+    db: aiosqlite.Connection,
+    query: str,
+    limit: int = 50,
+) -> list[SearchResult]:
+    """Search entries using FTS5. Returns matching entry IDs sorted by relevance."""
+    if not query.strip():
+        return []
+    try:
+        async with db.execute(
+            """
+            SELECT rowid, rank,
+                   snippet(entries_fts, 0, '<b>', '</b>', '…', 10)
+            FROM entries_fts
+            WHERE entries_fts MATCH ?
+            ORDER BY rank
+            LIMIT ?
+            """,
+            (query, limit),
+        ) as cur:
+            rows = await cur.fetchall()
+        return [SearchResult(entry_id=r[0], rank=r[1], snippet=r[2]) for r in rows]
+    except Exception as e:
+        logger.debug("FTS search error: %s", e)
+        return []
+
+
+async def rebuild_index(db: aiosqlite.Connection) -> None:
+    """Rebuild the FTS index from the entries table."""
+    await db.execute("INSERT INTO entries_fts(entries_fts) VALUES ('rebuild')")
+    await db.commit()

pypproxy/store/models.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+
+@dataclass
+class Entry:
+    id: int = 0
+    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+    # Request
+    method: str = ""
+    scheme: str = ""
+    host: str = ""
+    path: str = ""
+    query: str = ""
+    req_headers: dict[str, list[str]] = field(default_factory=dict)
+    req_body: bytes = b""
+
+    # Response
+    status_code: int = 0
+    resp_headers: dict[str, list[str]] = field(default_factory=dict)
+    resp_body: bytes = b""
+    duration_ms: int = 0
+
+    # Meta
+    protocol: str = "http"
+    tags: list[str] = field(default_factory=list)
+    modified: bool = False
+    color: str = ""  # row highlight color (hex or name)
+    graphql_operation: str = ""  # operation name if GraphQL
+    graphql_op_type: str = ""  # query | mutation | subscription
+
+    def to_dict(self) -> dict:
+        import base64
+
+        return {
+            "id": self.id,
+            "created_at": self.created_at.isoformat(),
+            "method": self.method,
+            "scheme": self.scheme,
+            "host": self.host,
+            "path": self.path,
+            "query": self.query,
+            "req_headers": self.req_headers,
+            "req_body": base64.b64encode(self.req_body).decode() if self.req_body else "",
+            "status_code": self.status_code,
+            "resp_headers": self.resp_headers,
+            "resp_body": base64.b64encode(self.resp_body).decode() if self.resp_body else "",
+            "duration_ms": self.duration_ms,
+            "protocol": self.protocol,
+            "tags": self.tags,
+            "modified": self.modified,
+            "color": self.color,
+            "graphql_operation": self.graphql_operation,
+            "graphql_op_type": self.graphql_op_type,
+        }
+
+
+@dataclass
+class Filter:
+    method: str = ""
+    host: str = ""
+    search: str = ""
+    protocol: str = ""
+    expression: str = ""  # PacketProxy-style filter expression
+
+    def matches(self, entry: Entry) -> bool:
+        # Advanced expression takes precedence when set
+        if self.expression:
+            from pypproxy.store.filter_parser import FilterExpression
+
+            return FilterExpression(self.expression).matches(entry)
+        if self.method and entry.method != self.method:
+            return False
+        if self.host and entry.host != self.host:
+            return False
+        if self.protocol and entry.protocol != self.protocol:
+            return False
+        return not (self.search and self.search.lower() not in (entry.host + entry.path).lower())

pypproxy/store/scope.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import fnmatch
+import re
+import threading
+from dataclasses import dataclass
+
+
+@dataclass
+class ScopeRule:
+    pattern: str
+    mode: str = "glob"  # glob or regex
+    enabled: bool = True
+
+    def matches(self, host: str) -> bool:
+        if not self.enabled:
+            return False
+        if self.mode == "regex":
+            try:
+                return bool(re.search(self.pattern, host))
+            except re.error:
+                return False
+        return fnmatch.fnmatch(host, self.pattern)
+
+    def to_dict(self) -> dict:
+        return {"pattern": self.pattern, "mode": self.mode, "enabled": self.enabled}
+
+
+class ScopeManager:
+    """Controls which hosts are in-scope for capture."""
+
+    def __init__(self) -> None:
+        self._rules: list[ScopeRule] = []
+        self._lock = threading.Lock()
+        self._enabled = False  # when False, all hosts are in-scope
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled
+
+    def set_enabled(self, value: bool) -> None:
+        self._enabled = value
+
+    def add(self, rule: ScopeRule) -> None:
+        with self._lock:
+            self._rules.append(rule)
+
+    def remove(self, pattern: str) -> None:
+        with self._lock:
+            self._rules = [r for r in self._rules if r.pattern != pattern]
+
+    def list(self) -> list[ScopeRule]:
+        with self._lock:
+            return list(self._rules)
+
+    def is_in_scope(self, host: str) -> bool:
+        if not self._enabled:
+            return True
+        with self._lock:
+            rules = list(self._rules)
+        if not rules:
+            return True
+        return any(r.matches(host) for r in rules)