bqsqlparse 0.1.0__py3-none-any.whl

bqsqlparse/__init__.py

@@ -0,0 +1,62 @@
+"""bqsqlparse — a pure-Python BigQuery (GoogleSQL) parser with
+column-level lineage extraction.
+
+Quick start::
+
+    import bqsqlparse
+
+    ast = bqsqlparse.parse_one("SELECT a, SUM(b) AS total FROM ds.t GROUP BY a")
+    print(ast.sql())
+
+    lineage = bqsqlparse.extract_lineage(
+        "CREATE TABLE ds.out AS SELECT o.id, SUM(o.amount) t FROM ds.orders o GROUP BY 1"
+    )
+    print(lineage.to_json())
+"""
+
+from . import nodes
+from .errors import (
+    BQSQLError, LexError, LineageError, ParseError, UnsupportedStatementError,
+)
+from .functions import (
+    AGGREGATE_FUNCTIONS, ALL_FUNCTIONS, NAVIGATION_FUNCTIONS,
+    SCALAR_FUNCTIONS, is_aggregate, is_known_function, is_navigation,
+)
+from .lexer import Token, tokenize
+from .lineage import (
+    ColumnLineage, ColumnUsage, LineageResult, SourceColumn, extract_lineage,
+    extract_script_lineage,
+)
+from .parser import Parser, parse, parse_one
+from .unparse import to_sql
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "parse",
+    "parse_one",
+    "Parser",
+    "tokenize",
+    "Token",
+    "to_sql",
+    "nodes",
+    "extract_lineage",
+    "extract_script_lineage",
+    "LineageResult",
+    "ColumnLineage",
+    "ColumnUsage",
+    "SourceColumn",
+    "BQSQLError",
+    "LexError",
+    "ParseError",
+    "UnsupportedStatementError",
+    "LineageError",
+    "AGGREGATE_FUNCTIONS",
+    "NAVIGATION_FUNCTIONS",
+    "SCALAR_FUNCTIONS",
+    "ALL_FUNCTIONS",
+    "is_aggregate",
+    "is_navigation",
+    "is_known_function",
+    "__version__",
+]

bqsqlparse/errors.py

@@ -0,0 +1,30 @@
+"""Exception types for bqsqlparse."""
+
+from typing import Optional
+
+
+class BQSQLError(Exception):
+    """Base class for all bqsqlparse errors."""
+
+    def __init__(self, message: str, line: Optional[int] = None, col: Optional[int] = None):
+        self.message = message
+        self.line = line
+        self.col = col
+        loc = f" at line {line}, column {col}" if line is not None else ""
+        super().__init__(f"{message}{loc}")
+
+
+class LexError(BQSQLError):
+    """Raised when the tokenizer encounters invalid input."""
+
+
+class ParseError(BQSQLError):
+    """Raised when the parser encounters invalid or unexpected syntax."""
+
+
+class UnsupportedStatementError(ParseError):
+    """Raised for statement types the parser does not support."""
+
+
+class LineageError(BQSQLError):
+    """Raised when lineage extraction cannot proceed."""

bqsqlparse/functions.py

@@ -0,0 +1,159 @@
+"""Registry of BigQuery (GoogleSQL) built-in functions.
+
+Used by the lineage engine to classify transformations (aggregation vs.
+window vs. scalar expression) and exposed for consumers who want to check
+whether a function name is a known BigQuery built-in.
+"""
+
+from typing import Dict, FrozenSet
+
+AGGREGATE_FUNCTIONS: FrozenSet[str] = frozenset({
+    "ANY_VALUE", "ARRAY_AGG", "ARRAY_CONCAT_AGG", "AVG", "BIT_AND", "BIT_OR",
+    "BIT_XOR", "COUNT", "COUNTIF", "GROUPING", "LOGICAL_AND", "LOGICAL_OR",
+    "MAX", "MAX_BY", "MIN", "MIN_BY", "STRING_AGG", "SUM", "CORR",
+    "COVAR_POP", "COVAR_SAMP", "STDDEV", "STDDEV_POP", "STDDEV_SAMP",
+    "VAR_POP", "VAR_SAMP", "VARIANCE",
+    "APPROX_COUNT_DISTINCT", "APPROX_QUANTILES", "APPROX_TOP_COUNT",
+    "APPROX_TOP_SUM",
+    "HLL_COUNT.INIT", "HLL_COUNT.MERGE", "HLL_COUNT.MERGE_PARTIAL",
+    "HLL_COUNT.EXTRACT",
+    "KLL_QUANTILES.INIT_INT64", "KLL_QUANTILES.INIT_FLOAT64",
+    "KLL_QUANTILES.MERGE_PARTIAL", "KLL_QUANTILES.MERGE",
+    "KLL_QUANTILES.MERGE_POINT", "KLL_QUANTILES.EXTRACT",
+    "KLL_QUANTILES.EXTRACT_POINT",
+    "PERCENTILE_CONT", "PERCENTILE_DISC", "ST_EXTENT", "ST_UNION_AGG",
+    "ST_CENTROID_AGG",
+})
+
+NAVIGATION_FUNCTIONS: FrozenSet[str] = frozenset({
+    "RANK", "DENSE_RANK", "ROW_NUMBER", "NTILE", "CUME_DIST", "PERCENT_RANK",
+    "LAG", "LEAD", "FIRST_VALUE", "LAST_VALUE", "NTH_VALUE",
+})
+
+SCALAR_FUNCTIONS: Dict[str, FrozenSet[str]] = {
+    "string": frozenset({
+        "ASCII", "BYTE_LENGTH", "CHAR_LENGTH", "CHARACTER_LENGTH", "CHR",
+        "CODE_POINTS_TO_BYTES", "CODE_POINTS_TO_STRING", "COLLATE", "CONCAT",
+        "CONTAINS_SUBSTR", "EDIT_DISTANCE", "ENDS_WITH", "FORMAT",
+        "FROM_BASE32", "FROM_BASE64", "FROM_HEX", "INITCAP", "INSTR", "LEFT",
+        "LENGTH", "LOWER", "LPAD", "LTRIM", "NORMALIZE",
+        "NORMALIZE_AND_CASEFOLD", "OCTET_LENGTH", "REGEXP_CONTAINS",
+        "REGEXP_EXTRACT", "REGEXP_EXTRACT_ALL", "REGEXP_INSTR",
+        "REGEXP_REPLACE", "REGEXP_SUBSTR", "REPEAT", "REPLACE", "REVERSE",
+        "RIGHT", "RPAD", "RTRIM", "SAFE_CONVERT_BYTES_TO_STRING", "SOUNDEX",
+        "SPLIT", "STARTS_WITH", "STRPOS", "SUBSTR", "SUBSTRING", "TO_BASE32",
+        "TO_BASE64", "TO_CODE_POINTS", "TO_HEX", "TRANSLATE", "TRIM",
+        "UNICODE", "UPPER",
+    }),
+    "math": frozenset({
+        "ABS", "ACOS", "ACOSH", "ASIN", "ASINH", "ATAN", "ATAN2", "ATANH",
+        "CBRT", "CEIL", "CEILING", "COS", "COSH", "COSINE_DISTANCE", "COT",
+        "COTH", "CSC", "CSCH", "DIV", "EUCLIDEAN_DISTANCE", "EXP", "FLOOR",
+        "GREATEST", "IEEE_DIVIDE", "IS_INF", "IS_NAN", "LEAST", "LN", "LOG",
+        "LOG10", "MOD", "POW", "POWER", "RAND", "RANGE_BUCKET", "ROUND",
+        "SAFE_ADD", "SAFE_DIVIDE", "SAFE_MULTIPLY", "SAFE_NEGATE",
+        "SAFE_SUBTRACT", "SEC", "SECH", "SIGN", "SIN", "SINH", "SQRT", "TAN",
+        "TANH", "TRUNC",
+    }),
+    "date_time": frozenset({
+        "CURRENT_DATE", "CURRENT_DATETIME", "CURRENT_TIME",
+        "CURRENT_TIMESTAMP", "DATE", "DATE_ADD", "DATE_BUCKET", "DATE_DIFF",
+        "DATE_FROM_UNIX_DATE", "DATE_SUB", "DATE_TRUNC", "DATETIME",
+        "DATETIME_ADD", "DATETIME_BUCKET", "DATETIME_DIFF", "DATETIME_SUB",
+        "DATETIME_TRUNC", "FORMAT_DATE", "FORMAT_DATETIME", "FORMAT_TIME",
+        "FORMAT_TIMESTAMP", "GENERATE_DATE_ARRAY", "GENERATE_TIMESTAMP_ARRAY",
+        "JUSTIFY_DAYS", "JUSTIFY_HOURS", "JUSTIFY_INTERVAL", "LAST_DAY",
+        "MAKE_INTERVAL", "PARSE_DATE", "PARSE_DATETIME", "PARSE_TIME",
+        "PARSE_TIMESTAMP", "TIME", "TIME_ADD", "TIME_DIFF", "TIME_SUB",
+        "TIME_TRUNC", "TIMESTAMP", "TIMESTAMP_ADD", "TIMESTAMP_BUCKET",
+        "TIMESTAMP_DIFF", "TIMESTAMP_MICROS", "TIMESTAMP_MILLIS",
+        "TIMESTAMP_SECONDS", "TIMESTAMP_SUB", "TIMESTAMP_TRUNC",
+        "UNIX_DATE", "UNIX_MICROS", "UNIX_MILLIS", "UNIX_SECONDS", "EXTRACT",
+    }),
+    "array": frozenset({
+        "ARRAY", "ARRAY_CONCAT", "ARRAY_FIRST", "ARRAY_LAST", "ARRAY_LENGTH",
+        "ARRAY_REVERSE", "ARRAY_SLICE", "ARRAY_TO_STRING", "GENERATE_ARRAY",
+        "ARRAY_ZIP", "FLATTEN",
+    }),
+    "json": frozenset({
+        "BOOL", "FLOAT64", "INT64", "JSON_ARRAY", "JSON_ARRAY_APPEND",
+        "JSON_ARRAY_INSERT", "JSON_EXTRACT", "JSON_EXTRACT_ARRAY",
+        "JSON_EXTRACT_SCALAR", "JSON_EXTRACT_STRING_ARRAY", "JSON_KEYS",
+        "JSON_OBJECT", "JSON_QUERY", "JSON_QUERY_ARRAY", "JSON_REMOVE",
+        "JSON_SET", "JSON_STRIP_NULLS", "JSON_TYPE", "JSON_VALUE",
+        "JSON_VALUE_ARRAY", "LAX_BOOL", "LAX_FLOAT64", "LAX_INT64",
+        "LAX_STRING", "PARSE_JSON", "STRING", "TO_JSON", "TO_JSON_STRING",
+    }),
+    "conditional": frozenset({
+        "COALESCE", "IF", "IFNULL", "NULLIF", "NULLIFZERO", "ZEROIFNULL",
+    }),
+    "conversion": frozenset({
+        "CAST", "SAFE_CAST", "PARSE_BIGNUMERIC", "PARSE_NUMERIC",
+    }),
+    "hash_crypto": frozenset({
+        "FARM_FINGERPRINT", "MD5", "SHA1", "SHA256", "SHA512",
+        "KEYS.NEW_KEYSET", "KEYS.ADD_KEY_FROM_RAW_BYTES",
+        "AEAD.DECRYPT_BYTES", "AEAD.DECRYPT_STRING", "AEAD.ENCRYPT",
+        "DETERMINISTIC_DECRYPT_BYTES", "DETERMINISTIC_DECRYPT_STRING",
+        "DETERMINISTIC_ENCRYPT", "KEYS.KEYSET_CHAIN", "KEYS.KEYSET_FROM_JSON",
+        "KEYS.KEYSET_TO_JSON", "KEYS.ROTATE_KEYSET", "KEYS.KEYSET_LENGTH",
+    }),
+    "net": frozenset({
+        "NET.HOST", "NET.IP_FROM_STRING", "NET.IP_NET_MASK", "NET.IP_TO_STRING",
+        "NET.IP_TRUNC", "NET.IPV4_FROM_INT64", "NET.IPV4_TO_INT64",
+        "NET.PUBLIC_SUFFIX", "NET.REG_DOMAIN", "NET.SAFE_IP_FROM_STRING",
+    }),
+    "geography": frozenset({
+        "ST_AREA", "ST_ASBINARY", "ST_ASGEOJSON", "ST_ASTEXT", "ST_ANGLE",
+        "ST_AZIMUTH", "ST_BOUNDARY", "ST_BOUNDINGBOX", "ST_BUFFER",
+        "ST_BUFFERWITHTOLERANCE", "ST_CENTROID", "ST_CLOSESTPOINT",
+        "ST_CLUSTERDBSCAN", "ST_CONTAINS", "ST_CONVEXHULL", "ST_COVEREDBY",
+        "ST_COVERS", "ST_DIFFERENCE", "ST_DIMENSION", "ST_DISJOINT",
+        "ST_DISTANCE", "ST_DUMP", "ST_DWITHIN", "ST_ENDPOINT", "ST_EQUALS",
+        "ST_EXTERIORRING", "ST_GEOGFROM", "ST_GEOGFROMGEOJSON",
+        "ST_GEOGFROMTEXT", "ST_GEOGFROMWKB", "ST_GEOGPOINT",
+        "ST_GEOGPOINTFROMGEOHASH", "ST_GEOHASH", "ST_GEOMETRYTYPE",
+        "ST_HAUSDORFFDISTANCE", "ST_INTERIORRINGS", "ST_INTERSECTION",
+        "ST_INTERSECTS", "ST_INTERSECTSBOX", "ST_ISCLOSED", "ST_ISCOLLECTION",
+        "ST_ISEMPTY", "ST_ISRING", "ST_LENGTH", "ST_LINEINTERPOLATEPOINT",
+        "ST_LINELOCATEPOINT", "ST_LINESUBSTRING", "ST_MAKELINE",
+        "ST_MAKEPOLYGON", "ST_MAKEPOLYGONORIENTED", "ST_MAXDISTANCE",
+        "ST_NPOINTS", "ST_NUMGEOMETRIES", "ST_NUMPOINTS", "ST_PERIMETER",
+        "ST_POINTN", "ST_SIMPLIFY", "ST_SNAPTOGRID", "ST_STARTPOINT",
+        "ST_TOUCHES", "ST_UNION", "ST_WITHIN", "ST_X", "ST_Y",
+    }),
+    "range": frozenset({
+        "RANGE", "RANGE_CONTAINS", "RANGE_END", "RANGE_INTERSECT",
+        "RANGE_OVERLAPS", "RANGE_SESSIONIZE", "RANGE_START",
+    }),
+    "utility": frozenset({
+        "GENERATE_UUID", "SESSION_USER", "ERROR", "BIT_COUNT",
+        "TYPEOF", "VECTOR_SEARCH",
+    }),
+    "search": frozenset({
+        "SEARCH", "TEXT_ANALYZE",
+    }),
+    "interval": frozenset({
+        "JUSTIFY_DAYS", "JUSTIFY_HOURS", "JUSTIFY_INTERVAL", "MAKE_INTERVAL",
+    }),
+}
+
+_all = set(AGGREGATE_FUNCTIONS) | set(NAVIGATION_FUNCTIONS)
+for _names in SCALAR_FUNCTIONS.values():
+    _all |= set(_names)
+ALL_FUNCTIONS: FrozenSet[str] = frozenset(_all)
+
+
+def is_aggregate(name: str) -> bool:
+    """True if *name* (case-insensitive, dotted allowed) is an aggregate."""
+    return name.upper() in AGGREGATE_FUNCTIONS
+
+
+def is_navigation(name: str) -> bool:
+    """True if *name* is a numbering/navigation (window-only) function."""
+    return name.upper() in NAVIGATION_FUNCTIONS
+
+
+def is_known_function(name: str) -> bool:
+    """True if *name* is a known BigQuery built-in function."""
+    return name.upper() in ALL_FUNCTIONS

bqsqlparse/lexer.py

@@ -0,0 +1,242 @@
+"""Tokenizer for the BigQuery (GoogleSQL) dialect.
+
+Handles:
+- Reserved keywords vs. contextual identifiers
+- Backtick-quoted identifiers (including dotted paths inside backticks)
+- String / bytes literals with r/b prefixes, single, double and triple quotes
+- Escape sequences (\\n, \\xHH, \\uXXXX, \\UXXXXXXXX, octal, ...)
+- Numeric literals (int, float, exponent, hex)
+- Named (@param), system (@@var) and positional (?) parameters
+- Comments: ``--``, ``#`` and ``/* ... */``
+- All GoogleSQL operators including ``||``, ``<<``, ``>>``, ``=>``
+"""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_right
+from dataclasses import dataclass, field
+from typing import List
+
+from .errors import LexError
+
+# Official GoogleSQL reserved keywords.
+RESERVED = frozenset(
+    """
+    ALL AND ANY ARRAY AS ASC ASSERT_ROWS_MODIFIED AT BETWEEN BY CASE CAST
+    COLLATE CONTAINS CREATE CROSS CUBE CURRENT DEFAULT DEFINE DESC DISTINCT
+    ELSE END ENUM ESCAPE EXCEPT EXCLUDE EXISTS EXTRACT FALSE FETCH FOLLOWING
+    FOR FROM FULL GROUP GROUPING GROUPS HASH HAVING IF IGNORE IN INNER
+    INTERSECT INTERVAL INTO IS JOIN LATERAL LEFT LIKE LIMIT LOOKUP MERGE
+    NATURAL NEW NO NOT NULL NULLS OF ON OR ORDER OUTER OVER PARTITION
+    PRECEDING PROTO QUALIFY RANGE RECURSIVE RESPECT RIGHT ROLLUP ROWS SELECT
+    SET SOME STRUCT TABLESAMPLE THEN TO TREAT TRUE UNBOUNDED UNION UNNEST
+    USING WHEN WHERE WINDOW WITH WITHIN
+    """.split()
+)
+
+# Token types
+KEYWORD = "KEYWORD"
+IDENT = "IDENT"
+QIDENT = "QIDENT"  # backtick-quoted identifier
+STRING = "STRING"
+BYTES = "BYTES"
+NUMBER = "NUMBER"
+PARAM = "PARAM"
+OP = "OP"
+EOF = "EOF"
+
+_TWO_CHAR_OPS = ("<<", ">>", "<=", ">=", "!=", "<>", "||", "=>")
+_SINGLE_CHAR_OPS = set("+-*/=<>()[]{},.;&|^~:")
+
+_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z_0-9]*")
+_NUMBER_RE = re.compile(r"0[xX][0-9A-Fa-f]+|(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?")
+
+_ESCAPES = {
+    "a": "\a", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t",
+    "v": "\v", "\\": "\\", "'": "'", '"': '"', "`": "`", "?": "?", "/": "/",
+}
+
+
+@dataclass
+class Token:
+    type: str
+    value: str
+    pos: int = 0
+    line: int = 0
+    col: int = 0
+    flags: str = field(default="", compare=False)
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"Token({self.type}, {self.value!r}, L{self.line}:C{self.col})"
+
+
+class _Lexer:
+    def __init__(self, sql: str):
+        self.sql = sql
+        self.n = len(sql)
+        self.i = 0
+        self.tokens: List[Token] = []
+        self.line_starts = [0]
+        for idx, ch in enumerate(sql):
+            if ch == "\n":
+                self.line_starts.append(idx + 1)
+
+    def _linecol(self, pos: int):
+        li = bisect_right(self.line_starts, pos) - 1
+        return li + 1, pos - self.line_starts[li] + 1
+
+    def _err(self, msg: str, pos: int):
+        line, col = self._linecol(pos)
+        raise LexError(msg, line, col)
+
+    def _add(self, type_: str, value: str, pos: int, flags: str = ""):
+        line, col = self._linecol(pos)
+        self.tokens.append(Token(type_, value, pos, line, col, flags))
+
+    def run(self) -> List[Token]:
+        sql, n = self.sql, self.n
+        while self.i < n:
+            ch = sql[self.i]
+            if ch in " \t\r\n":
+                self.i += 1
+            elif ch == "-" and self.i + 1 < n and sql[self.i + 1] == "-":
+                self._skip_line()
+            elif ch == "#":
+                self._skip_line()
+            elif ch == "/" and self.i + 1 < n and sql[self.i + 1] == "*":
+                end = sql.find("*/", self.i + 2)
+                if end == -1:
+                    self._err("Unterminated block comment", self.i)
+                self.i = end + 2
+            elif ch == "`":
+                self._read_qident()
+            elif ch in "'\"":
+                self._read_string(self.i, "")
+            elif ch.isalpha() or ch == "_":
+                m = _IDENT_RE.match(sql, self.i)
+                word = m.group(0)
+                nxt = self.i + len(word)
+                if word.lower() in ("r", "b", "rb", "br") and nxt < n and sql[nxt] in "'\"":
+                    start = self.i
+                    self.i = nxt
+                    self._read_string(start, word.lower())
+                else:
+                    upper = word.upper()
+                    if upper in RESERVED:
+                        self._add(KEYWORD, upper, self.i)
+                    else:
+                        self._add(IDENT, word, self.i)
+                    self.i = nxt
+            elif ch.isdigit() or (ch == "." and self.i + 1 < n and sql[self.i + 1].isdigit()):
+                m = _NUMBER_RE.match(sql, self.i)
+                self._add(NUMBER, m.group(0), self.i)
+                self.i = m.end()
+            elif ch == "@":
+                start = self.i
+                self.i += 1
+                prefix = "@"
+                if self.i < n and sql[self.i] == "@":
+                    prefix = "@@"
+                    self.i += 1
+                m = _IDENT_RE.match(sql, self.i)
+                if not m:
+                    self._err("Invalid query parameter", start)
+                self._add(PARAM, prefix + m.group(0), start)
+                self.i = m.end()
+            elif ch == "?":
+                self._add(PARAM, "?", self.i)
+                self.i += 1
+            else:
+                two = sql[self.i : self.i + 2]
+                if two in _TWO_CHAR_OPS:
+                    self._add(OP, two, self.i)
+                    self.i += 2
+                elif ch in _SINGLE_CHAR_OPS:
+                    self._add(OP, ch, self.i)
+                    self.i += 1
+                else:
+                    self._err(f"Unexpected character {ch!r}", self.i)
+        self._add(EOF, "", n)
+        return self.tokens
+
+    def _skip_line(self):
+        end = self.sql.find("\n", self.i)
+        self.i = self.n if end == -1 else end + 1
+
+    def _read_qident(self):
+        start = self.i
+        self.i += 1
+        chars = []
+        while self.i < self.n:
+            ch = self.sql[self.i]
+            if ch == "\\" and self.i + 1 < self.n:
+                chars.append(self.sql[self.i + 1])
+                self.i += 2
+            elif ch == "`":
+                self.i += 1
+                self._add(QIDENT, "".join(chars), start)
+                return
+            else:
+                chars.append(ch)
+                self.i += 1
+        self._err("Unterminated quoted identifier", start)
+
+    def _read_string(self, start: int, flags: str):
+        sql, n = self.sql, self.n
+        quote = sql[self.i]
+        raw = "r" in flags
+        is_bytes = "b" in flags
+        triple = sql[self.i : self.i + 3] == quote * 3
+        self.i += 3 if triple else 1
+        terminator = quote * 3 if triple else quote
+        chars = []
+        while self.i < n:
+            if sql.startswith(terminator, self.i):
+                self.i += len(terminator)
+                self._add(BYTES if is_bytes else STRING, "".join(chars), start, flags)
+                return
+            ch = sql[self.i]
+            if not triple and ch == "\n":
+                self._err("Unterminated string literal", start)
+            if ch == "\\" and self.i + 1 < n:
+                if raw:
+                    chars.append(ch)
+                    chars.append(sql[self.i + 1])
+                    self.i += 2
+                else:
+                    self.i += 1
+                    self._read_escape(chars)
+            else:
+                chars.append(ch)
+                self.i += 1
+        self._err("Unterminated string literal", start)
+
+    def _read_escape(self, chars):
+        sql = self.sql
+        e = sql[self.i]
+        if e in _ESCAPES:
+            chars.append(_ESCAPES[e])
+            self.i += 1
+        elif e == "x" or e == "X":
+            hexs = sql[self.i + 1 : self.i + 3]
+            chars.append(chr(int(hexs, 16)))
+            self.i += 3
+        elif e == "u":
+            chars.append(chr(int(sql[self.i + 1 : self.i + 5], 16)))
+            self.i += 5
+        elif e == "U":
+            chars.append(chr(int(sql[self.i + 1 : self.i + 9], 16)))
+            self.i += 9
+        elif e.isdigit():
+            m = re.match(r"[0-7]{1,3}", sql[self.i :])
+            chars.append(chr(int(m.group(0), 8)))
+            self.i += len(m.group(0))
+        else:
+            chars.append(e)
+            self.i += 1
+
+
+def tokenize(sql: str) -> List[Token]:
+    """Tokenize a GoogleSQL string into a list of :class:`Token`."""
+    return _Lexer(sql).run()