nfscache 0.1.0__py3-none-any.whl

nfscache/__init__.py

@@ -0,0 +1,3 @@
+from nfscache.nfs_cache import NFSCache
+
+__all__ = ["NFSCache"]

nfscache/data/data_container.py

@@ -0,0 +1,29 @@
+from typing import Any
+
+import polars as pl
+
+from nfscache.data.data_holder import DataHolder
+
+
+class DataContainer:
+    __slots__ = ("data",)
+
+    def __init__(
+            self,
+            input_data: dict[str, Any],
+    ) -> None:
+        self.data = DataHolder()
+        self.data.headers = tuple(input_data["headers"])
+        self.data.rows_data_pl = input_data["data"]
+
+
+if __name__ == "__main__":
+    headers = ("COL1", "COL2", "COL3")
+    rows = [(1.1, 1, "A"), (2.2, 2, "B"), (3.3, 3, "C"), (4.4, 4, "D")]
+    INPUT_DATA: dict[str, Any] = {
+        "headers": headers,
+        "data": pl.DataFrame(rows, schema=headers, orient="row"),
+    }
+    data = DataContainer(INPUT_DATA)
+    print("headers:", data.data.headers)
+    print("table:", data.data.rows_data_pl)

nfscache/data/data_holder.py

@@ -0,0 +1,23 @@
+from typing import Any
+
+import polars as pl
+
+class DataHolder:
+    headers: tuple
+    rows_data_pl: pl.DataFrame | None = None
+
+    def __init__(self):
+        self.headers = tuple()
+        self.rows_data_pl = pl.DataFrame()
+
+
+if __name__ == "__main__":
+    INPUT_DATA: dict[str, Any] = {
+        "headers": ("COL1", "COL2", "COL3"),
+        "data": [(1.1, 1, "A"), (2.2, 2, "B"), (3.3, 3, "C"), (4.4, 4, "D")],
+    }
+    data = DataHolder()
+    data.headers = tuple(INPUT_DATA["headers"])
+    data.rows_data_pl = pl.DataFrame(INPUT_DATA["data"], schema=INPUT_DATA["headers"], orient="row")
+    print("headers:", data.headers)
+    print("table:", data.rows_data_pl)

nfscache/database/oracle_env.py

@@ -0,0 +1,77 @@
+from argparse import Namespace
+from collections.abc import Callable, Mapping
+from pathlib import Path
+
+EnvField = tuple[str, Callable[[str], object]]
+
+ORACLE_ENV_FIELDS: Mapping[str, EnvField] = {
+    "ORACLE_HOST": ("host", str),
+    "ORACLE_PORT": ("port", int),
+    "ORACLE_SERVICE": ("service", str),
+    "ORACLE_USER": ("user", str),
+    "ORACLE_PASSWORD": ("password", str),
+    "ORACLE_TABLE": ("table", str),
+    "ORACLE_BATCH_SIZE": ("batch_size", int),
+}
+
+
+def find_dotenv() -> Path | None:
+    for path in (Path.cwd(), *Path.cwd().parents):
+        dotenv_path = path / ".env"
+        try:
+            if dotenv_path.is_file():
+                return dotenv_path
+        except OSError:
+            # Unreadable .env (e.g. restricted perms): treat as absent.
+            continue
+    return None
+
+
+def read_dotenv() -> dict[str, str]:
+    dotenv_path = find_dotenv()
+    if dotenv_path is None:
+        return {}
+
+    try:
+        text = dotenv_path.read_text(encoding="utf-8")
+    except OSError:
+        # Fall back to defaults rather than crashing on an unreadable .env.
+        return {}
+
+    values: dict[str, str] = {}
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        if stripped.startswith("export "):
+            stripped = stripped.removeprefix("export ").strip()
+        if "=" not in stripped:
+            continue
+
+        key, value = stripped.split("=", 1)
+        key = key.strip()
+        value = value.strip()
+        if (
+            len(value) >= 2
+            and value[0] == value[-1]
+            and value[0] in {"'", '"'}
+        ):
+            value = value[1:-1]
+        values[key] = value
+
+    return values
+
+
+def apply_dotenv(
+    args: Namespace,
+    *,
+    fields: Mapping[str, EnvField] = ORACLE_ENV_FIELDS,
+) -> None:
+    values = read_dotenv()
+    for env_key, (arg_name, coerce) in fields.items():
+        if env_key not in values:
+            continue
+        value = values[env_key]
+        if value == "":
+            continue
+        setattr(args, arg_name, coerce(value))

nfscache/database/oracle_pool.py

@@ -0,0 +1,69 @@
+"""Process-local oracledb connection pool, exposed as an NFSCache `connect_factory`.
+
+`NFSCache.connect_factory` is an opaque `Callable[[], connection]` that the cache
+uses as `with connect_factory() as conn:`. A pooled connection's `__exit__`
+*releases* it back to the pool instead of closing the socket, so wiring a pool in
+here removes the per-call `oracledb.connect` cost on every version probe and warm
+hit while keeping `nfs_cache.py` free of any oracledb dependency.
+
+Pools are cached per process (keyed by pid + DSN + user): connections are not
+shareable across processes (ProcessPoolExecutor workers each build their own), and
+a forked child must not reuse the parent's pool. The pid in the key guards that.
+"""
+
+import argparse
+import os
+import threading
+
+import oracledb
+
+_pools: dict[tuple[int, str, str], "oracledb.ConnectionPool"] = {}
+_lock = threading.Lock()
+
+
+def _dsn(args: argparse.Namespace) -> str:
+    return f"{args.host}:{args.port}/{args.service}"
+
+
+def get_pool(
+    args: argparse.Namespace,
+    *,
+    min_size: int = 1,
+    max_size: int = 4,
+) -> "oracledb.ConnectionPool":
+    """Return a process-local pool for `args`, creating it once per process."""
+    key = (os.getpid(), _dsn(args), args.user)
+    pool = _pools.get(key)
+    if pool is not None:
+        return pool
+
+    with _lock:
+        pool = _pools.get(key)
+        if pool is None:
+            pool = oracledb.create_pool(
+                user=args.user,
+                password=args.password,
+                dsn=_dsn(args),
+                min=min_size,
+                max=max_size,
+            )
+            _pools[key] = pool
+        return pool
+
+
+def make_pool_factory(
+    args: argparse.Namespace,
+    *,
+    min_size: int = 1,
+    max_size: int = 4,
+):
+    """Build a `connect_factory` that acquires from a process-local pool.
+
+    The pool is created lazily on first call, so building the factory never
+    touches the database (safe to do at import time).
+    """
+
+    def factory() -> "oracledb.Connection":
+        return get_pool(args, min_size=min_size, max_size=max_size).acquire()
+
+    return factory

nfscache/database/oracle_read.py

@@ -0,0 +1,100 @@
+import argparse
+import oracledb
+import polars as pl
+
+from nfscache.database.oracle_env import apply_dotenv
+from nfscache.database.oracle_pool import make_pool_factory
+from nfscache.util.main import nfscache
+from nfscache.data.data_container import DataContainer
+
+DEFAULT_BATCH_SIZE = 10000
+
+
+def oracle_args() -> argparse.Namespace:
+    args = argparse.Namespace(
+        host="localhost",
+        port=1521,
+        service="FREEPDB1",
+        user="SOMEUSER",
+        password="cache",
+        batch_size=DEFAULT_BATCH_SIZE,
+    )
+    apply_dotenv(args)
+    return args
+
+
+def connect(args: argparse.Namespace) -> oracledb.Connection:
+    dsn = f"{args.host}:{args.port}/{args.service}"
+    return oracledb.connect(
+        user=args.user,
+        password=args.password,
+        dsn=dsn,
+    )
+
+
+def _fetch_data_container(
+    connection: oracledb.Connection,
+    sql: str,
+    *,
+    batch_size: int,
+) -> DataContainer:
+    batches: list[pl.DataFrame] = []
+    with connection.cursor() as cursor:
+        cursor.arraysize = batch_size
+        cursor.execute(sql)
+        headers = tuple(column[0] for column in cursor.description)
+
+        while True:
+            rows = cursor.fetchmany(batch_size)
+            if not rows:
+                break
+            batches.append(pl.DataFrame(rows, schema=headers, orient="row"))
+
+    df = pl.concat(batches) if batches else pl.DataFrame(schema=headers)
+    return DataContainer({"headers": headers, "data": df})
+
+
+# Pool-backed factory: the version probe (every call) and the cold-load fetch
+# both borrow from one process-local pool instead of opening a fresh connection.
+nfscache.connect_factory = make_pool_factory(oracle_args())
+
+
+@nfscache.sql
+def read_data_container(sql: str) -> DataContainer:
+    # This body only runs on a cache miss, so reaching it means a live read.
+    print(f"Serving from Oracle (cache miss): {sql}", flush=True)
+    args = oracle_args()
+    with nfscache.connect_factory() as connection:
+        return _fetch_data_container(
+            connection,
+            sql,
+            batch_size=int(args.batch_size),
+        )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Read Oracle SQL into a DataContainer."
+    )
+    parser.add_argument("sql")
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=1521)
+    parser.add_argument("--service", default="FREEPDB1")
+    parser.add_argument("--user", default="SOMEUSER")
+    parser.add_argument("--password", default="cache")
+    parser.add_argument("--batch-size", type=int, default=10000)
+    args = parser.parse_args()
+    apply_dotenv(args)
+    return args
+
+
+def main() -> int:
+    args = parse_args()
+    # Go through the cache so the run reports Oracle (miss) vs cache (hit).
+    container = read_data_container(args.sql)
+    print("table:", container.data.rows_data_pl)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nfscache/database/oracle_write.py

@@ -0,0 +1,182 @@
+import argparse
+import re
+from pathlib import Path
+
+import oracledb
+import polars as pl
+
+from nfscache.database.oracle_env import apply_dotenv
+from nfscache.data.data_container import DataContainer
+
+IDENTIFIER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_$#]{0,127}$")
+
+INTEGER_TYPES = {
+    pl.Int8,
+    pl.Int16,
+    pl.Int32,
+    pl.Int64,
+    pl.UInt8,
+    pl.UInt16,
+    pl.UInt32,
+    pl.UInt64,
+}
+FLOAT_TYPES = {pl.Float32, pl.Float64}
+STRING_TYPES = {pl.String, pl.Categorical, pl.Enum}
+
+
+def oracle_identifier(name: str) -> str:
+    if not IDENTIFIER_RE.fullmatch(name):
+        raise ValueError(f"Invalid Oracle identifier: {name!r}")
+    return name.upper()
+
+
+def table_name_from_path(path: Path) -> str:
+    return oracle_identifier(path.stem)
+
+
+def oracle_type(dtype: pl.DataType) -> str:
+    if dtype in INTEGER_TYPES:
+        return "NUMBER(38)"
+    if dtype in FLOAT_TYPES:
+        return "BINARY_DOUBLE"
+    if dtype in STRING_TYPES:
+        return "VARCHAR2(4000)"
+    if dtype == pl.Boolean:
+        return "NUMBER(1)"
+    if dtype == pl.Date:
+        return "DATE"
+    if dtype == pl.Datetime:
+        return "TIMESTAMP"
+    return "CLOB"
+
+
+def read_data_container(path: Path) -> DataContainer:
+    print(f"Reading: {path}...")
+    df = pl.read_parquet(path)
+    return DataContainer({"headers": tuple(df.columns), "data": df})
+
+
+def connect(args: argparse.Namespace) -> oracledb.Connection:
+    dsn = f"{args.host}:{args.port}/{args.service}"
+    return oracledb.connect(
+        user=args.user,
+        password=args.password,
+        dsn=dsn,
+    )
+
+
+def current_scn(connection: oracledb.Connection) -> int:
+    with connection.cursor() as cursor:
+        scn, = cursor.execute("select current_scn from v$database").fetchone()
+    return int(scn)
+
+
+def drop_table_if_exists(connection: oracledb.Connection, table_name: str) -> None:
+    with connection.cursor() as cursor:
+        try:
+            cursor.execute(f"drop table {table_name} purge")
+        except oracledb.DatabaseError as exc:
+            error, = exc.args
+            if getattr(error, "code", None) != 942:
+                raise
+
+
+def create_table(
+    connection: oracledb.Connection,
+    table_name: str,
+    df: pl.DataFrame,
+) -> list[str]:
+    columns = [oracle_identifier(column) for column in df.columns]
+    definitions = [
+        f"{column} {oracle_type(dtype)}"
+        for column, dtype in zip(columns, df.dtypes, strict=True)
+    ]
+    ddl = f"create table {table_name} ({', '.join(definitions)})"
+    with connection.cursor() as cursor:
+        cursor.execute(ddl)
+    return columns
+
+
+def insert_data_container(
+    connection: oracledb.Connection,
+    table_name: str,
+    data_container: DataContainer,
+    *,
+    batch_size: int,
+) -> tuple[int, int]:
+    df = data_container.data.rows_data_pl
+    if not isinstance(df, pl.DataFrame):
+        raise TypeError("DataContainer.data.rows_data_pl must be a Polars DataFrame")
+
+    columns = create_table(connection, table_name, df)
+    placeholders = ", ".join(f":{index}" for index in range(1, len(columns) + 1))
+    sql = (
+        f"insert into {table_name} ({', '.join(columns)}) "
+        f"values ({placeholders})"
+    )
+
+    inserted = 0
+    with connection.cursor() as cursor:
+        for batch in df.iter_slices(n_rows=batch_size):
+            rows = list(batch.iter_rows(named=False))
+            if not rows:
+                continue
+            cursor.executemany(sql, rows)
+            inserted += len(rows)
+    connection.commit()
+    return inserted, len(columns)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Read a parquet-backed DataContainer and write it to Oracle."
+    )
+    parser.add_argument("parquet_path", type=Path)
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=1521)
+    parser.add_argument("--service", default="FREEPDB1")
+    parser.add_argument("--user", default="SOMEUSER")
+    parser.add_argument("--password", default="cache")
+    parser.add_argument(
+        "--table",
+        help="Oracle table name. Defaults to the parquet file stem.",
+    )
+    parser.add_argument("--batch-size", type=int, default=1000)
+    args = parser.parse_args()
+    apply_dotenv(args)
+    return args
+
+
+def main() -> int:
+    args = parse_args()
+    path = args.parquet_path
+    table_name = (
+        oracle_identifier(args.table)
+        if args.table is not None
+        else table_name_from_path(path)
+    )
+    data_container = read_data_container(path)
+    df = data_container.data.rows_data_pl
+    if not isinstance(df, pl.DataFrame):
+        raise TypeError("DataContainer.data.rows_data_pl must be a Polars DataFrame")
+
+    print(f"DataContainer: rows={df.height} cols={df.width}")
+    with connect(args) as connection:
+        before_scn = current_scn(connection)
+        print(f"Oracle current_scn before write: {before_scn}")
+        drop_table_if_exists(connection, table_name)
+        rows, cols = insert_data_container(
+            connection,
+            table_name,
+            data_container,
+            batch_size=int(args.batch_size),
+        )
+        after_scn = current_scn(connection)
+
+    print(f"Wrote DataContainer to {args.user.upper()}.{table_name}: rows={rows} cols={cols}")
+    print(f"Oracle current_scn after write: {after_scn}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nfscache/database/oracle_write_container.py

@@ -0,0 +1,186 @@
+import argparse
+import re
+from pathlib import Path
+
+import oracledb
+import polars as pl
+
+from nfscache.database.oracle_env import apply_dotenv
+from nfscache.data.data_container import DataContainer
+from nfscache.util.generate_parquets import ensure_one_parquet
+
+IDENTIFIER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_$#]{0,127}$")
+
+INTEGER_TYPES = {
+    pl.Int8,
+    pl.Int16,
+    pl.Int32,
+    pl.Int64,
+    pl.UInt8,
+    pl.UInt16,
+    pl.UInt32,
+    pl.UInt64,
+}
+FLOAT_TYPES = {pl.Float32, pl.Float64}
+STRING_TYPES = {pl.String, pl.Categorical, pl.Enum}
+
+
+def oracle_identifier(name: str) -> str:
+    if not IDENTIFIER_RE.fullmatch(name):
+        raise ValueError(f"Invalid Oracle identifier: {name!r}")
+    return name.upper()
+
+
+def oracle_type(dtype: pl.DataType) -> str:
+    if dtype in INTEGER_TYPES:
+        return "NUMBER(38)"
+    if dtype in FLOAT_TYPES:
+        return "BINARY_DOUBLE"
+    if dtype in STRING_TYPES:
+        return "VARCHAR2(4000)"
+    if dtype == pl.Boolean:
+        return "NUMBER(1)"
+    if dtype == pl.Date:
+        return "DATE"
+    if dtype == pl.Datetime:
+        return "TIMESTAMP"
+    return "CLOB"
+
+
+def connect(args: argparse.Namespace) -> oracledb.Connection:
+    dsn = f"{args.host}:{args.port}/{args.service}"
+    return oracledb.connect(
+        user=args.user,
+        password=args.password,
+        dsn=dsn,
+    )
+
+
+def make_data_container(args: argparse.Namespace) -> tuple[Path, DataContainer]:
+    path = ensure_one_parquet(
+        out_dir=Path(args.data_dir),
+        base_name=f"ORACLE_TEST_{args.rows}.parquet",
+        prefix="A_",
+        n_rows=int(args.rows),
+        n_cols=int(args.cols),
+        seed=None,
+        float_scale=float(args.float_scale),
+        n_int_cols=int(args.n_int_cols),
+        n_str_cols=int(args.n_str_cols),
+    )
+    df = pl.read_parquet(path)
+    return path, DataContainer({"headers": tuple(df.columns), "data": df})
+
+
+def current_scn(connection: oracledb.Connection) -> int:
+    with connection.cursor() as cursor:
+        scn, = cursor.execute("select current_scn from v$database").fetchone()
+    return int(scn)
+
+
+def drop_table_if_exists(connection: oracledb.Connection, table_name: str) -> None:
+    with connection.cursor() as cursor:
+        try:
+            cursor.execute(f"drop table {table_name} purge")
+        except oracledb.DatabaseError as exc:
+            error, = exc.args
+            if getattr(error, "code", None) != 942:
+                raise
+
+
+def create_table(
+    connection: oracledb.Connection,
+    table_name: str,
+    df: pl.DataFrame,
+) -> list[str]:
+    columns = [oracle_identifier(column) for column in df.columns]
+    definitions = [
+        f"{column} {oracle_type(dtype)}"
+        for column, dtype in zip(columns, df.dtypes, strict=True)
+    ]
+    ddl = f"create table {table_name} ({', '.join(definitions)})"
+    with connection.cursor() as cursor:
+        cursor.execute(ddl)
+    return columns
+
+
+def insert_data_container(
+    connection: oracledb.Connection,
+    table_name: str,
+    data_container: DataContainer,
+    *,
+    batch_size: int,
+) -> tuple[int, int]:
+    df = data_container.data.rows_data_pl
+    if not isinstance(df, pl.DataFrame):
+        raise TypeError("DataContainer.data.rows_data_pl must be a Polars DataFrame")
+
+    columns = create_table(connection, table_name, df)
+    placeholders = ", ".join(f":{index}" for index in range(1, len(columns) + 1))
+    sql = (
+        f"insert into {table_name} ({', '.join(columns)}) "
+        f"values ({placeholders})"
+    )
+
+    inserted = 0
+    with connection.cursor() as cursor:
+        for batch in df.iter_slices(n_rows=batch_size):
+            rows = list(batch.iter_rows(named=False))
+            if not rows:
+                continue
+            cursor.executemany(sql, rows)
+            inserted += len(rows)
+    connection.commit()
+    return inserted, len(columns)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Generate a DataContainer and write it to the local Oracle DB."
+    )
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=1521)
+    parser.add_argument("--service", default="FREEPDB1")
+    parser.add_argument("--user", default="SOMEUSER")
+    parser.add_argument("--password", default="cache")
+    parser.add_argument("--table", default="DATA_CONTAINER_DEMO")
+    parser.add_argument("--rows", type=int, default=4096)
+    parser.add_argument("--cols", type=int, default=20)
+    parser.add_argument("--batch-size", type=int, default=1000)
+    parser.add_argument("--data-dir", default="parquet")
+    parser.add_argument("--float-scale", type=float, default=5.0)
+    parser.add_argument("--n-int-cols", type=int, default=4)
+    parser.add_argument("--n-str-cols", type=int, default=8)
+    args = parser.parse_args()
+    apply_dotenv(args)
+    return args
+
+
+def main() -> int:
+    args = parse_args()
+    table_name = oracle_identifier(args.table)
+    path, data_container = make_data_container(args)
+    df = data_container.data.rows_data_pl
+    if not isinstance(df, pl.DataFrame):
+        raise TypeError("DataContainer.data.rows_data_pl must be a Polars DataFrame")
+
+    print(f"Generated: {path} rows={df.height} cols={df.width}")
+    with connect(args) as connection:
+        before_scn = current_scn(connection)
+        print(f"Oracle current_scn before write: {before_scn}")
+        drop_table_if_exists(connection, table_name)
+        rows, cols = insert_data_container(
+            connection,
+            table_name,
+            data_container,
+            batch_size=int(args.batch_size),
+        )
+        after_scn = current_scn(connection)
+
+    print(f"Wrote DataContainer to {args.user.upper()}.{table_name}: rows={rows} cols={cols}")
+    print(f"Oracle current_scn after write: {after_scn}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())