c2r-collect 0.1.0__tar.gz

c2r_collect-0.1.0/MANIFEST.in

@@ -0,0 +1 @@
+include tests/conftest.py

c2r_collect-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: c2r-collect
+Version: 0.1.0
+Summary: Oracle data dictionary + PL/SQL source collector that writes okf-loom input folders
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: oracledb>=2.0
+Requires-Dist: tomli>=2.0; python_version < "3.11"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == "s3"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+
+# c2r-collect
+
+Pulls PL/SQL package sources, relationship CSVs, dictionary and codebook tables from an Oracle
+schema into `raw/<date>/`, then builds an okf-loom input folder from that raw.
+
+```
+pip install c2r-collect             # add [s3] extra for upload/download
+cp config.example.toml config.toml  # edit; password via env C2R_DB_PASSWORD
+c2r-collect collect                 # → <root>/raw/<today>/
+c2r-collect upload   --date YYYY-MM-DD
+c2r-collect download --date YYYY-MM-DD
+c2r-collect make-input --date YYYY-MM-DD [--prev YYYY-MM-DD] [--add add.txt]
+```
+
+Design and release notes live in the source repository (`docs/`), not in the package.
+
+## First run checks
+See `docs/release.md` (release steps, QA-DB run, VDI production run).

c2r_collect-0.1.0/README.md

@@ -0,0 +1,18 @@
+# c2r-collect
+
+Pulls PL/SQL package sources, relationship CSVs, dictionary and codebook tables from an Oracle
+schema into `raw/<date>/`, then builds an okf-loom input folder from that raw.
+
+```
+pip install c2r-collect             # add [s3] extra for upload/download
+cp config.example.toml config.toml  # edit; password via env C2R_DB_PASSWORD
+c2r-collect collect                 # → <root>/raw/<today>/
+c2r-collect upload   --date YYYY-MM-DD
+c2r-collect download --date YYYY-MM-DD
+c2r-collect make-input --date YYYY-MM-DD [--prev YYYY-MM-DD] [--add add.txt]
+```
+
+Design and release notes live in the source repository (`docs/`), not in the package.
+
+## First run checks
+See `docs/release.md` (release steps, QA-DB run, VDI production run).

c2r_collect-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "c2r-collect"
+version = "0.1.0"
+description = "Oracle data dictionary + PL/SQL source collector that writes okf-loom input folders"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+  "oracledb>=2.0",
+  "tomli>=2.0; python_version < '3.11'",
+]
+
+[project.optional-dependencies]
+s3 = ["boto3>=1.34"]
+dev = ["pytest>=8", "build", "twine"]
+
+[project.scripts]
+c2r-collect = "c2r_collect.cli:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+c2r_collect = ["queries/*.sql", "templates/*.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

c2r_collect-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

c2r_collect-0.1.0/src/c2r_collect/__init__.py

@@ -0,0 +1,13 @@
+"""c2r-collect: Oracle data dictionary + PL/SQL package sources -> okf-loom input folders.
+
+PURPOSE   Replace manual DBA spool requests. Pull everything from one Oracle schema into a dated
+          raw/ folder, then derive the folder okf-loom (a separate, frozen tool) reads.
+ENTRY     console script `c2r-collect` -> c2r_collect.cli:main. Run `c2r-collect --help`.
+PIPELINE  collect (DB -> raw/<date>/) -> [upload -> download] -> make-input (raw -> input/<date>/)
+          -> okf-loom enrich/build (outside this package).
+MODULES   cli (args) . config (config.toml) . db (ONLY module touching Oracle) . queries/*.sql
+          . codebook (dynamic SQL) . writers (file contract) . counters . report . collect
+          . make_input . transfer (S3) . discover
+EXIT      0 ok . 2 input/config error or partial success . 3 DB/required output failed . 4 transfer failed
+"""
+__version__ = "0.1.0"

c2r_collect-0.1.0/src/c2r_collect/__main__.py

@@ -0,0 +1,6 @@
+"""`python -m c2r_collect ...` == `c2r-collect ...` (works even when Scripts/ is not on PATH, e.g. pip --user on VDI)."""
+import sys
+
+from .cli import main
+
+sys.exit(main())

c2r_collect-0.1.0/src/c2r_collect/cli.py

@@ -0,0 +1,183 @@
+"""Command line interface. Parses arguments and wires modules; holds no business logic.
+
+PURPOSE   One entry point for humans, schedulers and LLM agents (`c2r-collect <command>`).
+INPUTS    config file (--config PATH, default ./config.toml; may be given before or after the command)
+          env C2R_DB_PASSWORD (collect, discover-codebook; prompts if unset)
+          env AWS_* or EC2 instance role (upload, download; read by boto3, never by this code)
+OUTPUTS   see each run_* function. Console output is ASCII-only; the LAST line is either
+          `ok: <path> ...` or `error: <reason>`, so a caller can parse it.
+EXIT      0 ok . 2 input/config error, or collect finished with non-required errors
+          . 3 DB connect/required output failed . 4 S3 transfer failed
+NOT HERE  any decision about what to collect or select (collect.py, make_input.py).
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import os
+import platform
+import re
+import sys
+import traceback
+from pathlib import Path
+from typing import List, Optional
+
+from . import __version__
+from .collect import EXIT_DB, EXIT_INPUT, run_collect
+from .config import Config, ConfigError, load_config, resolve_password
+from .db import DbError
+from .discover import run_discover
+from .make_input import run_make_input
+from .transfer import make_client, run_download, run_upload
+
+DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+DEFAULT_CONFIG = "config.toml"
+
+# ASCII only: Windows cp949 consoles (VDI) garble arrows and dashes.
+EPILOG = """\
+PIPELINE (one cycle; <root> = config [paths] root):
+  1. collect      DB -> <root>/raw/<date>/            pulls ALL packages, CSVs, dictionary, codebook
+  2. upload       <root>/raw/<date>/ -> s3://<bucket>/<prefix>/<date>/     (when collect ran on VDI)
+  3. download     s3 -> <root>/raw/<date>/                                  (on the build PC)
+  4. make-input   <root>/raw/<date>/ -> <root>/input/<date>/   okf-loom input
+                  selects packages with a previous description (--prev) plus --add names
+  5. okf-loom enrich/build --input <root>/input/<date>      (separate tool, not run here)
+
+OUTPUTS:
+  raw/<date>/collect-report.md      read this first after collect (errors, counts, '?' evidence)
+  raw/<date>/MANIFEST.sha256        integrity; verified by upload/download
+  input/<date>/make-input-report.md selected / added / vanished packages
+
+ENVIRONMENT:
+  C2R_DB_PASSWORD   DB password for collect and discover-codebook (prompted if unset)
+  AWS_*             upload/download credentials (SSO portal keys); EC2 uses its instance role
+
+EXIT CODES:
+  0 ok
+  2 input or config error; or collect finished but a non-required file failed (see report)
+  3 cannot connect, or a required output failed (raw/<date>.partial/ kept for inspection)
+  4 S3 transfer failed (credentials, AccessDenied, network, manifest mismatch)
+
+LAST LINE of output is always one of:
+  ok: <path> ...                      exit 0
+  partial: <path> - N problem(s) ...  exit 2 (collect wrote raw/<date>/ but some files are missing)
+  error: <reason>                     exit 2, 3 or 4
+"""
+
+
+def _default_db_factory(cfg: Config, password: str):
+    from .db import OracleDb
+    return OracleDb(cfg.db.dsn, cfg.db.user, password)
+
+
+def _driver_version() -> str:
+    try:
+        import oracledb
+        return f"python-oracledb {oracledb.__version__}"
+    except Exception:  # pragma: no cover
+        return "python-oracledb (not installed)"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """argparse parser. The epilog is the machine-oriented manual shown by --help."""
+    p = argparse.ArgumentParser(prog="c2r-collect", description="Oracle -> okf-loom input collector",
+                                epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--version", action="version", version=f"c2r-collect {__version__}")
+    p.add_argument("--config", dest="config_top", metavar="PATH", default=None, help=f"path to config (default ./{DEFAULT_CONFIG})")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    def sp(name, help_, dated=True):
+        s = sub.add_parser(name, help=help_)
+        s.add_argument("--config", dest="config_sub", default=None, help=argparse.SUPPRESS)
+        if dated:
+            s.add_argument("--date", default=dt.date.today().isoformat(), help="YYYY-MM-DD (default today)")
+            s.add_argument("--force", action="store_true", help="overwrite an existing output folder")
+        return s
+
+    sp("collect", "DB -> raw/<date>/")
+    mi = sp("make-input", "raw/<date>/ -> input/<date>/ for okf-loom")
+    mi.add_argument("--prev", help="previous input date whose enrichments/ to carry (default: latest before --date)")
+    mi.add_argument("--add", help="file path or http(s) URL listing extra package names")
+    sp("upload", "raw/<date>/ -> S3")
+    sp("download", "S3 -> raw/<date>/")
+    sp("discover-codebook", "list candidate codebook tables by name pattern", dated=False)
+    return p
+
+
+def main(argv: Optional[List[str]] = None, db_factory=_default_db_factory, client_factory=make_client,
+         env=None) -> int:
+    try:
+        return _main(argv, db_factory, client_factory, env)
+    except SystemExit:
+        raise
+    except Exception as e:   # keep the last-line contract even for bugs, disk full, locked files
+        traceback.print_exc(file=sys.stderr)
+        print(f"error: unexpected {type(e).__name__}: {str(e).splitlines()[0] if str(e) else ''}")
+        return EXIT_DB
+
+
+def _main(argv, db_factory, client_factory, env) -> int:
+    """Run one command and return its exit code (also used as the process exit status).
+
+        Args:
+            argv: arguments without the program name; None means sys.argv[1:].
+            db_factory: (Config, password) -> Db. Tests inject FakeDb here.
+            client_factory: (region) -> boto3-like S3 client. Tests inject a fake.
+            env: environment mapping; None means os.environ.
+        Returns:
+            0, 2, 3 or 4 (see module docstring). Never raises for expected failures.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(errors="replace")   # cp949 consoles must not crash on a stray character
+    args = build_parser().parse_args(argv)
+    env = os.environ if env is None else env
+    cfg_path = Path(args.config_sub or args.config_top or DEFAULT_CONFIG)
+    try:
+        cfg = load_config(cfg_path)
+    except ConfigError as e:
+        print(f"error: {e}")
+        return EXIT_INPUT
+    date = getattr(args, "date", None)
+    if date is not None and not DATE_RE.match(date):
+        print(f"error: --date must be YYYY-MM-DD, got {date!r}")
+        return EXIT_INPUT
+
+    if args.cmd in ("collect", "discover-codebook"):
+        try:
+            pw = resolve_password(env)
+        except ConfigError as e:
+            print(f"error: {e}")
+            return EXIT_INPUT
+        try:
+            db = db_factory(cfg, pw)
+        except DbError as e:
+            print(f"error: cannot connect: {e}")
+            return EXIT_DB
+        try:
+            if args.cmd == "collect":
+                return run_collect(cfg, db, date, force=args.force, host=platform.node(), driver=_driver_version())
+            return run_discover(cfg, db)
+        finally:
+            db.close()
+
+    if args.cmd == "make-input":
+        return run_make_input(cfg.root, date, cfg.scope.source_owner, args.prev, args.add, force=args.force)
+
+    if cfg.s3 is None:
+        print("error: [s3] section missing in config")
+        return EXIT_INPUT
+    try:
+        client = client_factory(cfg.s3.region)
+    except ImportError:
+        print("error: boto3 not installed - pip install 'c2r-collect[s3]'")
+        return EXIT_INPUT
+    if args.cmd == "upload":
+        return run_upload(cfg.root, date, cfg.s3, client)
+    if args.force:
+        print("note: --force is ignored for download (an existing raw/<date>/ is never overwritten)")
+    return run_download(cfg.root, date, cfg.s3, client)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

c2r_collect-0.1.0/src/c2r_collect/codebook.py

@@ -0,0 +1,107 @@
+"""SQL builders for codebook (parameter/code) tables and for discover-codebook.
+
+PURPOSE   Codebook queries need table names in the SQL text (Oracle cannot bind identifiers).
+SECURITY  every owner/table name passes check_identifier() before being inlined:
+          ^[A-Z][A-Z0-9_$#]{0,127}$ after upper-casing. Anything else raises ValueError.
+          No other value is ever inlined; there are no user-supplied literals.
+OUTPUTS   SQL strings only. Execution happens in collect.py / discover.py via Db.query_sql.
+"""
+from __future__ import annotations
+
+import re
+from typing import Sequence
+
+IDENT = re.compile(r"^[A-Z][A-Z0-9_$#]{0,127}$")
+PATTERN = re.compile(r"^[A-Z0-9_$#%]{1,128}$")
+
+# Generic, vendor-neutral defaults. Override per site in config [scope] discover_include / discover_exclude.
+DEFAULT_DISCOVER_INCLUDE = ("%CODE%", "%PARAM%", "%LABEL%", "%LANG%", "%MESSAGE%", "%LIST%", "%TYPE%", "REF%", "%REF")
+DEFAULT_DISCOVER_EXCLUDE = ("TMP%", "STG%", "TEST%", "%BKP%", "%BACKUP%", "%OLD")
+
+
+def check_identifier(s: str) -> str:
+    """Upper-case and validate an Oracle identifier before it is inlined into SQL.
+
+    Raises:
+        ValueError: not ^[A-Z][A-Z0-9_$#]{0,127}$ (blocks injection such as "T; DROP").
+    """
+    u = (s or "").upper()
+    if not IDENT.match(u):
+        raise ValueError(f"invalid Oracle identifier: {s!r}")
+    return u
+
+
+def check_pattern(s: str) -> str:
+    """Validate a LIKE pattern from config before inlining (letters, digits, _ $ # % only)."""
+    u = (s or "").upper()
+    if not PATTERN.match(u):
+        raise ValueError(f"invalid LIKE pattern: {s!r}")
+    return u
+
+
+def _names(tables: Sequence[str]) -> list:
+    names = [check_identifier(t) for t in tables]
+    if not names:
+        raise ValueError("codebook table list is empty")
+    return names
+
+
+def _in_list(tables: Sequence[str]) -> str:
+    return ", ".join(f"'{n}'" for n in _names(tables))
+
+
+def _name_rows(tables: Sequence[str]) -> str:
+    return " UNION ALL ".join(f"SELECT '{n}' AS table_name FROM dual" for n in _names(tables))
+
+
+def sql_exists(owner: str, tables: Sequence[str]) -> str:
+    o = check_identifier(owner)
+    return (
+        "SELECT g.table_name, "
+        "CASE WHEN t.table_name IS NULL THEN 'MISSING' ELSE 'EXISTS' END AS status, "
+        "t.num_rows AS stat_rows, t.last_analyzed, "
+        f"(SELECT COUNT(*) FROM all_tab_columns c WHERE c.owner = '{o}' AND c.table_name = g.table_name) AS col_cnt "
+        f"FROM ({_name_rows(tables)}) g "
+        f"LEFT JOIN all_tables t ON t.owner = '{o}' AND t.table_name = g.table_name "
+        "ORDER BY g.table_name"
+    )
+
+
+def sql_rowcount_one(owner: str, table: str) -> str:
+    """COUNT(*) for one table. One statement per table so a missing table loses only its own row."""
+    o, n = check_identifier(owner), check_identifier(table)
+    return f"SELECT '{n}' AS table_name, COUNT(*) AS real_rows FROM {o}.{n}"
+
+
+def sql_owners(tables: Sequence[str]) -> str:
+    return ("SELECT t.owner, t.table_name, t.num_rows AS stat_rows, t.last_analyzed "
+            f"FROM all_tables t WHERE t.table_name IN ({_in_list(tables)}) "
+            "ORDER BY t.table_name, t.owner")
+
+
+def sql_columns(owner: str, tables: Sequence[str]) -> str:
+    o = check_identifier(owner)
+    return ("SELECT c.table_name, c.column_id, c.column_name, c.data_type, c.data_length, c.nullable, cc.comments "
+            "FROM all_tab_columns c "
+            "LEFT JOIN all_col_comments cc ON cc.owner = c.owner AND cc.table_name = c.table_name "
+            "AND cc.column_name = c.column_name "
+            f"WHERE c.owner = '{o}' AND c.table_name IN ({_in_list(tables)}) "
+            "ORDER BY c.table_name, c.column_id")
+
+
+def sql_dump(owner: str, table: str) -> str:
+    return f"SELECT * FROM {check_identifier(owner)}.{check_identifier(table)}"
+
+
+def sql_discover(owner: str, include: Sequence[str] = DEFAULT_DISCOVER_INCLUDE,
+                 exclude: Sequence[str] = DEFAULT_DISCOVER_EXCLUDE) -> str:
+    o = check_identifier(owner)
+    inc = [check_pattern(p) for p in include] or ["%"]
+    exc = [check_pattern(p) for p in exclude]
+    inc_sql = " OR ".join(f"UPPER(t.table_name) LIKE '{p}'" for p in inc)
+    exc_sql = " AND ".join(f"t.table_name NOT LIKE '{p}'" for p in exc) or "1 = 1"
+    return ("SELECT t.table_name, t.num_rows AS stat_rows, "
+            "(SELECT COUNT(*) FROM all_tab_columns c WHERE c.owner = t.owner AND c.table_name = t.table_name) AS col_cnt, "
+            "(SELECT tc.comments FROM all_tab_comments tc WHERE tc.owner = t.owner AND tc.table_name = t.table_name) AS table_comment "
+            f"FROM all_tables t WHERE t.owner = '{o}' AND ({inc_sql}) AND {exc_sql} "
+            "ORDER BY t.table_name")