inthub-cli 0.1.4__py3-none-any.whl

inthub/__main__.py

@@ -0,0 +1,4 @@
+from inthub.cli import main
+
+if __name__ == "__main__":
+    main()

inthub/bulk/csv_state.py

@@ -0,0 +1,118 @@
+import csv
+import json
+import os
+import tempfile
+from pathlib import Path
+
+BULK_DIR = Path.home() / ".inthub" / "bulk"
+
+REQUIRED_COLUMNS = [
+    "sqlserver_instance",
+    "sqlserver_port",
+    "sqlserver_db",
+    "sqlserver_gcp_secret",
+    "connector_name",
+    "tables",
+    "snowflake_db",
+    "snowflake_gcp_secret",
+    "kcc_source",
+    "kcc_sink",
+    "snowflake_wh",
+    "snowflake_account",
+]
+
+_STATUS_COL = "status"
+_REASON_COL = "reason"
+
+
+def _progress_path(ulid: str) -> Path:
+    return BULK_DIR / f"csv_{ulid}.csv"
+
+
+def _state_path(ulid: str) -> Path:
+    return BULK_DIR / f"state_{ulid}.json"
+
+
+def validate_columns(path: Path) -> None:
+    with path.open(newline="") as fh:
+        reader = csv.DictReader(fh)
+        actual = set(reader.fieldnames or [])
+    missing = set(REQUIRED_COLUMNS) - actual
+    if missing:
+        raise ValueError(f"CSV missing columns: {', '.join(sorted(missing))}")
+
+
+def init_progress_csv(source: Path, ulid: str) -> Path:
+    BULK_DIR.mkdir(parents=True, exist_ok=True)
+    progress = _progress_path(ulid)
+    with source.open(newline="") as src_fh:
+        reader = csv.DictReader(src_fh)
+        fieldnames = list(reader.fieldnames or []) + [_STATUS_COL, _REASON_COL]
+        rows = []
+        for row in reader:
+            row[_STATUS_COL] = "pending"
+            row[_REASON_COL] = ""
+            rows.append(row)
+
+    with progress.open("w", newline="") as dst_fh:
+        writer = csv.DictWriter(dst_fh, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+    return progress
+
+
+def load_rows(progress_path: Path) -> list[dict[str, str]]:
+    with progress_path.open(newline="") as fh:
+        return [{k: str(v) for k, v in row.items()} for row in csv.DictReader(fh)]
+
+
+def update_row(
+    progress_path: Path,
+    connector_name: str,
+    status: str,
+    reason: str = "",
+) -> None:
+    rows = load_rows(progress_path)
+    fieldnames = list(rows[0].keys()) if rows else []
+    for row in rows:
+        if row["connector_name"] == connector_name:
+            row[_STATUS_COL] = status
+            row[_REASON_COL] = reason
+
+    fd, tmp = tempfile.mkstemp(dir=progress_path.parent, suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", newline="") as fh:
+            writer = csv.DictWriter(fh, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(rows)
+        os.replace(tmp, progress_path)
+    except Exception:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+
+
+def save_state(
+    ulid: str,
+    gcp_project: str,
+    sqlserver_source_plugin: str,
+    snowflake_sink_plugin: str,
+    tags: str,
+) -> None:
+    BULK_DIR.mkdir(parents=True, exist_ok=True)
+    _state_path(ulid).write_text(json.dumps({
+        "gcp_project": gcp_project,
+        "sqlserver_source_plugin": sqlserver_source_plugin,
+        "snowflake_sink_plugin": snowflake_sink_plugin,
+        "tags": tags,
+    }, indent=2))
+
+
+def load_state(ulid: str) -> dict[str, str]:
+    path = _state_path(ulid)
+    if not path.exists():
+        raise FileNotFoundError(f"State file not found: {path}")
+    return json.loads(path.read_text())  # type: ignore[no-any-return]

inthub/bulk/ddl_converter.py

@@ -0,0 +1,86 @@
+from typing import Any
+
+_TYPE_MAP: dict[str, str] = {
+    "int": "NUMBER",
+    "bigint": "NUMBER",
+    "smallint": "NUMBER",
+    "tinyint": "NUMBER",
+    "float": "FLOAT",
+    "real": "FLOAT",
+    "text": "TEXT",
+    "ntext": "TEXT",
+    "datetime": "TIMESTAMP_NTZ",
+    "datetime2": "TIMESTAMP_NTZ",
+    "smalldatetime": "TIMESTAMP_NTZ",
+    "date": "DATE",
+    "time": "TIME",
+    "bit": "BOOLEAN",
+    "varbinary": "BINARY",
+    "binary": "BINARY",
+    "image": "BINARY",
+    "uniqueidentifier": "VARCHAR(36)",
+    "xml": "VARIANT",
+    "money": "NUMBER(19,4)",
+    "smallmoney": "NUMBER(19,4)",
+}
+
+_IH_COLUMNS = (
+    "  IH_TOPIC     VARCHAR(255) NOT NULL,\n"
+    "  IH_PARTITION INT          NOT NULL,\n"
+    "  IH_OFFSET    INT          NOT NULL,\n"
+    "  IH_OP        VARCHAR(1)   NOT NULL,\n"
+    "  CONSTRAINT PKEY PRIMARY KEY (IH_TOPIC, IH_PARTITION, IH_OFFSET)"
+)
+
+
+def convert_type(
+    sql_server_type: str,
+    max_length: int | None,
+    precision: int | None,
+    scale: int | None,
+) -> str:
+    base = sql_server_type.lower().split("(")[0].strip()
+
+    if base in ("decimal", "numeric"):
+        if precision is not None and scale is not None:
+            return f"NUMBER({precision},{scale})"
+        return "NUMBER"
+
+    if base in ("varchar", "nvarchar", "char", "nchar"):
+        if max_length and max_length > 0:
+            return f"VARCHAR({max_length})"
+        return "TEXT"
+
+    return _TYPE_MAP.get(base, "VARCHAR(255)")
+
+
+def to_create_table_ddl(
+    database: str,
+    schema: str,
+    table: str,
+    columns: list[dict[str, Any]],
+) -> str:
+    col_defs: list[str] = []
+    for col in columns:
+        name = col["COLUMN_NAME"]
+        sql_type = str(col.get("DATA_TYPE", "varchar"))
+        max_len_raw = col.get("CHARACTER_MAXIMUM_LENGTH")
+        precision_raw = col.get("NUMERIC_PRECISION")
+        scale_raw = col.get("NUMERIC_SCALE")
+        nullable = str(col.get("IS_NULLABLE", "YES")).upper() != "NO"
+
+        max_len = int(str(max_len_raw)) if max_len_raw not in (None, "", -1) else None
+        precision = int(str(precision_raw)) if precision_raw not in (None, "") else None
+        scale = int(str(scale_raw)) if scale_raw not in (None, "") else None
+
+        sf_type = convert_type(sql_type, max_len, precision, scale)
+        null_clause = "" if nullable else " NOT NULL"
+        col_defs.append(f'  "{name}" {sf_type}{null_clause}')
+
+    col_block = ",\n".join(col_defs)
+    return (
+        f'CREATE TABLE IF NOT EXISTS "{database}"."{schema}"."{table}_INGEST" (\n'
+        f"{col_block},\n"
+        f"{_IH_COLUMNS}\n"
+        f");"
+    )

inthub/bulk/gcp_secrets.py

@@ -0,0 +1,23 @@
+import json
+
+from google.cloud import secretmanager
+
+
+def fetch_secret(project_id: str, secret_name: str) -> dict[str, str]:
+    client = secretmanager.SecretManagerServiceClient()
+    resource = f"projects/{project_id}/secrets/{secret_name}/versions/latest"
+    response = client.access_secret_version(request={"name": resource})
+    payload = response.payload.data.decode("UTF-8")
+
+    try:
+        data: dict[str, str] = json.loads(payload)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Secret '{secret_name}' is not valid JSON") from exc
+
+    missing = {"username", "password"} - set(data.keys())
+    if missing:
+        raise ValueError(
+            f"Secret '{secret_name}' missing required fields: {', '.join(sorted(missing))}"
+        )
+
+    return {"username": data["username"], "password": data["password"]}

inthub/bulk/snowflake_client.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+
+def connect(account: str, user: str, password: str) -> Any:
+    import snowflake.connector
+
+    return snowflake.connector.connect(
+        account=account,
+        user=user,
+        password=password,
+    )
+
+
+def ingest_table_exists(conn: Any, database: str, schema: str, table: str) -> bool:
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT 1 FROM INFORMATION_SCHEMA.TABLES "
+        "WHERE TABLE_CATALOG = %s AND TABLE_SCHEMA = %s AND TABLE_NAME = %s",
+        (database.upper(), schema.upper(), f"{table.upper()}_INGEST"),
+    )
+    return cursor.fetchone() is not None
+
+
+def create_ingest_table(conn: Any, ddl: str) -> None:
+    conn.cursor().execute(ddl)
+
+
+def create_view(
+    conn: Any,
+    database: str,
+    schema: str,
+    table: str,
+    columns: list[str],
+) -> None:
+    col_list = ", ".join(f'"{c}"' for c in columns)
+    sql = (
+        f'CREATE OR REPLACE VIEW "{database}"."{schema}"."{table}" AS '
+        f'SELECT {col_list} FROM "{database}"."{schema}"."{table}_INGEST"'
+    )
+    conn.cursor().execute(sql)
+
+
+def create_stage(conn: Any, database: str, schema: str, table: str) -> None:
+    sql = (
+        f'CREATE OR REPLACE STAGE "{database}"."{schema}"."{table}" '
+        "FILE_FORMAT = ("
+        "TYPE = 'CSV', "
+        "FIELD_OPTIONALLY_ENCLOSED_BY = '\"', "
+        "SKIP_HEADER = 0, "
+        "FIELD_DELIMITER = ',', "
+        "NULL_IF = ('\\\\N', 'NULL')"
+        ")"
+    )
+    conn.cursor().execute(sql)
+
+
+def row_count(conn: Any, database: str, schema: str, table: str) -> int:
+    cursor = conn.cursor()
+    cursor.execute(f'SELECT COUNT(*) FROM "{database}"."{schema}"."{table}"')
+    row = cursor.fetchone()
+    return int(row[0]) if row else 0

inthub/bulk/sqlserver.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+
+def connect(host: str, port: int, db: str, user: str, password: str) -> Any:
+    import pyodbc
+
+    conn_str = (
+        "DRIVER={ODBC Driver 17 for SQL Server};"
+        f"SERVER={host},{port};"
+        f"DATABASE={db};"
+        f"UID={user};"
+        f"PWD={password}"
+    )
+    return pyodbc.connect(conn_str)
+
+
+def table_exists(conn: Any, db: str, table: str) -> bool:
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT 1 FROM INFORMATION_SCHEMA.TABLES "
+        "WHERE TABLE_CATALOG = ? AND TABLE_NAME = ?",
+        db,
+        table,
+    )
+    return cursor.fetchone() is not None
+
+
+def cdc_enabled(conn: Any, db: str, table: str) -> bool:
+    cursor = conn.cursor()
+    cursor.execute(f"USE [{db}]")
+    cursor.execute(
+        "SELECT is_tracked_by_cdc FROM sys.tables WHERE name = ?",
+        table,
+    )
+    row = cursor.fetchone()
+    return bool(row and row[0])
+
+
+def get_columns(conn: Any, db: str, table: str) -> list[dict[str, Any]]:
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, "
+        "NUMERIC_PRECISION, NUMERIC_SCALE, IS_NULLABLE "
+        "FROM INFORMATION_SCHEMA.COLUMNS "
+        "WHERE TABLE_CATALOG = ? AND TABLE_NAME = ? "
+        "ORDER BY ORDINAL_POSITION",
+        db,
+        table,
+    )
+    cols = cursor.description
+    return [dict(zip([c[0] for c in cols], row)) for row in cursor.fetchall()]

inthub/cli.py

@@ -0,0 +1,27 @@
+import os
+from importlib.metadata import version as pkg_version
+
+import typer
+
+from inthub.commands import connectors
+from inthub.commands.auth import login
+from inthub.commands.bulk import app as bulk_app
+
+app = typer.Typer(help="inthub CLI — manage your inthub.io resources from the terminal.",
+                  add_completion=False)
+
+app.command("login")(login)
+app.add_typer(connectors.app, name="connectors")
+app.add_typer(bulk_app, name="bulk")
+
+
+@app.command("version")
+def version() -> None:
+    """Show the CLI version."""
+    typer.echo(f"inthub {pkg_version('inthub-cli')}")
+
+
+def main() -> None:
+    for key in [k for k in os.environ if k.endswith("_COMPLETE")]:
+        del os.environ[key]
+    app()

inthub/client.py

@@ -0,0 +1,85 @@
+import json as json_mod
+import os
+import sys
+from typing import Any
+
+import httpx
+from rich.console import Console
+from rich.syntax import Syntax
+
+DEFAULT_BASE_URL = "https://console.inthub.io/api"
+
+_err = Console(stderr=True)
+
+
+def get_base_url() -> str:
+    return os.environ.get("INTHUB_BASE_URL", DEFAULT_BASE_URL).rstrip("/")
+
+
+class InthubClient:
+    def __init__(self, token: str, debug: bool = False) -> None:
+        self._token = token
+        self._base = get_base_url()
+        self._debug = debug
+
+    def _headers(self) -> dict[str, str]:
+        return {"Authorization": f"Bearer {self._token}"}
+
+    def _handle(self, response: httpx.Response) -> httpx.Response:
+        if self._debug:
+            self._print_debug(response)
+        if response.is_error:
+            try:
+                msg = response.json().get("error", response.text)
+            except Exception:
+                msg = response.text
+            _err.print(f"[bold red]Error {response.status_code}:[/] {msg}")
+            sys.exit(1)
+        return response
+
+    def _print_debug(self, response: httpx.Response) -> None:
+        req = response.request
+
+        _err.rule("[bold cyan]REQUEST[/]")
+        _err.print(f"[bold]{req.method}[/] [cyan]{req.url}[/]")
+        _err.print()
+        for name, value in req.headers.items():
+            _err.print(f"  [dim]{name}:[/] {value}")
+        if req.content:
+            _err.print()
+            try:
+                pretty = json_mod.dumps(json_mod.loads(req.content), indent=2)
+                _err.print(Syntax(pretty, "json", theme="monokai"))
+            except Exception:
+                _err.print(req.content.decode(errors="replace"))
+
+        _err.print()
+        status_color = "green" if response.status_code < 400 else "red"
+        _err.rule("[bold cyan]RESPONSE[/]")
+        _err.print(f"[bold {status_color}]{response.status_code}[/]")
+        _err.print()
+        for name, value in response.headers.items():
+            _err.print(f"  [dim]{name}:[/] {value}")
+        if response.text:
+            _err.print()
+            try:
+                pretty = json_mod.dumps(json_mod.loads(response.text), indent=2)
+                _err.print(Syntax(pretty, "json", theme="monokai"))
+            except Exception:
+                _err.print(response.text)
+        _err.print()
+
+    def get(self, path: str, **kwargs: Any) -> httpx.Response:
+        return self._handle(
+            httpx.get(f"{self._base}{path}", headers=self._headers(), **kwargs)
+        )
+
+    def post(self, path: str, json: dict[str, Any] | None = None, **kwargs: Any) -> httpx.Response:
+        return self._handle(
+            httpx.post(f"{self._base}{path}", json=json, headers=self._headers(), **kwargs)
+        )
+
+    def delete(self, path: str, **kwargs: Any) -> httpx.Response:
+        return self._handle(
+            httpx.delete(f"{self._base}{path}", headers=self._headers(), **kwargs)
+        )

inthub/commands/auth.py

@@ -0,0 +1,57 @@
+
+import httpx
+import typer
+from rich.console import Console
+
+from inthub import config
+from inthub.client import get_base_url
+
+console = Console()
+err = Console(stderr=True)
+
+
+def login() -> None:
+    """Authenticate with inthub and store credentials locally."""
+    base = get_base_url()
+
+    username = typer.prompt("Username")
+    password = typer.prompt("Password", hide_input=True)
+
+    r1 = httpx.post(f"{base}/auth/login", json={"username": username, "password": password})
+    if r1.is_error:
+        err.print(f"[bold red]Login failed ({r1.status_code}):[/] {r1.text}")
+        raise typer.Exit(1)
+
+    body1 = r1.json()
+    temp_token: str = body1["tempToken"]
+    companies: list[dict[str, str]] = body1["companies"]
+
+    if not companies:
+        err.print("[bold red]No companies found for this user.[/]")
+        raise typer.Exit(1)
+
+    console.print("Select company:")
+    for i, c in enumerate(companies, 1):
+        console.print(f"  [{i}] {c['displayName']} ({c['slug']})")
+
+    choice = typer.prompt("Company", default="1")
+    try:
+        selected = companies[int(choice) - 1]
+    except (ValueError, IndexError):
+        err.print("[bold red]Invalid selection.[/]")
+        raise typer.Exit(1)
+
+    r2 = httpx.post(
+        f"{base}/auth/select-company",
+        json={"companyId": selected["id"]},
+        headers={"Authorization": f"Bearer {temp_token}"},
+    )
+    if r2.is_error:
+        err.print(f"[bold red]Company selection failed ({r2.status_code}):[/] {r2.text}")
+        raise typer.Exit(1)
+
+    token: str = r2.json()["token"]
+    config.save(token=token, company_id=selected["id"], company_slug=selected["slug"])
+    console.print(
+        f"[green]Logged in as[/] [bold]{username}[/] @ [bold]{selected['displayName']}[/]"
+    )

inthub/commands/bulk/__init__.py

@@ -0,0 +1,11 @@
+import typer
+
+from inthub.commands.bulk import manage, progress
+from inthub.commands.bulk.src_sqlserver_sink_snowflake import app as _src_sf_app
+
+app = typer.Typer(help="Bulk connector provisioning commands.", add_completion=False)
+app.command("progress")(progress.progress)
+app.command("list")(manage.list_bulks)
+app.command("delete")(manage.delete_bulk)
+app.command("cleanup")(manage.cleanup)
+app.add_typer(_src_sf_app, name="src-sqlserver-sink-snowflake")

inthub/commands/bulk/manage.py

@@ -0,0 +1,98 @@
+import json
+from collections import Counter
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from inthub.bulk.csv_state import BULK_DIR, _progress_path, _state_path, load_rows
+
+console = Console()
+err = Console(stderr=True)
+
+
+def _state_files() -> list[tuple[str, dict[str, str]]]:
+    if not BULK_DIR.exists():
+        return []
+    results = []
+    for f in sorted(BULK_DIR.glob("state_*.json")):
+        ulid = f.stem.removeprefix("state_")
+        try:
+            state: dict[str, str] = json.loads(f.read_text())
+        except (json.JSONDecodeError, OSError):
+            continue
+        results.append((ulid, state))
+    return results
+
+
+def _delete_bulk(ulid: str) -> bool:
+    csv_file = _progress_path(ulid)
+    state_file = _state_path(ulid)
+    found = False
+    for path in (csv_file, state_file):
+        if path.exists():
+            path.unlink()
+            found = True
+    return found
+
+
+def list_bulks() -> None:
+    """List all bulk runs."""
+    entries = _state_files()
+    if not entries:
+        console.print("No bulk runs found.")
+        return
+
+    table = Table(show_header=True, header_style="bold", show_lines=True, expand=True)
+    table.add_column("ID", overflow="fold")
+    table.add_column("GCP PROJECT", overflow="fold")
+    table.add_column("SOURCE PLUGIN", overflow="fold")
+    table.add_column("SINK PLUGIN", overflow="fold")
+    table.add_column("TAGS", overflow="fold")
+    table.add_column("DONE", justify="right")
+    table.add_column("ERROR", justify="right")
+    table.add_column("PENDING", justify="right")
+
+    for ulid, state in entries:
+        counts: Counter[str] = Counter()
+        csv_file = _progress_path(ulid)
+        if csv_file.exists():
+            try:
+                counts.update(r.get("status", "") for r in load_rows(csv_file))
+            except Exception:
+                pass
+
+        table.add_row(
+            ulid,
+            state.get("gcp_project", ""),
+            state.get("sqlserver_source_plugin", ""),
+            state.get("snowflake_sink_plugin", ""),
+            state.get("tags", ""),
+            f"[green]{counts['done']}[/]" if counts["done"] else "0",
+            f"[red]{counts['error']}[/]" if counts["error"] else "0",
+            str(counts["pending"]),
+        )
+
+    console.print(table)
+
+
+def delete_bulk(ulid: str = typer.Argument(..., help="ULID of the bulk run to delete.")) -> None:
+    """Delete a bulk run (CSV + state)."""
+    if _delete_bulk(ulid):
+        console.print(f"Deleted bulk run [bold]{ulid}[/].")
+    else:
+        err.print(f"[bold red]Not found:[/] no bulk run with ID '{ulid}'.")
+        raise typer.Exit(1)
+
+
+def cleanup() -> None:
+    """Delete all bulk runs."""
+    entries = _state_files()
+    if not entries:
+        console.print("Nothing to clean up.")
+        return
+
+    for ulid, _ in entries:
+        _delete_bulk(ulid)
+
+    console.print(f"Deleted {len(entries)} bulk run(s).")

inthub/commands/bulk/progress.py

@@ -0,0 +1,46 @@
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from inthub.bulk.csv_state import _progress_path, load_rows
+
+console = Console()
+
+_STATUS_STYLE = {
+    "pending": "dim",
+    "working": "yellow",
+    "error": "red",
+    "done": "green",
+}
+
+
+def progress(ulid: str = typer.Argument(..., help="ULID of the bulk run.")) -> None:
+    """Show the current status of a bulk provisioning run."""
+    path = _progress_path(ulid)
+    if not path.exists():
+        console.print(f"[bold red]File not found:[/] {path}")
+        raise typer.Exit(1)
+
+    rows = load_rows(path)
+    if not rows:
+        console.print("No rows found.")
+        return
+
+    table = Table(show_header=True, header_style="bold", show_lines=True, expand=True)
+    table.add_column("CONNECTOR", overflow="fold")
+    table.add_column("TABLES", overflow="fold")
+    table.add_column("STATUS", no_wrap=True)
+    table.add_column("REASON", overflow="fold")
+
+    for row in rows:
+        status = row.get("status", "")
+        style = _STATUS_STYLE.get(status, "")
+        status_cell = f"[{style}]{status}[/]" if style else status
+        table.add_row(
+            row.get("connector_name", ""),
+            row.get("tables", ""),
+            status_cell,
+            row.get("reason", ""),
+        )
+
+    console.print(table)