icegraph-client 0.14.4__py3-none-any.whl

icegraph_client/__init__.py

@@ -0,0 +1,6 @@
+from icegraph_client.clients.icegraph_client import IceGraphClient
+from icegraph_client.clients.snapshots_client import SnapshotInfo
+from icegraph_client.clients.graph_client import GraphResult, Issues
+from icegraph_client.utils.json_utils import jsonify
+
+__all__ = ["IceGraphClient", "SnapshotInfo", "GraphResult", "Issues", "jsonify"]

icegraph_client/cli.py

@@ -0,0 +1,81 @@
+import argparse
+import inspect
+import os
+import sys
+
+import requests
+
+from icegraph_client.clients.icegraph_client import IceGraphClient
+from icegraph_client.utils.json_utils import jsonify
+
+BASE_URL_ENV_VAR = "ICEGRAPH_BASE_URL"
+
+
+def _tables(client: IceGraphClient, _: argparse.Namespace):
+    print("Fetching tables...", file=sys.stderr)
+    return client.list_tables()
+
+
+def _snapshots(client: IceGraphClient, args: argparse.Namespace):
+    print(f"Fetching snapshots for '{args.table}'...", file=sys.stderr)
+    return client.get_snapshot_map(args.table)
+
+
+def _graph(client: IceGraphClient, args: argparse.Namespace):
+    print(f"Building graph for '{args.table}'...", file=sys.stderr)
+    return client.get_graph(args.table, args.start_snapshot_id, args.end_snapshot_id)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="icegraph", description="CLI for the IceGraph server API")
+    parser.add_argument("--base-url", default=None, help=f"IceGraph server URL. Falls back to the {BASE_URL_ENV_VAR} environment variable")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    tables_parser = subparsers.add_parser("tables", help="List all available tables")
+    tables_parser.set_defaults(handler=_tables)
+
+    snapshots_parser = subparsers.add_parser("snapshots", help="Get the snapshot map for a table")
+    snapshots_parser.add_argument("table", help="Full table name")
+    snapshots_parser.set_defaults(handler=_snapshots)
+
+    graph_parser = subparsers.add_parser("graph", help="Build the metadata graph for a table")
+    graph_parser.add_argument("table", help="Full table name")
+    graph_parser.add_argument("-s", "--start-snapshot-id", default=None, help="Start snapshot id")
+    graph_parser.add_argument("-e", "--end-snapshot-id", default=None, help="End snapshot id")
+    graph_parser.set_defaults(handler=_graph)
+
+    return parser
+
+
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+
+    base_url = args.base_url or os.environ.get(BASE_URL_ENV_VAR)
+    if not base_url:
+        print(
+            inspect.cleandoc(f"""
+                No IceGraph server address was given.
+
+                Set it with one of the following:
+                  As an argument: --base-url http://<icegraph-server-host>
+                  As an environment variable (bash/zsh): export {BASE_URL_ENV_VAR}=http://<icegraph-server-host>
+                  As an environment variable (cmd): set {BASE_URL_ENV_VAR}=http://<icegraph-server-host>
+            """),
+            file=sys.stderr,
+        )
+        sys.exit(2)
+
+    client = IceGraphClient(base_url)
+
+    try:
+        result = args.handler(client, args)
+    except requests.RequestException as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    print(jsonify(result))
+
+
+if __name__ == "__main__":
+    main()

icegraph_client/clients/graph_client.py

@@ -0,0 +1,72 @@
+import time
+from dataclasses import dataclass
+
+import arrow
+import requests
+
+from icegraph_client.utils.http_utils import raise_for_status
+
+
+@dataclass
+class Issues:
+    errors: dict
+    warnings: dict
+
+
+@dataclass
+class GraphResult:
+    nodes: list
+    metadata: dict
+    issues: Issues
+
+
+class GraphClient:
+    def __init__(self, base_url: str, **requests_kwargs):
+        self.base_url = base_url
+        self.requests_kwargs = requests_kwargs
+
+    def get_graph(self, table: str, start_snapshot_id: str = None, end_snapshot_id: str = None, poll_interval_seconds: float = 1.0) -> GraphResult:
+        data = self._get_graph(table, start_snapshot_id, end_snapshot_id, poll_interval_seconds)
+        nodes = self._extract_node_details(data["nodes"])
+        metadata = data["metadata"]
+        issues = Issues(errors=data["errors"], warnings=data["warnings"])
+
+        return GraphResult(nodes=nodes, metadata=metadata, issues=issues)
+
+    def _extract_node_details(self, nodes: list) -> list:
+        details_list = []
+        for node in nodes:
+            details = node.get("details", {})
+            for key, value in details.items():
+                if value and "timestamp" in key.lower():
+                    details[key] = arrow.get(value).to("local")
+            details_list.append(details)
+
+        return details_list
+
+    def _get_graph(self, table: str, start_snapshot_id: str, end_snapshot_id: str, poll_interval_seconds: float) -> dict:
+        job_id = self._submit_graph_job(table, start_snapshot_id, end_snapshot_id)
+
+        while True:
+            result = self._get_graph_job(job_id)
+            if result.get("status") != "processing":
+                return result
+            time.sleep(poll_interval_seconds)
+
+    def _submit_graph_job(self, table: str, start_snapshot_id=None, end_snapshot_id=None) -> str:
+        response = requests.post(
+            f"{self.base_url}/api/v1/graph-data",
+            data={
+                "table_name": table,
+                "start_snapshot_id": start_snapshot_id,
+                "end_snapshot_id": end_snapshot_id,
+            },
+            **self.requests_kwargs,
+        )
+        raise_for_status(response)
+        return response.json()["key"]
+
+    def _get_graph_job(self, job_id: str) -> dict:
+        response = requests.get(f"{self.base_url}/api/v1/graph-data/{job_id}", **self.requests_kwargs)
+        raise_for_status(response)
+        return response.json()

icegraph_client/clients/icegraph_client.py

@@ -0,0 +1,42 @@
+import sys
+from urllib.parse import urlparse
+
+from icegraph_client.clients.tables_client import TablesClient
+from icegraph_client.clients.snapshots_client import SnapshotsClient
+from icegraph_client.clients.graph_client import GraphClient
+
+
+class IceGraphClient:
+    def __init__(self, base_url: str, **requests_kwargs):
+        if "://" not in base_url:
+            base_url = f"https://{base_url}"
+
+        parsed_base_url = urlparse(base_url)
+        self._base_url = f"{parsed_base_url.scheme}://{parsed_base_url.netloc}"
+
+        self._tables_client = TablesClient(self._base_url, **requests_kwargs)
+        self._snapshots_client = SnapshotsClient(self._base_url, **requests_kwargs)
+        self._graph_client = GraphClient(self._base_url, **requests_kwargs)
+
+    def list_tables(self):
+        return self._tables_client.list_tables()
+
+    def get_snapshot_map(self, table: str):
+        return self._snapshots_client.get_snapshot_map(table)
+
+    def get_graph(self, table: str, start_snapshot_id: str = None, end_snapshot_id: str = None, poll_interval_seconds: float = 1.0):
+        if start_snapshot_id is None and end_snapshot_id is None:
+            snapshots = self.get_snapshot_map(table)
+
+            if snapshots:
+                latest_snapshot = max(snapshots, key=lambda snapshot: snapshot.timestamp)
+                print(
+                    f"No snapshot range given, selected only the latest snapshot {latest_snapshot.snapshot_id} at {latest_snapshot.timestamp.humanize()}",
+                    file=sys.stderr,
+                )
+                start_snapshot_id = latest_snapshot.snapshot_id
+
+            else:
+                print(f"Table {table} has no data, both start and end snapshot remain unset", file=sys.stderr)
+
+        return self._graph_client.get_graph(table, start_snapshot_id, end_snapshot_id, poll_interval_seconds)

icegraph_client/clients/snapshots_client.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+import requests
+import arrow
+
+from icegraph_client.utils.http_utils import raise_for_status
+
+
+@dataclass
+class SnapshotInfo:
+    timestamp: arrow.Arrow
+    snapshot_id: str
+    operation: str
+
+
+class SnapshotsClient:
+    def __init__(self, base_url: str, **requests_kwargs):
+        self.base_url = base_url
+        self.requests_kwargs = requests_kwargs
+
+    def get_snapshot_map(self, table: str) -> list[SnapshotInfo]:
+        response = requests.get(f"{self.base_url}/api/v1/snapshot-map/{table}", **self.requests_kwargs)
+        raise_for_status(response)
+        data = response.json()
+
+        return [
+            SnapshotInfo(timestamp=arrow.get(timestamp).to("local"), snapshot_id=snapshot["snapshot_id"], operation=snapshot["operation"])
+            for timestamp, snapshot in data.items()
+        ]

icegraph_client/clients/tables_client.py

@@ -0,0 +1,15 @@
+import requests
+
+from icegraph_client.utils.http_utils import raise_for_status
+
+
+class TablesClient:
+    def __init__(self, base_url: str, **requests_kwargs):
+        self.base_url = base_url
+        self.requests_kwargs = requests_kwargs
+
+    def list_tables(self) -> list[str]:
+        response = requests.get(f"{self.base_url}/api/v1/tables", **self.requests_kwargs)
+        raise_for_status(response)
+
+        return response.json()["tables"]

icegraph_client/utils/http_utils.py

@@ -0,0 +1,13 @@
+import requests
+
+
+def raise_for_status(response: requests.Response) -> None:
+    try:
+        response.raise_for_status()
+    except requests.HTTPError as e:
+        try:
+            message = response.json().get("error", response.text)
+        except ValueError:
+            message = response.text
+
+        raise requests.HTTPError(f"{response.status_code} {response.reason} for {response.url}: {message}", response=response) from e

icegraph_client/utils/json_utils.py

@@ -0,0 +1,22 @@
+import json
+from dataclasses import asdict, is_dataclass
+
+import arrow
+
+
+def _default(obj):
+    if isinstance(obj, arrow.Arrow):
+        return obj.isoformat()
+    return str(obj)
+
+
+def _serializable(obj):
+    if is_dataclass(obj) and not isinstance(obj, type):
+        return asdict(obj)
+    if isinstance(obj, list):
+        return [_serializable(item) for item in obj]
+    return obj
+
+
+def jsonify(obj, **json_dumps_kwargs) -> str:
+    return json.dumps(_serializable(obj), default=_default, **json_dumps_kwargs)

icegraph_client-0.14.4.dist-info/METADATA

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: icegraph-client
+Version: 0.14.4
+Summary: Python client and CLI for the IceGraph server API
+Requires-Python: >=3.9
+Requires-Dist: requests>=2.25
+Requires-Dist: arrow>=1.0

icegraph_client-0.14.4.dist-info/RECORD

@@ -0,0 +1,13 @@
+icegraph_client/__init__.py,sha256=jp9woEe38ArpTL6doi5dybvMK6lszDRL2bnlKoSGifo,337
+icegraph_client/cli.py,sha256=Ey0e4p9lgmTd86iT0jkaIM8-sDkThb8FqdvjMtwWAH4,2866
+icegraph_client/clients/graph_client.py,sha256=sGkmNEtTfP2JTnPEqKozomMLrYbzRE3ooJB018LYCl0,2439
+icegraph_client/clients/icegraph_client.py,sha256=l0k_VvlNyNztMrRarbkNOOACBsjJhPaRPX1ID5qIi28,1857
+icegraph_client/clients/snapshots_client.py,sha256=usa_8Ft_LxYDXwDNNLGIZDN-SL7MoifhlFYaBgIYGqk,846
+icegraph_client/clients/tables_client.py,sha256=MfLZdOs-DFZkD_njDRJ-CR26NSoqVjkUiLEh_o7C7Nw,447
+icegraph_client/utils/http_utils.py,sha256=08WQk5nIKU8q0aKBLfrC7peLEBO3DjUa-vO4wtc2IeA,435
+icegraph_client/utils/json_utils.py,sha256=FPWtAeP8g7Gb6fDCNAs0vDAI6_ZMKO5i6Vk7s-16JUo,516
+icegraph_client-0.14.4.dist-info/METADATA,sha256=IJJJ641JzixJ4SbCBhyzIqCRXx7ZPle2CSFrS2Gzrhg,198
+icegraph_client-0.14.4.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+icegraph_client-0.14.4.dist-info/entry_points.txt,sha256=kNyT7PJlvfpA7kxYUN58lg7swfOWc21JmB4XkPoXoL4,54
+icegraph_client-0.14.4.dist-info/top_level.txt,sha256=0lADl2qhSGgqJE-E6A5wDpCycX_9tgOAamhPyqr2Ik8,16
+icegraph_client-0.14.4.dist-info/RECORD,,

icegraph_client-0.14.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

icegraph_client-0.14.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+icegraph = icegraph_client.cli:main

icegraph_client-0.14.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+icegraph_client