bigeye-cli 0.8.1__tar.gz → 0.8.3__tar.gz

{bigeye_cli-0.8.1 → bigeye_cli-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigeye-cli
-Version: 0.8.1
+Version: 0.8.3
 Summary: Bigeye CLI offers developer tools for maintaining your developer workspace.
 Home-page: https://docs.bigeye.com/docs
 License: Proprietary
@@ -12,13 +12,13 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: PyGithub (>=1.59.1,<2.0.0)
-Requires-Dist: bigeye-sdk (>=0.11.0,<0.12.0)
+Requires-Dist: bigeye-sdk (>=0.11.2,<0.12.0)
 Requires-Dist: click (>=8.1.8,<9.0.0)
 Requires-Dist: rich (>=14.1.0,<15.0.0)
 Description-Content-Type: text/markdown
 
 # **Bigeye CLI**
-Version: 0.8.1
+Version: 0.8.3
 ## Installation
 
 **Setting up your local/build environment**  
@@ -197,7 +197,7 @@ Configuration Commands for Bigeye CLI
 **Usage**:
 
 ```console
-$ bigeye configure [OPTIONS] COMMAND [ARGS]...
+$ bigeye configure [OPTIONS] [COMMAND] [ARGS]...
 ```
 
 **Options**:
@@ -1343,6 +1343,8 @@ $ bigeye catalog custom-repo import [OPTIONS]
 * `-w, --workspace TEXT`: Name of the workspace configuration.  [env var: BIGEYE_WORKSPACE; default: DEFAULT]
 * `-f, --file TEXT`: Path to the custom repository .xlsx workbook.  [required]
 * `--cleanup-stale`: Delete nodes/edges present in Bigeye but missing from the sheet. Off by default.
+* `--skip-unresolved-edges / --no-skip-unresolved-edges`: Import the rest instead of aborting when some edge endpoints don&#x27;t resolve to a Bigeye node (default: on). Skipped endpoints are written to the unresolved log. Pass --no-skip-unresolved-edges to abort on any unresolved endpoint.  [default: skip-unresolved-edges]
+* `--unresolved-log TEXT`: Path for the unresolved-endpoints log. Defaults to custom_repo_unresolved.log in the current working directory.  [default: custom_repo_unresolved.log]
 * `--help`: Show this message and exit.
 
 ### `bigeye catalog custom-repo export`
@@ -2015,6 +2017,8 @@ $ bigeye lineage export-graphs [OPTIONS]
 * `--direction [upstream|downstream|both]`: Direction to traverse: upstream, downstream, or both.  [default: both]
 * `--content [all|nodes|edges|attributes]`: What each per-graph report contains: all, nodes, edges, or attributes.  [default: all]
 * `--include-etl / --no-include-etl`: Include or exclude ETL operator nodes. Defaults to including them.  [default: include-etl]
+* `--depth INTEGER`: Per-window traversal depth. When set, the graph is built in bounded depth-hop windows (re-rooting at the lineage frontier) up to the workspace&#x27;s lineage.max_depth, avoiding the oversized single response that fails on very large graphs. Applies to every FQN; a per-row &#x27;depth&#x27; column in the FQNs file overrides it for that row. Omit to request the full graph in one call (the previous behavior).
+* `--concurrency INTEGER`: Max concurrent window requests when --depth is used. Lower it to reduce client memory and server load on wide graphs.  [default: 8]
 * `-o, --output TEXT`: Output .zip path. Defaults to lineage_graphs.zip in the current directory.
 * `--help`: Show this message and exit.
 
@@ -2042,6 +2046,8 @@ $ bigeye lineage export-end-to-end [OPTIONS]
 * `--direction [upstream|downstream|both]`: Direction to trace: upstream, downstream, or both.  [default: both]
 * `--granularity [schema|table|column]`: Hop level: schema, table, or column. Defaults to each root&#x27;s own level.
 * `--max-paths INTEGER`: Per-root cap on enumerated paths. Truncation is logged, never silent.  [default: 10000]
+* `--depth INTEGER`: Per-window traversal depth. When set, each root&#x27;s graph is fetched in bounded depth-hop windows, re-rooting at the lineage frontier until the lineage is exhausted end-to-end. This escapes the backend&#x27;s per-request depth ceiling (lineage.max_depth) that truncates a single full fetch, and keeps each request small so very large graphs don&#x27;t fail. Omit to request the graph in one call (the previous behavior, capped at the backend depth limit).
+* `--concurrency INTEGER`: Max concurrent window requests when --depth is used. Lower it to reduce client memory and server load on wide graphs. Ignored without --depth.  [default: 8]
 * `-o, --output TEXT`: Output .csv path. Defaults to end_to_end_&lt;root&gt;.csv in the current directory.
 * `--help`: Show this message and exit.
 

{bigeye_cli-0.8.1 → bigeye_cli-0.8.3}/bigeye_cli/CLI_DOCS.md

@@ -1,5 +1,5 @@
 # **Bigeye CLI**
-Version: 0.8.1
+Version: 0.8.3
 ## Installation
 
 **Setting up your local/build environment**  
@@ -178,7 +178,7 @@ Configuration Commands for Bigeye CLI
 **Usage**:
 
 ```console
-$ bigeye configure [OPTIONS] COMMAND [ARGS]...
+$ bigeye configure [OPTIONS] [COMMAND] [ARGS]...
 ```
 
 **Options**:
@@ -1324,6 +1324,8 @@ $ bigeye catalog custom-repo import [OPTIONS]
 * `-w, --workspace TEXT`: Name of the workspace configuration.  [env var: BIGEYE_WORKSPACE; default: DEFAULT]
 * `-f, --file TEXT`: Path to the custom repository .xlsx workbook.  [required]
 * `--cleanup-stale`: Delete nodes/edges present in Bigeye but missing from the sheet. Off by default.
+* `--skip-unresolved-edges / --no-skip-unresolved-edges`: Import the rest instead of aborting when some edge endpoints don't resolve to a Bigeye node (default: on). Skipped endpoints are written to the unresolved log. Pass --no-skip-unresolved-edges to abort on any unresolved endpoint.  [default: skip-unresolved-edges]
+* `--unresolved-log TEXT`: Path for the unresolved-endpoints log. Defaults to custom_repo_unresolved.log in the current working directory.  [default: custom_repo_unresolved.log]
 * `--help`: Show this message and exit.
 
 ### `bigeye catalog custom-repo export`
@@ -1996,6 +1998,8 @@ $ bigeye lineage export-graphs [OPTIONS]
 * `--direction [upstream|downstream|both]`: Direction to traverse: upstream, downstream, or both.  [default: both]
 * `--content [all|nodes|edges|attributes]`: What each per-graph report contains: all, nodes, edges, or attributes.  [default: all]
 * `--include-etl / --no-include-etl`: Include or exclude ETL operator nodes. Defaults to including them.  [default: include-etl]
+* `--depth INTEGER`: Per-window traversal depth. When set, the graph is built in bounded depth-hop windows (re-rooting at the lineage frontier) up to the workspace's lineage.max_depth, avoiding the oversized single response that fails on very large graphs. Applies to every FQN; a per-row 'depth' column in the FQNs file overrides it for that row. Omit to request the full graph in one call (the previous behavior).
+* `--concurrency INTEGER`: Max concurrent window requests when --depth is used. Lower it to reduce client memory and server load on wide graphs.  [default: 8]
 * `-o, --output TEXT`: Output .zip path. Defaults to lineage_graphs.zip in the current directory.
 * `--help`: Show this message and exit.
 
@@ -2023,6 +2027,8 @@ $ bigeye lineage export-end-to-end [OPTIONS]
 * `--direction [upstream|downstream|both]`: Direction to trace: upstream, downstream, or both.  [default: both]
 * `--granularity [schema|table|column]`: Hop level: schema, table, or column. Defaults to each root's own level.
 * `--max-paths INTEGER`: Per-root cap on enumerated paths. Truncation is logged, never silent.  [default: 10000]
+* `--depth INTEGER`: Per-window traversal depth. When set, each root's graph is fetched in bounded depth-hop windows, re-rooting at the lineage frontier until the lineage is exhausted end-to-end. This escapes the backend's per-request depth ceiling (lineage.max_depth) that truncates a single full fetch, and keeps each request small so very large graphs don't fail. Omit to request the graph in one call (the previous behavior, capped at the backend depth limit).
+* `--concurrency INTEGER`: Max concurrent window requests when --depth is used. Lower it to reduce client memory and server load on wide graphs. Ignored without --depth.  [default: 8]
 * `-o, --output TEXT`: Output .csv path. Defaults to end_to_end_<root>.csv in the current directory.
 * `--help`: Show this message and exit.
 

{bigeye_cli-0.8.1 → bigeye_cli-0.8.3}/bigeye_cli/catalog/custom_repo_commands.py

@@ -1,4 +1,5 @@
 import os
+from datetime import datetime
 from typing import Dict, List, Optional, Tuple
 
 import typer
@@ -6,6 +7,7 @@ import typer
 from bigeye_cli import global_options
 from bigeye_cli.functions import cli_client_factory
 from bigeye_sdk.controller.custom_repository_controller import CustomRepositoryController
+from bigeye_sdk.generated.com.bigeye.models.generated import DataNodeType
 from bigeye_sdk.log import get_logger
 from bigeye_sdk.model.custom_repository_excel import (
     SHEET_EDGES,
@@ -34,6 +36,38 @@ def _print_errors(header: str, errors: List[WorkbookValidationError]) -> None:
         typer.echo(f"  {error.format()}", err=True)
 
 
+def _write_unresolved_log(
+    log_path: str, repo_name: str, unresolved: List[WorkbookValidationError]
+) -> bool:
+    """Write a log file naming the edge endpoints that could not be resolved to a
+    unique Bigeye node — either no match (the column/object does not exist) or more
+    than one match (ambiguous). Best-effort: returns False and warns if the file
+    cannot be written (the same endpoints are also printed to stderr).
+
+    Each endpoint is a catalog path 'Repo.schema.table.column' (or a warehouse FQN);
+    its final segment is the column/object in question.
+    """
+    lines = [
+        "Custom repository import — unresolved edge endpoints",
+        f"Repository: {repo_name}",
+        f"Generated:  {datetime.now().isoformat(timespec='seconds')}",
+        f"Count:      {len(unresolved)}",
+        "",
+        "These endpoints could not be resolved to a unique Bigeye node (no match, or",
+        "more than one), so the edges referencing them were skipped. Each line below",
+        "states the reason and the endpoint value.",
+        "",
+    ]
+    lines.extend(error.format() for error in unresolved)
+    try:
+        with open(log_path, "w") as f:
+            f.write("\n".join(lines) + "\n")
+        return True
+    except OSError as e:
+        typer.echo(f"Warning: could not write unresolved log to '{log_path}': {e}", err=True)
+        return False
+
+
 def _resolve_paths(
     wb: CustomRepoWorkbook, controller: CustomRepositoryController
 ) -> Tuple[Dict[str, int], List[WorkbookValidationError]]:
@@ -46,7 +80,11 @@ def _resolve_paths(
     controller.warm_warehouse_cache(endpoints)
     for endpoint_type, value in endpoints:
         if endpoint_type == EdgeEndpointType.CUSTOM.value:
-            matches = controller.resolve_catalog_path(parse_catalog_path(value))
+            # CUSTOM endpoints are catalog paths into custom-repository nodes; the
+            # type filter keeps the search off same-named warehouse/MetaCenter nodes.
+            matches = controller.resolve_catalog_path(
+                parse_catalog_path(value), search_type=DataNodeType.DATA_NODE_TYPE_CUSTOM
+            )
         else:
             matches = controller.resolve_warehouse_fqn(endpoint_type, value)
         if not matches:
@@ -108,6 +146,19 @@ def import_repo(
         "--cleanup-stale",
         help="Delete nodes/edges present in Bigeye but missing from the sheet. Off by default.",
     ),
+    skip_unresolved_edges: bool = typer.Option(
+        True,
+        "--skip-unresolved-edges/--no-skip-unresolved-edges",
+        help="Import the rest instead of aborting when some edge endpoints don't resolve "
+        "to a Bigeye node (default: on). Skipped endpoints are written to the unresolved "
+        "log. Pass --no-skip-unresolved-edges to abort on any unresolved endpoint.",
+    ),
+    unresolved_log: str = typer.Option(
+        "custom_repo_unresolved.log",
+        "--unresolved-log",
+        help="Path for the unresolved-endpoints log. Defaults to "
+        "custom_repo_unresolved.log in the current working directory.",
+    ),
 ):
     """Import a custom repository from an Excel workbook."""
     client = cli_client_factory(bigeye_conf, config_file, workspace)
@@ -126,10 +177,28 @@ def import_repo(
 
     path_node_ids, unresolved = _resolve_paths(wb, controller)
     if unresolved:
+        log_written = _write_unresolved_log(unresolved_log, wb.repo_name, unresolved)
         _print_errors("Could not resolve all catalog path references", unresolved)
-        raise typer.Exit(code=1)
+        log_hint = f" (see '{unresolved_log}')" if log_written else ""
+        if not skip_unresolved_edges:
+            typer.echo(
+                f"{len(unresolved)} unresolved endpoint(s){log_hint}. "
+                f"Re-run with --skip-unresolved-edges to import the rest.",
+                err=True,
+            )
+            raise typer.Exit(code=1)
+        typer.echo(
+            f"Skipping {len(unresolved)} unresolvable endpoint(s){log_hint} "
+            f"and importing the rest.",
+            err=True,
+        )
 
-    config = build_config(wb, path_node_ids=path_node_ids, cleanup_stale=cleanup_stale)
+    config = build_config(
+        wb,
+        path_node_ids=path_node_ids,
+        cleanup_stale=cleanup_stale,
+        drop_unresolved_edges=skip_unresolved_edges,
+    )
     result = controller.sync_from_config(config, wait_for_completion=True)
 
     log.info(

{bigeye_cli-0.8.1 → bigeye_cli-0.8.3}/bigeye_cli/lineage/lineage_commands.py

@@ -266,6 +266,21 @@ def export_graphs(
             "--include-etl/--no-include-etl",
             help="Include or exclude ETL operator nodes. Defaults to including them.",
         ),
+        depth: Optional[int] = typer.Option(
+            None,
+            "--depth",
+            help="Per-window traversal depth. When set, the graph is built in bounded depth-hop "
+                 "windows (re-rooting at the lineage frontier) up to the workspace's lineage.max_depth, "
+                 "avoiding the oversized single response that fails on very large graphs. Applies to "
+                 "every FQN; a per-row 'depth' column in the FQNs file overrides it for that row. "
+                 "Omit to request the full graph in one call (the previous behavior).",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            help="Max concurrent window requests when --depth is used. Lower it to reduce client "
+                 "memory and server load on wide graphs.",
+        ),
         output: Optional[str] = typer.Option(
             None,
             "--output",
@@ -279,19 +294,31 @@ def export_graphs(
     and writes one folder per FQN (nodes/edges/attributes) plus a top-level manifest of
     successes and failures.
     """
+    if depth is not None and depth < 1:
+        typer.echo("--depth must be a positive integer (1 or greater).", err=True)
+        raise typer.Exit(code=1)
+
     client = cli_client_factory(bigeye_conf, config_file, workspace)
     controller = LineageCustomReporting(client)
 
     try:
-        fqns = read_fqns_file(fqns_file)
+        specs = read_fqns_file(fqns_file)
     except OSError as e:
         typer.echo(f"Could not read FQNs file '{fqns_file}': {e}", err=True)
         raise typer.Exit(code=1)
 
-    if not fqns:
+    if not specs:
         typer.echo(f"No FQNs found in '{fqns_file}'.", err=True)
         raise typer.Exit(code=1)
 
+    fqns = [spec.fqn for spec in specs]
+    # Per-row depth wins; --depth is the default for rows without one. No depth anywhere -> full graph.
+    depth_by_fqn = {
+        spec.fqn: (spec.depth if spec.depth is not None else depth)
+        for spec in specs
+        if (spec.depth if spec.depth is not None else depth) is not None
+    }
+
     output_path = output or "lineage_graphs.zip"
     summary = controller.export_graphs(
         fqns=fqns,
@@ -299,6 +326,8 @@ def export_graphs(
         content=content,
         include_etl=include_etl,
         output_path=output_path,
+        depth_by_fqn=depth_by_fqn or None,
+        concurrency=concurrency,
     )
 
     log.info(
@@ -337,6 +366,21 @@ def export_end_to_end(
             "--max-paths",
             help="Per-root cap on enumerated paths. Truncation is logged, never silent.",
         ),
+        depth: Optional[int] = typer.Option(
+            None,
+            "--depth",
+            help="Per-window traversal depth. When set, each root's graph is fetched in bounded depth-hop "
+                 "windows, re-rooting at the lineage frontier until the lineage is exhausted end-to-end. "
+                 "This escapes the backend's per-request depth ceiling (lineage.max_depth) that truncates a "
+                 "single full fetch, and keeps each request small so very large graphs don't fail. Omit to "
+                 "request the graph in one call (the previous behavior, capped at the backend depth limit).",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            help="Max concurrent window requests when --depth is used. Lower it to reduce client "
+                 "memory and server load on wide graphs. Ignored without --depth.",
+        ),
         output: Optional[str] = typer.Option(
             None,
             "--output",
@@ -351,6 +395,10 @@ def export_end_to_end(
     collapsed to the chosen (or root-inferred) granularity — column/field hops roll
     up to their containing table/object (warehouse tables, ETL targets, BI objects).
     """
+    if depth is not None and depth < 1:
+        typer.echo("--depth must be a positive integer (1 or greater).", err=True)
+        raise typer.Exit(code=1)
+
     client = cli_client_factory(bigeye_conf, config_file, workspace)
     controller = LineageCustomReporting(client)
 
@@ -361,6 +409,8 @@ def export_end_to_end(
         granularity=granularity,
         output_path=output_path,
         max_paths=max_paths,
+        depth=depth,
+        concurrency=concurrency,
     )
 
     log.info(

{bigeye_cli-0.8.1 → bigeye_cli-0.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bigeye-cli"
-version = "0.8.1"
+version = "0.8.3"
 description = "Bigeye CLI offers developer tools for maintaining your developer workspace."
 license = "Proprietary"
 authors = ["Bigeye <support@bigeye.com>"]
@@ -19,7 +19,7 @@ bigeye = "bigeye_cli.__main__:app"
 [tool.poetry.dependencies]
 python = "^3.11.0"
 click = "^8.1.8"
-bigeye-sdk = "^0.11.0"
+bigeye-sdk = "^0.11.2"
 rich = "^14.1.0"
 PyGithub = '^1.59.1'