acryl-datahub 1.1.0.5rc7__py3-none-any.whl → 1.1.0.5rc9__py3-none-any.whl

datahub/cli/ingest_cli.py

@@ -22,9 +22,7 @@ from datahub.ingestion.graph.config import ClientMode
 from datahub.ingestion.run.connection import ConnectionManager
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 from datahub.utilities.ingest_utils import deploy_source_vars
-from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
@@ -178,14 +176,7 @@ def run(
         no_progress=no_progress,
         raw_config=raw_pipeline_config,
     )
-    with PerfTimer() as timer:
-        ret = run_pipeline_to_completion(pipeline)
-
-    # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
-    if ret == 0:
-        upgrade.check_upgrade_post(
-            main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
-        )
+    ret = run_pipeline_to_completion(pipeline)
 
     if ret:
         sys.exit(ret)
@@ -193,8 +184,6 @@ def run(
 
 
 @ingest.command()
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 @click.option(
     "-n",
     "--name",
@@ -385,8 +374,6 @@ def mcps(path: str) -> None:
 @click.option(
     "--source", type=str, default=None, help="Filter by ingestion source name."
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
     """
     List ingestion source runs with their details, optionally filtered by URN or source.
@@ -514,8 +501,6 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
     default=False,
     help="If enabled, will list ingestion runs which have been soft deleted",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> None:
     """List recent ingestion runs to datahub"""
 
@@ -564,8 +549,6 @@ def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> N
     help="If enabled, will include aspects that have been soft deleted",
 )
 @click.option("-a", "--show-aspect", required=False, is_flag=True)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def show(
     run_id: str, start: int, count: int, include_soft_deletes: bool, show_aspect: bool
 ) -> None:
@@ -614,8 +597,6 @@ def show(
     default="./rollback-reports",
     help="Path to directory where rollback reports will be saved to",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def rollback(
     run_id: str, force: bool, dry_run: bool, safe: bool, report_dir: str
 ) -> None:

datahub/cli/put_cli.py

@@ -14,8 +14,6 @@ from datahub.metadata.schema_classes import (
     PlatformTypeClass,
     SystemMetadataClass,
 )
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 from datahub.utilities.urns.data_platform_urn import DataPlatformUrn
 from datahub.utilities.urns.urn import guess_entity_type
 
@@ -44,8 +42,6 @@ def put() -> None:
     required=False,
     help="Run ID into which we should log the aspect.",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> None:
     """Update a single aspect of an entity"""
 
@@ -75,8 +71,6 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
 
 @put.command()
 @click.pass_context
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 @click.option(
     "--name",
     type=str,

datahub/cli/quickstart_versioning.py

@@ -6,17 +6,40 @@ import re
 from typing import Dict, Optional
 
 import click
+import packaging
 import requests
 import yaml
 from packaging.version import parse
 from pydantic import BaseModel
 
+from datahub._version import nice_version_name
+
 logger = logging.getLogger(__name__)
 
 LOCAL_QUICKSTART_MAPPING_FILE = os.environ.get("FORCE_LOCAL_QUICKSTART_MAPPING", "")
 DEFAULT_LOCAL_CONFIG_PATH = "~/.datahub/quickstart/quickstart_version_mapping.yaml"
 DEFAULT_REMOTE_CONFIG_PATH = "https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/quickstart_version_mapping.yaml"
 
+MINIMUM_SUPPORTED_VERSION = "v1.1.0"
+
+
+def get_minimum_supported_version_message(version: str) -> str:
+    MINIMUM_SUPPORTED_VERSION_MESSAGE = f"""
+    DataHub CLI Version Compatibility Issue
+
+    You're trying to install DataHub server version {version} which is not supported by this CLI version.
+
+    This CLI (version {nice_version_name()}) only supports installing DataHub server versions {MINIMUM_SUPPORTED_VERSION} and above.
+
+    To install older server versions:
+    1. Uninstall current CLI: pip uninstall acryl-datahub
+    2. Install older CLI: pip install acryl-datahub==1.1
+    3. Run quickstart with your desired version: datahub docker quickstart --version <version>
+
+    For more information: https://docs.datahub.com/docs/quickstart#install-datahub-server
+    """
+    return MINIMUM_SUPPORTED_VERSION_MESSAGE
+
 
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
@@ -126,15 +149,25 @@ class QuickstartVersionMappingConfig(BaseModel):
                 mysql_tag=str(mysql_tag),
             ),
         )
+
+        if not is_minimum_supported_version(requested_version):
+            click.secho(
+                get_minimum_supported_version_message(version=requested_version),
+                fg="red",
+            )
+            raise click.ClickException("Minimum supported version not met")
+
         # new CLI version is downloading the composefile corresponding to the requested version
-        # if the version is older than v0.10.1, it doesn't contain the setup job labels and the
-        # the checks will fail, so in those cases we pick the composefile from v0.10.1 which contains
-        # the setup job labels
+        # if the version is older than <MINIMUM_SUPPORTED_VERSION>, it doesn't contain the
+        # docker compose based resolved compose file. In those cases, we pick up the composefile from
+        # MINIMUM_SUPPORTED_VERSION which contains the compose file.
         if _is_it_a_version(result.composefile_git_ref):
-            if parse("v0.10.1") > parse(result.composefile_git_ref):
+            if (
+                parse("v1.2.0") > parse(result.composefile_git_ref)
+            ):  # TODO: Once we decide what the new version that supports profile based compose.
                 # The merge commit where the labels were added
                 # https://github.com/datahub-project/datahub/pull/7473
-                result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e"
+                result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e"  # TODO update after compose file is merged
 
         return result
 
@@ -148,3 +181,15 @@ def save_quickstart_config(
     with open(path, "w") as f:
         yaml.dump(config.dict(), f)
     logger.info(f"Saved quickstart config to {path}.")
+
+
+def is_minimum_supported_version(version: str) -> bool:
+    if not _is_it_a_version(version):
+        return True
+
+    requested_version = packaging.version.parse(version)
+    minimum_supported_version = packaging.version.parse(MINIMUM_SUPPORTED_VERSION)
+    if requested_version < minimum_supported_version:
+        return False
+
+    return True

datahub/cli/specific/assertions_cli.py

@@ -17,8 +17,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.integrations.assertion.registry import ASSERTION_PLATFORMS
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -33,8 +31,6 @@ def assertions() -> None:
 
 @assertions.command()
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: str) -> None:
     """Upsert (create or update) a set of assertions in DataHub."""
 
@@ -71,8 +67,6 @@ def upsert(file: str) -> None:
     default=[],
     help="Platform-specific extra key-value inputs in form key=value",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def compile(
     file: str, platform: str, output_to: Optional[str], extras: List[str]
 ) -> None:

datahub/cli/specific/datacontract_cli.py

@@ -7,8 +7,6 @@ from click_default_group import DefaultGroup
 from datahub.api.entities.datacontract.datacontract import DataContract
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -21,8 +19,6 @@ def datacontract() -> None:
 
 @datacontract.command()
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: str) -> None:
     """Upsert (create or update) a Data Contract in DataHub."""
 
@@ -59,8 +55,6 @@ def upsert(file: str) -> None:
     help="The file containing the data contract definition",
 )
 @click.option("--hard/--soft", required=False, is_flag=True, default=False)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
     """Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
 

datahub/cli/specific/dataproduct_cli.py

@@ -23,8 +23,6 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass
 from datahub.specific.dataproduct import DataProductPatchBuilder
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
 logger = logging.getLogger(__name__)
@@ -129,8 +127,6 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
 @click.option("--external-url", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def update(file: Path, validate_assets: bool, external_url: str) -> None:
     """Create or Update a Data Product in DataHub. Use upsert if you want to apply partial updates."""
 
@@ -145,8 +141,6 @@ def update(file: Path, validate_assets: bool, external_url: str) -> None:
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
 @click.option("--external-url", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
     """Upsert attributes to a Data Product in DataHub."""
 
@@ -158,8 +152,6 @@ def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
 )
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @click.option("--update", required=False, is_flag=True, default=False)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def diff(file: Path, update: bool) -> None:
     """Diff a Data Product file with its twin in DataHub"""
 
@@ -205,8 +197,6 @@ def diff(file: Path, update: bool) -> None:
     help="The file containing the data product definition",
 )
 @click.option("--hard/--soft", required=False, is_flag=True, default=False)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def delete(urn: str, file: Path, hard: bool) -> None:
     """Delete a Data Product in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
 
@@ -241,8 +231,6 @@ def delete(urn: str, file: Path, hard: bool) -> None:
 )
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get a Data Product from DataHub"""
 
@@ -278,8 +266,6 @@ def get(urn: str, to_file: str) -> None:
     type=click.Path(exists=True),
     help="A markdown file that contains documentation for this data product",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def set_description(urn: str, description: str, md_file: Path) -> None:
     """Set description for a Data Product in DataHub"""
 
@@ -329,8 +315,6 @@ def set_description(urn: str, description: str, md_file: Path) -> None:
     ),
     default=OwnershipTypeClass.TECHNICAL_OWNER,
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def add_owner(urn: str, owner: str, owner_type: str) -> None:
     """Add owner for a Data Product in DataHub"""
 
@@ -352,8 +336,6 @@ def add_owner(urn: str, owner: str, owner_type: str) -> None:
 @dataproduct.command(name="remove_owner", help="Remove an owner from a Data Product")
 @click.option("--urn", required=True, type=str)
 @click.argument("owner_urn", required=True, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def remove_owner(urn: str, owner_urn: str) -> None:
     """Remove owner for a Data Product in DataHub"""
 
@@ -374,8 +356,6 @@ def remove_owner(urn: str, owner_urn: str) -> None:
 @click.option(
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
     """Add asset for a Data Product in DataHub"""
 
@@ -401,8 +381,6 @@ def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
 @click.option(
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def remove_asset(urn: str, asset: str, validate_assets: bool) -> None:
     """Remove asset for a Data Product in DataHub"""
 

datahub/cli/specific/dataset_cli.py

@@ -14,8 +14,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.metadata.com.linkedin.pegasus2avro.common import Siblings
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -33,8 +31,6 @@ def dataset() -> None:
 @click.option(
     "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, dry_run: bool) -> None:
     """Upsert attributes to a Dataset in DataHub."""
     # Call the sync command with to_datahub=True to perform the upsert operation
@@ -47,8 +43,6 @@ def upsert(file: Path, dry_run: bool) -> None:
 )
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get a Dataset from DataHub"""
 
@@ -77,7 +71,6 @@ def get(urn: str, to_file: str) -> None:
     help="URN of secondary sibling(s)",
     multiple=True,
 )
-@telemetry.with_telemetry()
 def add_sibling(urn: str, sibling_urns: Tuple[str]) -> None:
     all_urns = set()
     all_urns.add(urn)
@@ -117,8 +110,6 @@ def _get_existing_siblings(graph: DataHubGraph, urn: str) -> Set[str]:
 @click.option("--lintCheck", required=False, is_flag=True)
 @click.option("--lintFix", required=False, is_flag=True)
 @click.argument("file", type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def file(lintcheck: bool, lintfix: bool, file: str) -> None:
     """Operate on a Dataset file"""
 
@@ -174,8 +165,6 @@ def file(lintcheck: bool, lintfix: bool, file: str) -> None:
 @click.option(
     "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def sync(file: str, to_datahub: bool, dry_run: bool) -> None:
     """Sync a Dataset file to/from DataHub"""
 

datahub/cli/specific/forms_cli.py

@@ -8,8 +8,6 @@ from click_default_group import DefaultGroup
 from datahub.api.entities.forms.forms import Forms
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -24,8 +22,6 @@ def forms() -> None:
     name="upsert",
 )
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path) -> None:
     """Upsert forms in DataHub."""
 
@@ -37,8 +33,6 @@ def upsert(file: Path) -> None:
 )
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get form from DataHub"""
     with get_default_graph(ClientMode.CLI) as graph:

datahub/cli/specific/group_cli.py

@@ -11,8 +11,6 @@ from datahub.api.entities.corpgroup.corpgroup import (
 from datahub.cli.specific.file_loader import load_file
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -34,8 +32,6 @@ def group() -> None:
     default=False,
     help="When set, writes to the editable section of the metadata graph, overwriting writes from the UI",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, override_editable: bool) -> None:
     """Create or Update a Group with embedded Users"""
 

datahub/cli/specific/structuredproperties_cli.py

@@ -12,8 +12,6 @@ from datahub.api.entities.structuredproperties.structuredproperties import (
 )
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
 logger = logging.getLogger(__name__)
@@ -29,8 +27,6 @@ def properties() -> None:
     name="upsert",
 )
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path) -> None:
     """Upsert structured properties in DataHub."""
 
@@ -43,8 +39,6 @@ def upsert(file: Path) -> None:
 )
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get structured properties from DataHub"""
     urn = Urn.make_structured_property_urn(urn)
@@ -71,7 +65,6 @@ def get(urn: str, to_file: str) -> None:
 )
 @click.option("--details/--no-details", is_flag=True, default=True)
 @click.option("--to-file", required=False, type=str)
-@telemetry.with_telemetry()
 def list(details: bool, to_file: str) -> None:
     """List structured properties in DataHub"""
 

datahub/cli/specific/user_cli.py

@@ -9,8 +9,6 @@ from datahub.api.entities.corpuser.corpuser import CorpUser, CorpUserGenerationC
 from datahub.cli.specific.file_loader import load_file
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -32,8 +30,6 @@ def user() -> None:
     is_flag=True,
     help="Use this flag to overwrite the information that is set via the UI",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, override_editable: bool) -> None:
     """Create or Update a User in DataHub"""
 

datahub/cli/state_cli.py

@@ -6,8 +6,6 @@ from click_default_group import DefaultGroup
 
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
 
@@ -21,8 +19,6 @@ def state() -> None:
 @state.command()
 @click.option("--pipeline-name", required=True, type=str)
 @click.option("--platform", required=True, type=str)
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def inspect(pipeline_name: str, platform: str) -> None:
     """
     Get the latest stateful ingestion state for a given pipeline.

datahub/cli/timeline_cli.py

@@ -10,8 +10,6 @@ from requests import Response
 from datahub.emitter.mce_builder import dataset_urn_to_key, schema_field_urn_to_key
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
-from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
 logger = logging.getLogger(__name__)
@@ -129,8 +127,6 @@ def get_timeline(
 )
 @click.option("--raw", type=bool, is_flag=True, help="Show the raw diff")
 @click.pass_context
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def timeline(
     ctx: Any,
     urn: str,

datahub/entrypoints.py

@@ -10,6 +10,7 @@ import click
 import datahub._version as datahub_version
 from datahub.cli.check_cli import check
 from datahub.cli.cli_utils import (
+    enable_auto_decorators,
     fixup_gms_url,
     generate_access_token,
     make_shim_command,
@@ -38,7 +39,6 @@ from datahub.cli.timeline_cli import timeline
 from datahub.configuration.common import should_show_stack_trace
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.utilities._custom_package_loader import model_version_name
 from datahub.utilities.logging_manager import configure_logging
 from datahub.utilities.server_config_util import get_gms_config
@@ -111,7 +111,6 @@ def datahub(
     default=False,
     help="If passed will show server config. Assumes datahub init has happened.",
 )
-@telemetry.with_telemetry()
 def version(include_server: bool = False) -> None:
     """Print version number and exit."""
 
@@ -131,7 +130,6 @@ def version(include_server: bool = False) -> None:
     default=False,
     help="If passed then uses password to initialise token.",
 )
-@telemetry.with_telemetry()
 def init(use_password: bool = False) -> None:
     """Configure which datahub instance to connect to"""
 
@@ -218,6 +216,9 @@ except ImportError as e:
         make_shim_command("actions", "run `pip install acryl-datahub-actions`")
     )
 
+# Adding telemetry and upgrade decorators to all commands
+enable_auto_decorators(datahub)
+
 
 def main(**kwargs):
     # We use threads in a variety of places within our CLI. The multiprocessing