acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.2.0.1rc1__py3-none-any.whl

datahub/cli/ingest_cli.py

@@ -24,7 +24,6 @@ from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 from datahub.utilities.ingest_utils import deploy_source_vars
-from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
@@ -114,6 +113,7 @@ def ingest() -> None:
         "no_progress",
     ]
 )
+@upgrade.check_upgrade
 def run(
     config: str,
     dry_run: bool,
@@ -178,14 +178,7 @@ def run(
         no_progress=no_progress,
         raw_config=raw_pipeline_config,
     )
-    with PerfTimer() as timer:
-        ret = run_pipeline_to_completion(pipeline)
-
-    # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
-    if ret == 0:
-        upgrade.check_upgrade_post(
-            main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
-        )
+    ret = run_pipeline_to_completion(pipeline)
 
     if ret:
         sys.exit(ret)
@@ -193,8 +186,6 @@ def run(
 
 
 @ingest.command()
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 @click.option(
     "-n",
     "--name",
@@ -252,6 +243,7 @@ def run(
     required=False,
     default=None,
 )
+@upgrade.check_upgrade
 def deploy(
     name: Optional[str],
     config: str,
@@ -386,7 +378,6 @@ def mcps(path: str) -> None:
     "--source", type=str, default=None, help="Filter by ingestion source name."
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
     """
     List ingestion source runs with their details, optionally filtered by URN or source.
@@ -515,7 +506,6 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
     help="If enabled, will list ingestion runs which have been soft deleted",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> None:
     """List recent ingestion runs to datahub"""
 
@@ -565,7 +555,6 @@ def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> N
 )
 @click.option("-a", "--show-aspect", required=False, is_flag=True)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def show(
     run_id: str, start: int, count: int, include_soft_deletes: bool, show_aspect: bool
 ) -> None:
@@ -615,7 +604,6 @@ def show(
     help="Path to directory where rollback reports will be saved to",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def rollback(
     run_id: str, force: bool, dry_run: bool, safe: bool, report_dir: str
 ) -> None:

datahub/cli/migrate.py

@@ -37,6 +37,7 @@ from datahub.metadata.schema_classes import (
     SystemMetadataClass,
 )
 from datahub.telemetry import telemetry
+from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
 log = logging.getLogger(__name__)
@@ -119,6 +120,7 @@ def _get_type_from_urn(urn: str) -> str:
     help="When enabled, will not delete (hard/soft) the previous entities.",
 )
 @telemetry.with_telemetry()
+@upgrade.check_upgrade
 def dataplatform2instance(
     instance: str,
     platform: str,

datahub/cli/put_cli.py

@@ -14,7 +14,6 @@ from datahub.metadata.schema_classes import (
     PlatformTypeClass,
     SystemMetadataClass,
 )
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 from datahub.utilities.urns.data_platform_urn import DataPlatformUrn
 from datahub.utilities.urns.urn import guess_entity_type
@@ -45,7 +44,6 @@ def put() -> None:
     help="Run ID into which we should log the aspect.",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> None:
     """Update a single aspect of an entity"""
 
@@ -75,8 +73,6 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
 
 @put.command()
 @click.pass_context
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 @click.option(
     "--name",
     type=str,
@@ -98,6 +94,7 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
 @click.option(
     "--run-id", type=str, help="Run ID into which we should log the platform."
 )
+@upgrade.check_upgrade
 def platform(
     ctx: click.Context, name: str, display_name: Optional[str], logo: str, run_id: str
 ) -> None:

datahub/cli/quickstart_versioning.py

@@ -6,17 +6,40 @@ import re
 from typing import Dict, Optional
 
 import click
+import packaging
 import requests
 import yaml
 from packaging.version import parse
 from pydantic import BaseModel
 
+from datahub._version import nice_version_name
+
 logger = logging.getLogger(__name__)
 
 LOCAL_QUICKSTART_MAPPING_FILE = os.environ.get("FORCE_LOCAL_QUICKSTART_MAPPING", "")
 DEFAULT_LOCAL_CONFIG_PATH = "~/.datahub/quickstart/quickstart_version_mapping.yaml"
 DEFAULT_REMOTE_CONFIG_PATH = "https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/quickstart_version_mapping.yaml"
 
+MINIMUM_SUPPORTED_VERSION = "v1.1.0"
+
+
+def get_minimum_supported_version_message(version: str) -> str:
+    MINIMUM_SUPPORTED_VERSION_MESSAGE = f"""
+    DataHub CLI Version Compatibility Issue
+
+    You're trying to install DataHub server version {version} which is not supported by this CLI version.
+
+    This CLI (version {nice_version_name()}) only supports installing DataHub server versions {MINIMUM_SUPPORTED_VERSION} and above.
+
+    To install older server versions:
+    1. Uninstall current CLI: pip uninstall acryl-datahub
+    2. Install older CLI: pip install acryl-datahub==1.1
+    3. Run quickstart with your desired version: datahub docker quickstart --version <version>
+
+    For more information: https://docs.datahub.com/docs/quickstart#install-datahub-server
+    """
+    return MINIMUM_SUPPORTED_VERSION_MESSAGE
+
 
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
@@ -126,15 +149,23 @@ class QuickstartVersionMappingConfig(BaseModel):
                 mysql_tag=str(mysql_tag),
             ),
         )
+
+        if not is_minimum_supported_version(requested_version):
+            click.secho(
+                get_minimum_supported_version_message(version=requested_version),
+                fg="red",
+            )
+            raise click.ClickException("Minimum supported version not met")
+
         # new CLI version is downloading the composefile corresponding to the requested version
-        # if the version is older than v0.10.1, it doesn't contain the setup job labels and the
-        # the checks will fail, so in those cases we pick the composefile from v0.10.1 which contains
-        # the setup job labels
+        # if the version is older than <MINIMUM_SUPPORTED_VERSION>, it doesn't contain the
+        # docker compose based resolved compose file. In those cases, we pick up the composefile from
+        # MINIMUM_SUPPORTED_VERSION which contains the compose file.
         if _is_it_a_version(result.composefile_git_ref):
-            if parse("v0.10.1") > parse(result.composefile_git_ref):
-                # The merge commit where the labels were added
-                # https://github.com/datahub-project/datahub/pull/7473
-                result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e"
+            if parse("v1.2.0") > parse(result.composefile_git_ref):
+                # The merge commit where profiles based resolved compose file was added.
+                # https://github.com/datahub-project/datahub/pull/13566
+                result.composefile_git_ref = "21726bc3341490f4182b904626c793091ac95edd"
 
         return result
 
@@ -148,3 +179,15 @@ def save_quickstart_config(
     with open(path, "w") as f:
         yaml.dump(config.dict(), f)
     logger.info(f"Saved quickstart config to {path}.")
+
+
+def is_minimum_supported_version(version: str) -> bool:
+    if not _is_it_a_version(version):
+        return True
+
+    requested_version = packaging.version.parse(version)
+    minimum_supported_version = packaging.version.parse(MINIMUM_SUPPORTED_VERSION)
+    if requested_version < minimum_supported_version:
+        return False
+
+    return True

datahub/cli/specific/assertions_cli.py

@@ -17,7 +17,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.integrations.assertion.registry import ASSERTION_PLATFORMS
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -34,7 +33,6 @@ def assertions() -> None:
 @assertions.command()
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: str) -> None:
     """Upsert (create or update) a set of assertions in DataHub."""
 
@@ -71,8 +69,6 @@ def upsert(file: str) -> None:
     default=[],
     help="Platform-specific extra key-value inputs in form key=value",
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def compile(
     file: str, platform: str, output_to: Optional[str], extras: List[str]
 ) -> None:

datahub/cli/specific/datacontract_cli.py

@@ -7,7 +7,6 @@ from click_default_group import DefaultGroup
 from datahub.api.entities.datacontract.datacontract import DataContract
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -22,7 +21,6 @@ def datacontract() -> None:
 @datacontract.command()
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: str) -> None:
     """Upsert (create or update) a Data Contract in DataHub."""
 
@@ -60,7 +58,6 @@ def upsert(file: str) -> None:
 )
 @click.option("--hard/--soft", required=False, is_flag=True, default=False)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
     """Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
 

datahub/cli/specific/dataproduct_cli.py

@@ -23,7 +23,6 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass
 from datahub.specific.dataproduct import DataProductPatchBuilder
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
@@ -130,7 +129,6 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
 )
 @click.option("--external-url", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def update(file: Path, validate_assets: bool, external_url: str) -> None:
     """Create or Update a Data Product in DataHub. Use upsert if you want to apply partial updates."""
 
@@ -146,7 +144,6 @@ def update(file: Path, validate_assets: bool, external_url: str) -> None:
 )
 @click.option("--external-url", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
     """Upsert attributes to a Data Product in DataHub."""
 
@@ -159,7 +156,6 @@ def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @click.option("--update", required=False, is_flag=True, default=False)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def diff(file: Path, update: bool) -> None:
     """Diff a Data Product file with its twin in DataHub"""
 
@@ -206,7 +202,6 @@ def diff(file: Path, update: bool) -> None:
 )
 @click.option("--hard/--soft", required=False, is_flag=True, default=False)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def delete(urn: str, file: Path, hard: bool) -> None:
     """Delete a Data Product in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
 
@@ -242,7 +237,6 @@ def delete(urn: str, file: Path, hard: bool) -> None:
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get a Data Product from DataHub"""
 
@@ -279,7 +273,6 @@ def get(urn: str, to_file: str) -> None:
     help="A markdown file that contains documentation for this data product",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def set_description(urn: str, description: str, md_file: Path) -> None:
     """Set description for a Data Product in DataHub"""
 
@@ -330,7 +323,6 @@ def set_description(urn: str, description: str, md_file: Path) -> None:
     default=OwnershipTypeClass.TECHNICAL_OWNER,
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def add_owner(urn: str, owner: str, owner_type: str) -> None:
     """Add owner for a Data Product in DataHub"""
 
@@ -353,7 +345,6 @@ def add_owner(urn: str, owner: str, owner_type: str) -> None:
 @click.option("--urn", required=True, type=str)
 @click.argument("owner_urn", required=True, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def remove_owner(urn: str, owner_urn: str) -> None:
     """Remove owner for a Data Product in DataHub"""
 
@@ -375,7 +366,6 @@ def remove_owner(urn: str, owner_urn: str) -> None:
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
     """Add asset for a Data Product in DataHub"""
 
@@ -402,7 +392,6 @@ def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
     "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def remove_asset(urn: str, asset: str, validate_assets: bool) -> None:
     """Remove asset for a Data Product in DataHub"""
 

datahub/cli/specific/dataset_cli.py

@@ -14,7 +14,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.metadata.com.linkedin.pegasus2avro.common import Siblings
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -33,8 +32,6 @@ def dataset() -> None:
 @click.option(
     "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
 )
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, dry_run: bool) -> None:
     """Upsert attributes to a Dataset in DataHub."""
     # Call the sync command with to_datahub=True to perform the upsert operation
@@ -48,7 +45,6 @@ def upsert(file: Path, dry_run: bool) -> None:
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get a Dataset from DataHub"""
 
@@ -77,7 +73,7 @@ def get(urn: str, to_file: str) -> None:
     help="URN of secondary sibling(s)",
     multiple=True,
 )
-@telemetry.with_telemetry()
+@upgrade.check_upgrade
 def add_sibling(urn: str, sibling_urns: Tuple[str]) -> None:
     all_urns = set()
     all_urns.add(urn)
@@ -117,8 +113,6 @@ def _get_existing_siblings(graph: DataHubGraph, urn: str) -> Set[str]:
 @click.option("--lintCheck", required=False, is_flag=True)
 @click.option("--lintFix", required=False, is_flag=True)
 @click.argument("file", type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def file(lintcheck: bool, lintfix: bool, file: str) -> None:
     """Operate on a Dataset file"""
 
@@ -175,7 +169,6 @@ def file(lintcheck: bool, lintfix: bool, file: str) -> None:
     "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def sync(file: str, to_datahub: bool, dry_run: bool) -> None:
     """Sync a Dataset file to/from DataHub"""
 

datahub/cli/specific/forms_cli.py

@@ -8,7 +8,6 @@ from click_default_group import DefaultGroup
 from datahub.api.entities.forms.forms import Forms
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -24,8 +23,6 @@ def forms() -> None:
     name="upsert",
 )
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
-@upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path) -> None:
     """Upsert forms in DataHub."""
 
@@ -38,7 +35,6 @@ def upsert(file: Path) -> None:
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get form from DataHub"""
     with get_default_graph(ClientMode.CLI) as graph:

datahub/cli/specific/group_cli.py

@@ -11,7 +11,6 @@ from datahub.api.entities.corpgroup.corpgroup import (
 from datahub.cli.specific.file_loader import load_file
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -35,7 +34,6 @@ def group() -> None:
     help="When set, writes to the editable section of the metadata graph, overwriting writes from the UI",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, override_editable: bool) -> None:
     """Create or Update a Group with embedded Users"""
 

datahub/cli/specific/structuredproperties_cli.py

@@ -12,7 +12,6 @@ from datahub.api.entities.structuredproperties.structuredproperties import (
 )
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
@@ -30,7 +29,6 @@ def properties() -> None:
 )
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path) -> None:
     """Upsert structured properties in DataHub."""
 
@@ -44,7 +42,6 @@ def upsert(file: Path) -> None:
 @click.option("--urn", required=True, type=str)
 @click.option("--to-file", required=False, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def get(urn: str, to_file: str) -> None:
     """Get structured properties from DataHub"""
     urn = Urn.make_structured_property_urn(urn)
@@ -71,7 +68,7 @@ def get(urn: str, to_file: str) -> None:
 )
 @click.option("--details/--no-details", is_flag=True, default=True)
 @click.option("--to-file", required=False, type=str)
-@telemetry.with_telemetry()
+@upgrade.check_upgrade
 def list(details: bool, to_file: str) -> None:
     """List structured properties in DataHub"""
 

datahub/cli/specific/user_cli.py

@@ -9,7 +9,6 @@ from datahub.api.entities.corpuser.corpuser import CorpUser, CorpUserGenerationC
 from datahub.cli.specific.file_loader import load_file
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -33,7 +32,6 @@ def user() -> None:
     help="Use this flag to overwrite the information that is set via the UI",
 )
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def upsert(file: Path, override_editable: bool) -> None:
     """Create or Update a User in DataHub"""
 

datahub/cli/state_cli.py

@@ -6,7 +6,6 @@ from click_default_group import DefaultGroup
 
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 
 logger = logging.getLogger(__name__)
@@ -22,7 +21,6 @@ def state() -> None:
 @click.option("--pipeline-name", required=True, type=str)
 @click.option("--platform", required=True, type=str)
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def inspect(pipeline_name: str, platform: str) -> None:
     """
     Get the latest stateful ingestion state for a given pipeline.

datahub/cli/timeline_cli.py

@@ -10,7 +10,6 @@ from requests import Response
 from datahub.emitter.mce_builder import dataset_urn_to_key, schema_field_urn_to_key
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode
-from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
 from datahub.utilities.urns.urn import Urn
 
@@ -130,7 +129,6 @@ def get_timeline(
 @click.option("--raw", type=bool, is_flag=True, help="Show the raw diff")
 @click.pass_context
 @upgrade.check_upgrade
-@telemetry.with_telemetry()
 def timeline(
     ctx: Any,
     urn: str,

datahub/configuration/pydantic_migration_helpers.py

@@ -1,12 +1,13 @@
 import pydantic.version
 from packaging.version import Version
 
-PYDANTIC_VERSION_2: bool
-if Version(pydantic.version.VERSION) >= Version("2.0"):
-    PYDANTIC_VERSION_2 = True
-else:
-    PYDANTIC_VERSION_2 = False
+_pydantic_version = Version(pydantic.version.VERSION)
+
+PYDANTIC_VERSION_2 = _pydantic_version >= Version("2.0")
 
+# The pydantic.Discriminator type was added in v2.5.0.
+# https://docs.pydantic.dev/latest/changelog/#v250-2023-11-13
+PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR = _pydantic_version >= Version("2.5.0")
 
 # This can be used to silence deprecation warnings while we migrate.
 if PYDANTIC_VERSION_2:
@@ -50,6 +51,7 @@ class v1_ConfigModel(v1_BaseModel):
 
 __all__ = [
     "PYDANTIC_VERSION_2",
+    "PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR",
     "PydanticDeprecatedSince20",
     "GenericModel",
     "v1_ConfigModel",