datatrail-cli 0.4.2__tar.gz → 0.4.4__tar.gz

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.1
 Name: datatrail-cli
-Version: 0.4.2
+Version: 0.4.4
 Summary: CHIME/FRB Datatrail CLI
 License: MIT
 Author: CHIME FRB Project Office
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.8.1,<4.0.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/clear.py

@@ -9,14 +9,18 @@ from rich.prompt import Confirm
 
 from dtcli.config import procure
 from dtcli.src.functions import clear_dataset_path, find_dataset_common_path
-from dtcli.utilities.utilities import validate_scope
+from dtcli.utilities.utilities import set_log_level, validate_scope
 
 logger = logging.getLogger("clear")
 
 console = Console()
+error_console = Console(stderr=True, style="bold red")
 
 
-@click.command(name="clear", help="Clear a dataset.")
+@click.command(
+    name="clear",
+    help="""Clear a dataset.""",
+)
 @click.argument("scope", type=click.STRING, required=True, nargs=1)
 @click.argument("dataset", type=click.STRING, required=True, nargs=1)
 @click.option(
@@ -26,7 +30,7 @@ console = Console()
         exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
     ),
     default=None,
-    help="Directory to clear data from.",
+    help="Root directory to use. Default: None, will use the value set in the config.",
 )
 @click.option(
     "--clear-parents",
@@ -35,8 +39,10 @@ console = Console()
 )
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
-@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation.")
+@click.option("--force", "-f", is_flag=True, help="Will not prompt for confirmation.")
+@click.pass_context
 def clear(
+    ctx: click.Context,
     scope: str,
     dataset: str,
     directory: str,
@@ -48,6 +54,7 @@ def clear(
     """Clear a dataset.
 
     Args:
+        ctx (click.Context): Click context.
         scope (str): Scope of dataset.
         dataset (str): Name of dataset.
         directory (str): Directory to clear data from.
@@ -56,13 +63,8 @@ def clear(
         quiet (bool): Minimal logging.
         force (bool): Automatically download files.
     """
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    # Set logging level.
+    set_log_level(logger, verbose, quiet)
     logger.debug("`clear` called with:")
     logger.debug(f"scope: {scope} [{type(scope)}]")
     logger.debug(f"dataset: {dataset} [{type(dataset)}]")
@@ -90,7 +92,10 @@ def clear(
         raise RuntimeError("Clear command not permitted at Chime or Outriggers!")
 
     if not validate_scope(scope):
-        raise ValueError("Scope does not exist.")
+        error_console.print("Scope does not exist!")
+        console.print("Valid scopes are:")
+        ctx.invoke(list)
+        return None
 
     # Find number of files in common directory and size.
     console.print(f"\nSearching for files for {dataset} {scope}...\n")

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/cli.py

@@ -6,6 +6,7 @@ from pkg_resources import get_distribution
 from rich import console, pretty
 
 from dtcli import clear, config, ls, ps, pull
+from dtcli.utilities import utilities
 
 pretty.install()
 terminal = console.Console()
@@ -15,7 +16,7 @@ terminal = console.Console()
 @click.group(cls=ClickAliasedGroup)
 def cli():
     """Datatrail Command Line Interface."""
-    pass
+    check_version()
 
 
 @cli.command(name="version", help="Show versions.")
@@ -41,5 +42,18 @@ cli.add_command(pull.pull)
 cli.add_command(clear.clear)
 cli.add_command(config.config)
 
+
+def check_version() -> None:
+    """Check if CLI is latest release."""
+    if not utilities.cli_is_latest_release():
+        current_version = get_distribution("datatrail-cli").version
+        latest_version = utilities.get_latest_released_version()
+        terminal.print(
+            f"A new release of datatrail-cli is available: {current_version} → {latest_version}",  # noqa: E501
+            style="bold yellow",
+        )
+        terminal.print()
+
+
 if __name__ == "__main__":
     cli()

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/config.py

@@ -17,7 +17,10 @@ CONFIG: Path = Path.home() / ".datatrail" / "config.yaml"
 
 @click.group(cls=ClickAliasedGroup)
 def config():
-    """Datatrail CLI Configuration."""
+    """Datatrail CLI Configuration.
+
+    For initialising and modifying the Datatrail CLI configuration file.
+    """
     pass
 
 
@@ -68,15 +71,26 @@ def list():
             print(exc)
 
 
-@config.command(name="init", help="Initialize configuration.")
+@config.command(
+    name="init",
+    help="""Initialise configuration.
+
+    This will create a configuration file in the home directory under `.datatrail`.
+    Datatrail MUST be initialised before is can be used.
+""",
+)
 @click.option(
-    "--site", "-s", type=click.STRING, help="Site to initialize.", required=True
+    "--site",
+    "-s",
+    type=click.STRING,
+    help="Site to initialise Datatrail CLI for.",
+    required=True,
 )
 def init(site: str):
-    """Initialize configuration.
+    """Initialise configuration.
 
     Args:
-        site (str): Site to initialize.
+        site (str): Site to initialise.
     """
     # Default configuration.
     defaults: Dict[str, Any] = {

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/ls.py

@@ -9,11 +9,12 @@ from rich.console import Console
 from rich.table import Table
 
 from dtcli.src import functions
-from dtcli.utilities.utilities import validate_scope
+from dtcli.utilities.utilities import set_log_level, validate_scope
 
 logger = logging.getLogger("ls")
 
 console = Console()
+error_console = Console(stderr=True, style="bold red")
 
 
 @click.command(help="List scopes & datasets")
@@ -32,21 +33,27 @@ console = Console()
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
 @click.option("--write", is_flag=True, help="Write the events to file.")
+@click.pass_context
 def list(
+    ctx: click.Context,
     scope: Optional[str] = None,
     datasets: Optional[str] = None,
     verbose: int = 0,
     quiet: bool = False,
     write: bool = False,
 ):
-    """List Datatrail Scopes & Datasets."""
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    """List Datatrail Scopes & Datasets.
+
+    Args:
+        ctx (click.Context): Click context.
+        scope (str): Scope of dataset.
+        datasets (str): Name of dataset.
+        verbose (int): Verbosity: v=INFO, vv=DEBUG.
+        quiet (bool): Only errors shown in logs.
+        write (bool): Write the events to file.
+    """
+    # Set logging level.
+    set_log_level(logger, verbose, quiet)
     logger.debug("`list` called with:")
     logger.debug(f"scope: {scope} [{type(scope)}]")
     logger.debug(f"datasets: {datasets} [{type(datasets)}]")
@@ -54,7 +61,10 @@ def list(
     logger.debug(f"quiet: {quiet} [{type(quiet)}]")
     if scope:
         if not validate_scope(scope):
-            raise ValueError("Scope does not exist.")
+            error_console.print("Scope does not exist!")
+            console.print("Valid scopes are:")
+            ctx.invoke(list)
+            return None
     results = functions.list(scope, datasets, verbose, quiet)
 
     # Display scopes.
@@ -88,7 +98,7 @@ def list(
 
     # Display datasets in parent dataset for scope.
     if "datasets" in results.keys():
-        results["datasets"] = sorted(results["datasets"], key=int, reverse=True)
+        results["datasets"] = sorted(results["datasets"], reverse=True)
         if write:
             with open(f"./dataset_list_for_{scope}_{datasets}.txt", "w") as file:
                 json.dump(results, file)
@@ -108,4 +118,4 @@ def list(
 
     # No contact with server.
     if "error" in results.keys():
-        console.print(results["error"], style="red bold")
+        error_console.print(results["error"])

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/ps.py

@@ -9,9 +9,10 @@ from requests.exceptions import SSLError
 from rich.console import Console
 from rich.table import Table
 
+from dtcli.ls import list
 from dtcli.src import functions
 from dtcli.utilities import cadcclient
-from dtcli.utilities.utilities import validate_scope
+from dtcli.utilities.utilities import set_log_level, validate_scope
 
 logger = logging.getLogger("ps")
 
@@ -25,7 +26,9 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-s", "--show-files", is_flag=True, help="Show file names.")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
-def ps(  # noqa: C901
+@click.pass_context
+def ps(
+    ctx: click.Context,
     scope: str,
     dataset: str,
     show_files: bool,
@@ -35,6 +38,7 @@ def ps(  # noqa: C901
     """Detailed status of a dataset.
 
     Args:
+        ctx (click.Context): Click context.
         scope (str): Scope of dataset.
         dataset (str): Name of dataset.
         show_files (bool): Show list of files.
@@ -45,13 +49,7 @@ def ps(  # noqa: C901
         None
     """
     # Set logging level.
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    set_log_level(logger, verbose, quiet)
     logger.debug("`ps` called with:")
     logger.debug(f"scope: {scope} [{type(scope)}]")
     logger.debug(f"dataset: {dataset} [{type(dataset)}]")
@@ -60,72 +58,77 @@ def ps(  # noqa: C901
     logger.debug(f"quiet: {quiet} [{type(quiet)}]")
 
     if not validate_scope(scope):
-        raise ValueError("Scope does not exist.")
+        error_console.print("Scope does not exist!")
+        console.print("Valid scopes are:")
+        ctx.invoke(list)
+        return None
     try:
         files, policies = functions.ps(scope, dataset, verbose, quiet)
     except Exception as e:
         logger.error(e)
         return None
 
+    if show_files and files:
+        # Files table
+        file_table = create_files_table(dataset, scope, files)
+
+        with console.pager():
+            logger.debug("Showing file table.")
+            console.print(file_table)
+        return None
+
     # Info table
+    if files:
+        info_table = create_info_table(dataset, scope, files)
+        logger.debug("Showing info table.")
+        console.print(info_table)
+
+    # Policy table
+    if policies:
+        policy_table = create_policy_table(dataset, scope, policies)
+        logger.debug("Showing policy table.")
+        console.print(policy_table)
+
+
+def create_info_table(dataset: str, scope: str, files: dict):
+    """Create info table."""
     logger.debug("Creating info table.")
     info_table = Table(
-        title=f"Datatrail: {dataset} {scope} at Minoc",
+        title=f"Datatrail: {dataset} {scope} at SEs",
         header_style="magenta",
         title_style="bold magenta",
     )
-    info_table.add_column("Storage Element", style="bold")
+    info_table.add_column("Storage Element (SE)", style="bold")
     info_table.add_column("Number of Files", style="green")
     info_table.add_column("Size of Files [GB]", style="green")
-    if files["file_replica_locations"].get("minoc"):
-        minoc_files = files["file_replica_locations"]["minoc"]
-        minoc_files = [f.replace("cadc:CHIMEFRB", "") for f in minoc_files]
-        # Make sure starts with a /
-        common_path = os.path.commonpath(
-            ["/" + f if not f.startswith("/") else f for f in minoc_files]
-        )
-        try:
-            size = cadcclient.size(common_path)
-        except SSLError as error:
-            logger.error(error)
-            error_console.print(
-                """
+    for se in files["file_replica_locations"]:
+        logger.debug(f"Creating row for: {se}")
+        se_files = files["file_replica_locations"][se]
+        if se == "minoc":
+            se_files = [f.replace("cadc:CHIMEFRB", "") for f in se_files]
+            # Make sure starts with a /
+            common_path = os.path.commonpath(
+                ["/" + f if not f.startswith("/") else f for f in se_files]
+            )
+            try:
+                size = cadcclient.size(common_path)
+            except SSLError as error:
+                logger.error(error)
+                error_console.print(
+                    """
 No valid CADC certificate found.
 Create one using 'cadc-get-cert -u <USERNAME>'.
 """
-            )
-            return None
-        info_table.add_row("minoc", f"{len(minoc_files)}", f"{size:.2f}")
-    else:
-        info_table.add_row("minoc", str(0), str(0))
+                )
+                return None
+            info_table.add_row(se, f"{len(se_files)}", f"{size:.2f}")
+        else:
+            info_table.add_row(se, f"{len(se_files)}", "Not available")
+    return info_table
 
-    # Files table
-    logger.debug("Creating files table.")
-    file_table = Table(
-        # header_style="magenta",
-        title_style="magenta",
-    )
-    file_table.add_column(
-        f"Datatrail: Files for {dataset} {scope}", style="bold magenta"
-    )
 
-    for se in files["file_replica_locations"]:
-        common_path = os.path.commonpath(files["file_replica_locations"][se])
-        names = [
-            Path(_).relative_to(common_path) for _ in files["file_replica_locations"][se]
-        ]
-        for idx, fn in enumerate(names):
-            if idx == 0:
-                file_table.add_row(f"Storage Element: [magenta]{se}")
-                file_table.add_row(f"Common Path: {common_path}/", style="bold green")
-                file_table.add_row(f"[green]- {fn}")
-                # file_table.add_row(se, common_path, fn)
-            else:
-                file_table.add_row(f"- {fn}", style="green")
-                # file_table.add_row("", "", fn)
-        file_table.add_section()
-
-    # Policy table
+def create_policy_table(dataset: str, scope: str, policies: dict):
+    """Create policy table."""
     logger.debug("Creating policy table.")
     policy_table = Table(
         title=f"Datatrail: Policies for {dataset} {scope}",
@@ -134,8 +137,11 @@ Create one using 'cadc-get-cert -u <USERNAME>'.
         show_footer=True,
         footer_style="bold red",
     )
+    belongs_to = (
+        policies["belongs_to"][0]["name"] if len(policies["belongs_to"]) > 0 else "None"
+    )
     policy_table.add_column("Policy", style="bold", footer="Belongs to")
-    policy_table.add_column("Storage Element", footer=policies["belongs_to"][0]["name"])
+    policy_table.add_column("Storage Element", footer=belongs_to)
     policy_table.add_column("Priority")
     policy_table.add_column("Default")
     policy_table.add_column(r"Delete After \[days]")
@@ -169,13 +175,33 @@ Create one using 'cadc-get-cert -u <USERNAME>'.
                 str(dp["delete_after_days"]),
             )
     policy_table.add_section()
+    return policy_table
 
-    if show_files:
-        with console.pager():
-            logger.debug("Showing file table.")
-            console.print(file_table)
-    else:
-        logger.debug("Showing info table.")
-        console.print(info_table)
-        logger.debug("Showing policy table.")
-        console.print(policy_table)
+
+def create_files_table(dataset: str, scope: str, files: dict):
+    """Create files table."""
+    logger.debug("Creating files table.")
+    file_table = Table(
+        # header_style="magenta",
+        title_style="magenta",
+    )
+    file_table.add_column(
+        f"Datatrail: Files for {dataset} {scope}", style="bold magenta"
+    )
+
+    for se in files["file_replica_locations"]:
+        common_path = os.path.commonpath(files["file_replica_locations"][se])
+        names = [
+            Path(_).relative_to(common_path) for _ in files["file_replica_locations"][se]
+        ]
+        for idx, fn in enumerate(names):
+            if idx == 0:
+                file_table.add_row(f"Storage Element: [magenta]{se}")
+                file_table.add_row(f"Common Path: {common_path}/", style="bold green")
+                file_table.add_row(f"[green]- {fn}")
+                # file_table.add_row(se, common_path, fn)
+            else:
+                file_table.add_row(f"- {fn}", style="green")
+                # file_table.add_row("", "", fn)
+        file_table.add_section()
+    return file_table

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/pull.py

@@ -11,7 +11,7 @@ from rich.prompt import Confirm
 from dtcli.config import procure
 from dtcli.src.functions import find_missing_dataset_files, get_files
 from dtcli.utilities.cadcclient import size
-from dtcli.utilities.utilities import validate_scope
+from dtcli.utilities.utilities import set_log_level, validate_scope
 
 logger = logging.getLogger("pull")
 
@@ -41,7 +41,9 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
 @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation.")
+@click.pass_context
 def pull(
+    ctx: click.Context,
     scope: str,
     dataset: str,
     directory: str,
@@ -53,6 +55,7 @@ def pull(
     """Download a dataset.
 
     Args:
+        ctx (click.Context): Click context.
         scope (str): Scope of dataset.
         dataset (str): Name of dataset.
         directory (str): Directory to pull data to.
@@ -62,13 +65,7 @@ def pull(
         force (bool): Automatically download files.
     """
     # Set logging level.
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    set_log_level(logger, verbose, quiet)
     logger.debug("`pull` called with:")
     logger.debug(f"scope: {scope} [{type(scope)}]")
     logger.debug(f"dataset: {dataset} [{type(dataset)}]")
@@ -91,26 +88,31 @@ def pull(
         raise click.Abort()
 
     if not validate_scope(scope):
-        raise ValueError("Scope does not exist.")
+        error_console.print("Scope does not exist!")
+        console.print("Valid scopes are:")
+        ctx.invoke(list)
+        return None
 
     # Find files missing from localhost.
     console.print(f"\nSearching for files for {dataset} {scope}...\n")
-    files = find_missing_dataset_files(scope, dataset)
-    if len(files["missing"]) == 0:
+    files = find_missing_dataset_files(scope, dataset, directory, verbose)
+    if len(files["missing"]) == 0 and len(files["existing"]) == 0:
         console.print("No files found at minoc.", style="bold red")
         return None
     files_paths = [f.replace("cadc:CHIMEFRB", "") for f in files["missing"]]
-    common_path = path.commonpath(["/" + f for f in files_paths])
-    try:
-        to_download_size = size(common_path)
-    except SSLError:
-        error_console.print(
-            """
+    to_download_size = 0.0
+    if len(files_paths) > 0:
+        common_path = path.commonpath(["/" + f for f in files_paths])
+        try:
+            to_download_size = size(common_path)
+        except SSLError:
+            error_console.print(
+                """
 No valid CADC certificate found.
 Create one using 'cadc-get-cert -u <USERNAME>'.
 """
-        )
-        return None
+            )
+            return None
     console.print(
         f" - {len(files['existing'])} files found at {site}.",
         style="green",
@@ -127,6 +129,8 @@ Create one using 'cadc-get-cert -u <USERNAME>'.
     # Confirm download.
     if force:
         is_download = True
+    elif to_download_size == 0:
+        return None
     else:
         is_download = Confirm.ask(
             f"Download {len(files['missing'])} files?",

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/dtcli/src/functions.py

@@ -33,13 +33,8 @@ def list(  # noqa: C901
         Dict[str, Any]: Keys 'error', 'scopes', or 'datasets'. Values are the
             results or error message.
     """
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    # Set logging level.
+    utilities.set_log_level(logger, verbose, quiet)
     # Load configuration.
     logger.debug("Loading configuration.")
     try:
@@ -108,7 +103,7 @@ def ps(
     verbose: int = 0,
     quiet: bool = False,
     base_url: Optional[str] = None,
-) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
     """List detailed information about a dataset.
 
     Args:
@@ -123,13 +118,7 @@ def ps(
             and dictionary of dataset's policies.
     """
     # Set logging level.
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    utilities.set_log_level(logger, verbose, quiet)
 
     # Load configuration.
     logger.debug("Loading configuration.")
@@ -145,7 +134,7 @@ def ps(
         logger.debug(f"Setting base_url to {server}.")
         base_url = server
     try:
-        files_response = get_dataset_file_info(scope, dataset)
+        files_response = get_dataset_file_info(scope, dataset, verbose, quiet)
 
         logger.info(f"Getting policy for {dataset} in {scope}.")
         url: str = str(base_url) + f"/query/dataset/{scope}/{dataset}"
@@ -153,11 +142,12 @@ def ps(
         r = requests.get(url)
         logger.debug(f"Status: {r.status_code}.")
         policy_response = utilities.decode_response(r)
-        if "object has no attribute" in policy_response or isinstance(
+        if "object has no attribute" in policy_response and isinstance(
             files_response, str
         ):
             raise Exception(f"Could not find {dataset} {scope} in Datatrail.")
-
+        elif isinstance(files_response, str):
+            return None, policy_response
         return files_response, policy_response  # type: ignore
 
     except requests.exceptions.ConnectionError as e:
@@ -185,13 +175,7 @@ def get_dataset_file_info(
         Dict[str, Any]: JSON response from server or error string.
     """
     # Set logging level.
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    utilities.set_log_level(logger, verbose, quiet)
 
     # Load configuration.
     config = procure()
@@ -216,48 +200,55 @@ def get_dataset_file_info(
         return {"error": "Datatrail Server at CHIME is not responding."}
 
 
-def find_missing_dataset_files(scope: str, dataset: str) -> Dict:
+def find_missing_dataset_files(
+    scope: str, dataset: str, root_path: Optional[str] = None, verbose: int = 0
+) -> Dict:
     """List missing files for a dataset.
 
     Args:
         scope (str): Scope of dataset. Defaults to None.
         dataset (str): Name of dataset. Defaults to None.
+        root_path (Optional[str]): Path to download files to. Defaults to None.
+        verbose (int): Verbosity. Defaults to 0.
 
     Returns:
         Dict: Dictionary of results.
     """
-    # Load configuration.
-    config = procure()
-    SITE = config["site"]
+    # Set logging level.
+    utilities.set_log_level(logger, verbose)
+
     # find dataset
-    dataset_locations = get_dataset_file_info(scope, dataset)
+    dataset_locations = get_dataset_file_info(scope, dataset, verbose=verbose)
     if isinstance(dataset_locations, str):
         print(f"Could not find the dataset: {scope}, {dataset}")
         return {}
 
-    # stage dataset
-    site_locations = ["chime", "allenby", "gbo", "hatcreek", "canfar"]
-
     # check for local copy of the data.
-    if SITE in site_locations and dataset_locations["file_replica_locations"].get(SITE):
-        file_uris = dataset_locations["file_replica_locations"][SITE]
-        print(f"Files found at {SITE}")
+    logger.info("Checking for local copies of files.")
+    if dataset_locations["file_replica_locations"].get("minoc"):
+        file_uris = dataset_locations["file_replica_locations"]["minoc"]
+        file_paths = []
+        # Clean up file paths
+        for f in file_uris:
+            if f.startswith("data/"):
+                file_paths.append(f)
+            elif f.startswith("cadc:CHIMEFRB/"):
+                file_paths.append(f.replace("//", "/").replace("cadc:CHIMEFRB/", ""))
+            elif f.startswith("/"):
+                file_paths.append(f.replace("//", "/")[1:])
         # check for missing files
         missing_files = []
         existing_files = []
-        for f in file_uris:
-            if Path(f).exists():
+        for f in file_paths:
+            if Path(root_path + f).exists():
+                logger.debug(f"- {f} : ✔")
                 existing_files.append(f)
             else:
+                logger.debug(f"- {f} : ✘")
                 missing_files.append(f)
 
-    # For local, assume no files exist.
     else:
-        file_replicas = dataset_locations.get("file_replica_locations")
-        if file_replicas:
-            missing_files = file_replicas.get("minoc")
-        else:
-            missing_files = []
+        missing_files = []
         existing_files = []
     return {"missing": missing_files, "existing": existing_files}
 
@@ -282,11 +273,8 @@ def get_files(
         None
     """
     # Set logging level.
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
+    utilities.set_log_level(logger, verbose)
+
     # Load configuration.
     config = procure()
     mounts = config["root_mounts"]
@@ -297,7 +285,9 @@ def get_files(
         files = [f.replace("cadc:CHIMEFRB/", "") for f in files]
         if not directory:
             directory = mounts[site]
-        destinations = [directory + "/" + f for f in files]
+        if not directory.endswith("/"):
+            directory += "/"
+        destinations = [(directory + f).replace("//", "/") for f in files]
         # make directory structure if it does not exist.
         folders = {os.path.dirname(path) for path in destinations}
         for folder in folders:
@@ -322,13 +312,8 @@ def clear_dataset_path(
     Returns:
         bool: True if path was deleted.
     """
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    # Set logging level.
+    utilities.set_log_level(logger, verbose, quiet)
 
     logger.debug(f"clear_parents: {clear_parents}")
 
@@ -379,13 +364,9 @@ def find_dataset_common_path(
     Returns:
         Optional[str]: Common path for dataset.
     """
-    logger.setLevel("WARNING")
-    if verbose == 1:
-        logger.setLevel("INFO")
-    elif verbose > 1:
-        logger.setLevel("DEBUG")
-    elif quiet:
-        logger.setLevel("ERROR")
+    # Set logging level.
+    utilities.set_log_level(logger, verbose, quiet)
+
     # Load configuration.
     logger.debug("Loading configuration.")
     try:

datatrail_cli-0.4.4/dtcli/utilities/utilities.py

@@ -0,0 +1,113 @@
+"""Utility functions."""
+
+import json
+import logging
+from typing import Any, Dict, List, Union
+
+import requests
+from requests.models import Response
+
+try:
+    from packaging.version import parse
+except ImportError:
+    from pip._vendor.packaging.version import parse
+
+
+def set_log_level(logger: logging.Logger, verbose: int = 0, quiet: bool = False) -> None:
+    """Set log level."""
+    if verbose == 1:
+        logger.setLevel("INFO")
+    elif verbose > 1:
+        logger.setLevel("DEBUG")
+    elif quiet:
+        logger.setLevel("ERROR")
+    else:
+        logger.setLevel("WARNING")
+
+
+def decode_response(response: Response) -> Union[Dict, str]:
+    """Decode response.
+
+    Args:
+        response (Response): Request response.
+
+    Returns:
+        Union[Dict, str]: JSON response or text.
+    """
+    if response.status_code in [200, 201]:
+        return response.json()
+    else:
+        return response.text
+
+
+def split(data: List[Any], count: int) -> List[List[Any]]:
+    """Split a list into batches.
+
+    Args:
+        data (List[Any]): List to split.
+        count (int): Number of batches to split into.
+
+    Returns:
+        List[List[Any]]: List of batches.
+    """
+    batch_size = len(data) // count
+    remainder = len(data) % count
+    batches: List[Any] = []
+    idx = 0
+    for i in range(count):
+        if i < remainder:
+            batch = data[idx : idx + batch_size + 1]  # noqa: E203
+            idx += batch_size + 1
+        else:
+            batch = data[idx : idx + batch_size]  # noqa: E203
+            idx += batch_size
+        if len(batch) > 0:
+            batches.append(batch)
+    return batches
+
+
+def validate_scope(scope: str) -> bool:
+    """Check if scope is valid.
+
+    Args:
+        scope (str): Scope to check.
+
+    Returns:
+        bool: True if scope is valid.
+    """
+    resp = requests.get("https://frb.chimenet.ca/datatrail/query/dataset/scopes")
+    scopes = decode_response(resp)
+    return scope in scopes
+
+
+def get_latest_released_version(
+    package: str = "datatrail-cli",
+    url_pattern: str = "https://pypi.python.org/pypi/{package}/json",
+):
+    """Get latest released version of a package from pypi.python.org.
+
+    Args:
+        package (str): Package name. Defaults to "datatrail-cli".
+        url_pattern (str): URL pattern. Defaults to "https://pypi.python.org/pypi/{package}/json".  # noqa: E501
+
+    Returns:
+        str: Latest released version.
+    """
+    req = requests.get(url_pattern.format(package=package))
+    version = parse("0")
+    if req.status_code == requests.codes.ok:
+        j = json.loads(req.text.encode(req.encoding))  # type: ignore
+        releases = j.get("releases", [])
+        for release in releases:
+            ver = parse(release)
+            if not ver.is_prerelease:
+                version = max(version, ver)
+    return version
+
+
+def cli_is_latest_release() -> bool:
+    """Check if CLI is latest release."""
+    from pkg_resources import get_distribution
+
+    current_version = parse(get_distribution("datatrail-cli").version)
+    return current_version == get_latest_released_version()

{datatrail_cli-0.4.2 → datatrail_cli-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datatrail-cli"
-version = "0.4.2"
+version = "0.4.4"
 description = "CHIME/FRB Datatrail CLI"
 authors = ["CHIME FRB Project Office"]
 license = "MIT"
@@ -8,7 +8,7 @@ readme = "README.md"
 packages = [{ include = "dtcli" }]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.8.1"
 click = "^8.1.3"
 requests = "^2.29.0"
 rich = "^13.3.5"
@@ -37,6 +37,7 @@ mkdocs = "^1.4.3"
 mkdocs-material = "^9.1.11"
 mkdocstrings = "^0.21.2"
 mkdocs-click = "^0.8.0"
+termynal = "^0.10.1"
 
 [tool.flake8]
 max-line-length=89

datatrail_cli-0.4.2/dtcli/utilities/utilities.py

@@ -1,61 +0,0 @@
-"""Utility functions."""
-
-from typing import Any, Dict, List, Union
-
-import requests
-from requests.models import Response
-
-
-def decode_response(response: Response) -> Union[Dict, str]:
-    """Decode response.
-
-    Args:
-        response (Response): Request response.
-
-    Returns:
-        Union[Dict, str]: JSON response or text.
-    """
-    if response.status_code in [200, 201]:
-        return response.json()
-    else:
-        return response.text
-
-
-def split(data: List[Any], count: int) -> List[List[Any]]:
-    """Split a list into batches.
-
-    Args:
-        data (List[Any]): List to split.
-        count (int): Number of batches to split into.
-
-    Returns:
-        List[List[Any]]: List of batches.
-    """
-    batch_size = len(data) // count
-    remainder = len(data) % count
-    batches: List[Any] = []
-    idx = 0
-    for i in range(count):
-        if i < remainder:
-            batch = data[idx : idx + batch_size + 1]  # noqa: E203
-            idx += batch_size + 1
-        else:
-            batch = data[idx : idx + batch_size]  # noqa: E203
-            idx += batch_size
-        if len(batch) > 0:
-            batches.append(batch)
-    return batches
-
-
-def validate_scope(scope: str) -> bool:
-    """Check if scope is valid.
-
-    Args:
-        scope (str): Scope to check.
-
-    Returns:
-        bool: True if scope is valid.
-    """
-    resp = requests.get("https://frb.chimenet.ca/datatrail/query/dataset/scopes")
-    scopes = decode_response(resp)
-    return scope in scopes