cloudcat 0.3.1__tar.gz → 0.3.3__tar.gz

{cloudcat-0.3.1 → cloudcat-0.3.3}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.3.1
-Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
+Version: 0.3.3
+Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Data Lake Storage Gen2 from your terminal
 Home-page: https://github.com/jonathansudhakar1/cloudcat
 Author: Jonathan Sudhakar
 Author-email: jonathan@example.com
@@ -9,7 +9,7 @@ Project-URL: Homepage, https://github.com/jonathansudhakar1/cloudcat
 Project-URL: Documentation, https://cloudcatcli.com
 Project-URL: Bug Reports, https://github.com/jonathansudhakar1/cloudcat/issues
 Project-URL: Source, https://github.com/jonathansudhakar1/cloudcat
-Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,azure-blob,data-engineering,etl,spark,bigquery,databricks,kafka,hive
+Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,adls,datalake,data-engineering,etl,spark,bigquery,databricks,kafka,hive
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
@@ -38,7 +38,7 @@ Requires-Dist: tabulate>=0.8.9
 Requires-Dist: colorama>=0.4.4
 Requires-Dist: google-cloud-storage>=2.0.0
 Requires-Dist: boto3>=1.18.0
-Requires-Dist: azure-storage-blob>=12.0.0
+Requires-Dist: azure-storage-file-datalake>=12.0.0
 Requires-Dist: azure-identity>=1.0.0
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=5.0.0; extra == "parquet"

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/__init__.py

@@ -2,4 +2,4 @@
 #!/usr/bin/env python
 """cloudcat - A CLI utility to read and display files from cloud storage."""
 
-__version__ = "0.3.1"
+__version__ = "0.3.3"

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/cli.py

@@ -29,6 +29,7 @@ from .compression import (
 )
 from .filtering import parse_where_clause, apply_where_filter
 from .formatters import colorize_json, format_table_with_colored_header
+from .progress import start_progress, update_progress, stop_progress
 from .storage import (
     parse_cloud_path,
     get_stream,
@@ -75,7 +76,8 @@ def get_files_for_multiread(
     bucket: str,
     prefix: str,
     input_format: Optional[str] = None,
-    max_size_mb: int = 25
+    max_size_mb: int = 25,
+    quiet: bool = False
 ) -> List[Tuple[str, int]]:
     """Get a list of files to read up to max_size_mb.
 
@@ -85,6 +87,7 @@ def get_files_for_multiread(
         prefix: Directory prefix.
         input_format: Optional format filter.
         max_size_mb: Maximum total size in MB.
+        quiet: If True, suppress progress messages.
 
     Returns:
         List of (filename, size) tuples.
@@ -137,18 +140,21 @@ def get_files_for_multiread(
     total_size = 0
 
     for file_name, file_size in filtered_files:
+        # Check if adding this file would exceed the limit
+        # Always include at least the first file even if it exceeds the limit
+        if selected_files and total_size + file_size > max_size_bytes:
+            break
+
         selected_files.append((file_name, file_size))
         total_size += file_size
 
-        if total_size >= max_size_bytes:
-            break
-
     if not selected_files:
         raise ValueError(f"No suitable files found in {service}://{bucket}/{prefix}")
 
     # Report on selected files
-    total_mb = total_size / (1024 * 1024)
-    click.echo(Fore.BLUE + f"Reading {len(selected_files)} files totaling {total_mb:.2f} MB" + Style.RESET_ALL)
+    if not quiet:
+        total_mb = total_size / (1024 * 1024)
+        click.echo(Fore.BLUE + f"Reading {len(selected_files)} files totaling {total_mb:.2f} MB" + Style.RESET_ALL)
 
     return selected_files
 
@@ -252,7 +258,8 @@ def read_data_from_multiple_files(
     num_rows: int,
     columns: Optional[str] = None,
     delimiter: Optional[str] = None,
-    offset: int = 0
+    offset: int = 0,
+    quiet: bool = False
 ) -> Tuple[pd.DataFrame, pd.Series, int]:
     """Read data from multiple files and concatenate the results.
 
@@ -265,6 +272,7 @@ def read_data_from_multiple_files(
         columns: Columns to select.
         delimiter: CSV delimiter.
         offset: Rows to skip.
+        quiet: Suppress progress messages.
 
     Returns:
         Tuple of (DataFrame, schema, total_rows).
@@ -275,16 +283,22 @@ def read_data_from_multiple_files(
     rows_skipped = 0
     total_rows = 0
 
-    def process_file(file_info, remaining_to_skip, remaining_to_read):
+    def process_file(file_info, remaining_to_skip, remaining_to_read, file_index, total_files):
         file_name, file_size = file_info
-        click.echo(Fore.BLUE + f"Reading file: {file_name} ({file_size/1024:.1f} KB)" + Style.RESET_ALL)
+        if not quiet:
+            click.echo(Fore.BLUE + f"Reading file: {file_name} ({file_size/1024:.1f} KB)" + Style.RESET_ALL)
+        else:
+            # Update progress indicator with current file
+            short_name = file_name.split('/')[-1]
+            update_progress(f"Reading file {file_index + 1}/{total_files}: {short_name}")
 
         stream = get_stream(service, bucket, file_name)
 
         # Check for compression and decompress if needed
         compression = detect_compression(file_name)
         if compression:
-            click.echo(Fore.BLUE + f"Detected {compression} compression, decompressing..." + Style.RESET_ALL)
+            if not quiet:
+                click.echo(Fore.BLUE + f"Detected {compression} compression, decompressing..." + Style.RESET_ALL)
             stream = decompress_stream(stream, compression)
 
         # Calculate how many rows to read from this file
@@ -311,10 +325,11 @@ def read_data_from_multiple_files(
     # Process files in order until we have enough rows
     remaining_offset = offset
     remaining_rows = num_rows if num_rows > 0 else float('inf')
+    total_files = len(file_list)
 
-    for file_info in file_list:
+    for file_index, file_info in enumerate(file_list):
         try:
-            df, schema, file_rows = process_file(file_info, remaining_offset, remaining_rows if remaining_rows != float('inf') else 0)
+            df, schema, file_rows = process_file(file_info, remaining_offset, remaining_rows if remaining_rows != float('inf') else 0, file_index, total_files)
 
             if not df.empty:
                 total_rows += len(df)
@@ -457,7 +472,8 @@ def read_data_streaming(
                 aws_profile=cloud_config.aws_profile,
                 gcp_project=cloud_config.gcp_project,
                 gcp_credentials=cloud_config.gcp_credentials,
-                azure_account=cloud_config.azure_account
+                azure_account=cloud_config.azure_account,
+                azure_access_key=cloud_config.azure_access_key
             )
             pyarrow_path = f"{bucket}/{object_path}"
 
@@ -706,7 +722,7 @@ def get_record_count_multiple_files(
 
 @click.command()
 @click.version_option(version=__version__, prog_name='cloudcat')
-@click.option('--path', '-p', required=True, help='Path to the file or directory (gcs://, s3://, or az://)')
+@click.option('--path', '-p', required=True, help='Path to the file or directory (gcs://, s3://, or abfss://)')
 @click.option('--output-format', '-o', type=click.Choice(['json', 'jsonp', 'csv', 'table']), default='table',
               help='Output format (default: table)')
 @click.option('--input-format', '-i', type=click.Choice(['json', 'csv', 'parquet', 'avro', 'orc', 'text']),
@@ -726,10 +742,11 @@ def get_record_count_multiple_files(
 @click.option('--profile', help='AWS profile name (for S3 access)')
 @click.option('--project', help='GCP project ID (for GCS access)')
 @click.option('--credentials', help='Path to GCP service account JSON file')
-@click.option('--account', help='Azure storage account name')
+@click.option('--az-access-key', help='Azure storage account access key')
+@click.option('--yes', '-y', is_flag=True, help='Skip confirmation prompts (for scripting)')
 def main(path, output_format, input_format, columns, num_rows, offset, where, schema, count,
-         multi_file_mode, max_size_mb, delimiter, profile, project, credentials, account):
-    """Display data from files in Google Cloud Storage, AWS S3, or Azure Blob Storage.
+         multi_file_mode, max_size_mb, delimiter, profile, project, credentials, az_access_key, yes):
+    """Display data from files in Google Cloud Storage, AWS S3, or Azure Data Lake Storage Gen2.
 
     Supported formats: CSV, JSON, Parquet, Avro, ORC, and plain text.
     Supports compressed files: .gz, .zst, .lz4, .snappy, .bz2
@@ -745,8 +762,8 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
     cloudcat --path s3://my-bucket/data.parquet --columns id,name,value
 
     \b
-    # Read from Azure Blob Storage
-    cloudcat --path az://my-container/data.json --output-format jsonp
+    # Read from Azure Data Lake Storage Gen2
+    cloudcat --path abfss://container@account.dfs.core.windows.net/data.json --output-format jsonp
 
     \b
     # Read Avro files from Kafka exports
@@ -758,7 +775,7 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
 
     \b
     # Read log files as plain text
-    cloudcat --path az://logs/app.log --input-format text
+    cloudcat --path abfss://logs@account.dfs.core.windows.net/app.log --input-format text
 
     \b
     # Read from a directory (reads first non-empty data file)
@@ -800,8 +817,8 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
     cloudcat --path gcs://bucket/data.csv --credentials /path/to/service-account.json
 
     \b
-    # Use specific Azure storage account
-    cloudcat --path az://container/data.csv --account mystorageaccount
+    # Use Azure storage account access key
+    cloudcat --path abfss://container@account.dfs.core.windows.net/data.csv --az-access-key YOUR_KEY
     """
     try:
         # Configure cloud credentials from CLI options
@@ -811,8 +828,8 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
             cloud_config.gcp_project = project
         if credentials:
             cloud_config.gcp_credentials = credentials
-        if account:
-            cloud_config.azure_account = account
+        if az_access_key:
+            cloud_config.azure_access_key = az_access_key
 
         # Handle special characters in delimiter
         if delimiter == "\\t":
@@ -830,50 +847,81 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
 
         # Handle directory paths based on multi-file-mode
         if is_directory:
-            click.echo(Fore.BLUE + f"Path is a directory" + Style.RESET_ALL)
-
             if multi_file_mode == 'first' or (multi_file_mode == 'auto' and max_size_mb <= 0):
                 # Use a single file
-                click.echo(Fore.BLUE + f"Looking for first suitable file..." + Style.RESET_ALL)
+                start_progress("Listing files...")
                 object_path = find_first_non_empty_file(service, bucket, object_path, input_format)
 
                 # Determine input format if not specified
                 if not input_format:
                     input_format = detect_format_from_path(object_path)
-                    click.echo(Fore.BLUE + f"Inferred input format: {input_format}" + Style.RESET_ALL)
+
+                # Get file name for display
+                file_name = object_path.split('/')[-1]
+                update_progress(f"Reading {file_name}...")
 
                 # Read the data from the single file with streaming
                 df, full_schema, streaming_stats = read_data_streaming(service, bucket, object_path, input_format, num_rows, columns, delimiter, offset)
                 total_record_count = None  # Will be computed later if needed
+
+                # Stop progress and show format info
+                stop_progress()
+                click.echo(Fore.BLUE + f"Inferred input format: {input_format}" + Style.RESET_ALL)
             else:
                 # Read from multiple files
-                click.echo(Fore.BLUE + f"Reading multiple files (up to {max_size_mb}MB)..." + Style.RESET_ALL)
+                start_progress("Listing files...")
 
                 # Determine input format if not specified (use the first file to infer)
                 if not input_format:
                     first_file = find_first_non_empty_file(service, bucket, object_path)
                     input_format = detect_format_from_path(first_file)
-                    click.echo(Fore.BLUE + f"Inferred input format from first file: {input_format}" + Style.RESET_ALL)
 
-                # Get files to read
-                file_list = get_files_for_multiread(service, bucket, object_path, input_format, max_size_mb)
+                update_progress(f"Selecting {input_format} files...")
+
+                # Get files to read for preview (limited by max_size_mb)
+                file_list = get_files_for_multiread(service, bucket, object_path, input_format, max_size_mb, quiet=True)
 
-                # Read data from multiple files
+                # Read data from multiple files with progress updates
+                update_progress(f"Reading {len(file_list)} files...")
                 df, full_schema, rows_in_files = read_data_from_multiple_files(
-                    service, bucket, file_list, input_format, num_rows, columns, delimiter, offset
+                    service, bucket, file_list, input_format, num_rows, columns, delimiter, offset, quiet=True
                 )
 
                 # Calculate total size for stats
                 total_size = sum(f[1] for f in file_list)
                 streaming_stats = StreamingStats(file_size=total_size, bytes_read=total_size, format_type=input_format)
 
-                # Store file_list for --count to use later
-                multi_file_list = file_list
+                # For --count, get ALL files (not limited by max_size_mb)
+                # so we can count records across the entire directory
+                if count:
+                    stop_progress()
+                    all_files = get_files_for_multiread(service, bucket, object_path, input_format, max_size_mb=999999, quiet=True)
+                    all_files_size = sum(f[1] for f in all_files)
+                    all_files_size_mb = all_files_size / (1024 * 1024)
+
+                    # Warn user about counting all files in directory
+                    if not yes:
+                        click.echo(Fore.YELLOW + f"\nWarning: --count will scan {len(all_files)} files ({all_files_size_mb:.1f} MB total)." + Style.RESET_ALL)
+                        if not click.confirm("Continue?", default=True):
+                            click.echo("Aborted.")
+                            return
+
+                    multi_file_list = all_files
+                    # Update stats to reflect all files
+                    streaming_stats.file_size = all_files_size
+                else:
+                    stop_progress()
+                    multi_file_list = file_list
+
+                # Show format info
+                click.echo(Fore.BLUE + f"Inferred input format: {input_format}" + Style.RESET_ALL)
+
                 # total_record_count will be computed later if --count is specified
                 total_record_count = None
 
                 # Update object_path for display/logging purposes
-                object_path = f"{object_path} ({len(file_list)} files)"
+                num_files_display = len(multi_file_list) if count else len(file_list)
+                object_path = f"{object_path} ({num_files_display} files)"
         else:
             # Single file path
             # Determine input format if not specified
@@ -881,10 +929,16 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
                 input_format = detect_format_from_path(object_path)
                 click.echo(Fore.BLUE + f"Inferred input format: {input_format}" + Style.RESET_ALL)
 
+            # Get file name for display
+            file_name = object_path.split('/')[-1]
+            start_progress(f"Reading {file_name}...")
+
             # Read the data with streaming
             df, full_schema, streaming_stats = read_data_streaming(service, bucket, object_path, input_format, num_rows, columns, delimiter, offset)
             total_record_count = None  # Will be computed later if needed
 
+            stop_progress()
+
         # Apply WHERE filter if specified
         if where and not df.empty:
             original_count = len(df)
@@ -942,6 +996,9 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
                     else:
                         # Single file count
                         total_record_count = get_record_count(service, bucket, object_path, input_format, delimiter)
+                    # Update stats to reflect full file was read for counting
+                    if streaming_stats:
+                        streaming_stats.bytes_read = streaming_stats.file_size
                 click.echo(Fore.CYAN + f"\nTotal records: {total_record_count:,}" + Style.RESET_ALL)
             except Exception as e:
                 click.echo(Fore.YELLOW + f"\nCould not count records: {str(e)}" + Style.RESET_ALL)

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/config.py

@@ -14,7 +14,8 @@ class CloudConfig:
         self.aws_profile: Optional[str] = None
         self.gcp_project: Optional[str] = None
         self.gcp_credentials: Optional[str] = None  # Path to service account JSON
-        self.azure_account: Optional[str] = None
+        self.azure_account: Optional[str] = None  # Extracted from abfss:// URL
+        self.azure_access_key: Optional[str] = None  # Storage account access key
 
     def reset(self) -> None:
         """Reset all configuration to defaults. Useful for testing."""
@@ -22,6 +23,7 @@ class CloudConfig:
         self.gcp_project = None
         self.gcp_credentials = None
         self.azure_account = None
+        self.azure_access_key = None
 
 
 # Global configuration instance

cloudcat-0.3.3/cloudcat/progress.py

@@ -0,0 +1,102 @@
+"""Progress indicator for CLI operations."""
+
+import sys
+import threading
+import time
+from typing import Optional
+from colorama import Fore, Style
+
+
+class ProgressIndicator:
+    """A simple animated progress indicator that updates in place.
+
+    Shows a spinner animation with a status message that updates on a single line.
+    """
+
+    SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+
+    def __init__(self):
+        self._message: str = ""
+        self._running: bool = False
+        self._thread: Optional[threading.Thread] = None
+        self._frame_index: int = 0
+        self._lock = threading.Lock()
+
+    def start(self, message: str = "Working...") -> None:
+        """Start the progress indicator with an initial message."""
+        with self._lock:
+            self._message = message
+            self._running = True
+            self._frame_index = 0
+
+        self._thread = threading.Thread(target=self._animate, daemon=True)
+        self._thread.start()
+
+    def update(self, message: str) -> None:
+        """Update the progress message."""
+        with self._lock:
+            self._message = message
+
+    def stop(self, final_message: Optional[str] = None) -> None:
+        """Stop the progress indicator and optionally show a final message."""
+        with self._lock:
+            self._running = False
+
+        if self._thread:
+            self._thread.join(timeout=0.5)
+            self._thread = None
+
+        # Clear the current line
+        sys.stderr.write('\r' + ' ' * 80 + '\r')
+        sys.stderr.flush()
+
+        # Show final message if provided
+        if final_message:
+            sys.stderr.write(final_message + '\n')
+            sys.stderr.flush()
+
+    def _animate(self) -> None:
+        """Animation loop running in a separate thread."""
+        while True:
+            with self._lock:
+                if not self._running:
+                    break
+                message = self._message
+                frame = self.SPINNER_FRAMES[self._frame_index]
+                self._frame_index = (self._frame_index + 1) % len(self.SPINNER_FRAMES)
+
+            # Write spinner and message, overwriting the line
+            output = f'\r{Fore.CYAN}{frame}{Style.RESET_ALL} {message}'
+            # Pad to clear any leftover characters from longer previous messages
+            output = output.ljust(80)
+            sys.stderr.write(output)
+            sys.stderr.flush()
+
+            time.sleep(0.08)
+
+
+# Global instance for easy access
+_progress: Optional[ProgressIndicator] = None
+
+
+def start_progress(message: str = "Working...") -> None:
+    """Start a global progress indicator."""
+    global _progress
+    if _progress is None:
+        _progress = ProgressIndicator()
+    _progress.start(message)
+
+
+def update_progress(message: str) -> None:
+    """Update the global progress indicator message."""
+    global _progress
+    if _progress:
+        _progress.update(message)
+
+
+def stop_progress(final_message: Optional[str] = None) -> None:
+    """Stop the global progress indicator."""
+    global _progress
+    if _progress:
+        _progress.stop(final_message)
+        _progress = None

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/storage/__init__.py

@@ -3,7 +3,7 @@
 from .base import parse_cloud_path, get_stream, list_directory, get_file_size
 from .gcs import get_gcs_client, get_gcs_stream, list_gcs_directory, HAS_GCS
 from .s3 import get_s3_client, get_s3_stream, list_s3_directory, HAS_S3
-from .azure import get_azure_blob_service_client, get_azure_stream, list_azure_directory, HAS_AZURE
+from .azure import get_azure_datalake_service_client, get_azure_blob_service_client, get_azure_stream, list_azure_directory, HAS_AZURE
 
 __all__ = [
     'parse_cloud_path',
@@ -18,7 +18,8 @@ __all__ = [
     'get_s3_stream',
     'list_s3_directory',
     'HAS_S3',
-    'get_azure_blob_service_client',
+    'get_azure_datalake_service_client',
+    'get_azure_blob_service_client',  # Alias for backwards compatibility
     'get_azure_stream',
     'list_azure_directory',
     'HAS_AZURE',

cloudcat-0.3.3/cloudcat/storage/azure.py

@@ -0,0 +1,133 @@
+"""Azure Data Lake Storage Gen2 (ADLS Gen2) client and operations."""
+
+import io
+import os
+import sys
+from typing import List, Tuple
+
+from colorama import Fore, Style
+
+from ..config import cloud_config
+
+# Try to import Azure Data Lake client
+try:
+    from azure.storage.filedatalake import DataLakeServiceClient
+    HAS_AZURE = True
+except ImportError:
+    DataLakeServiceClient = None
+    HAS_AZURE = False
+
+
+def get_azure_datalake_service_client():
+    """Get an Azure DataLakeServiceClient with optional account configuration.
+
+    Authentication priority:
+    1. Access key (--az-access-key or AZURE_STORAGE_ACCESS_KEY env var)
+    2. DefaultAzureCredential (az login, managed identity, etc.)
+
+    Returns:
+        azure.storage.filedatalake.DataLakeServiceClient instance.
+
+    Raises:
+        SystemExit: If azure-storage-file-datalake is not installed.
+        ValueError: If Azure credentials are not configured.
+    """
+    if not HAS_AZURE:
+        sys.stderr.write(
+            Fore.RED + "Error: azure-storage-file-datalake package is required for Azure access.\n" +
+            "Install it with: pip install azure-storage-file-datalake\n" + Style.RESET_ALL
+        )
+        sys.exit(1)
+
+    # Get account name (set by abfss:// URL parsing)
+    account_name = cloud_config.azure_account
+    if not account_name:
+        raise ValueError(
+            "Azure storage account not found. Use abfss:// URL format: "
+            "abfss://container@account.dfs.core.windows.net/path"
+        )
+
+    account_url = f"https://{account_name}.dfs.core.windows.net"
+
+    # Check for access key (CLI option or environment variable)
+    access_key = cloud_config.azure_access_key or os.environ.get('AZURE_STORAGE_ACCESS_KEY')
+
+    if access_key:
+        # Use access key authentication
+        return DataLakeServiceClient(account_url=account_url, credential=access_key)
+    else:
+        # Fall back to DefaultAzureCredential (az login, managed identity, etc.)
+        from azure.identity import DefaultAzureCredential
+        credential = DefaultAzureCredential()
+        return DataLakeServiceClient(account_url=account_url, credential=credential)
+
+
+# Keep old function name as alias for backwards compatibility
+get_azure_blob_service_client = get_azure_datalake_service_client
+
+
+def get_azure_stream(container_name: str, file_path: str) -> io.BytesIO:
+    """Get a file stream from Azure Data Lake Storage Gen2.
+
+    Args:
+        container_name: Azure filesystem (container) name.
+        file_path: File path within the filesystem.
+
+    Returns:
+        BytesIO buffer containing the file content.
+    """
+    datalake_service_client = get_azure_datalake_service_client()
+    file_system_client = datalake_service_client.get_file_system_client(file_system=container_name)
+    file_client = file_system_client.get_file_client(file_path)
+
+    # Download file to a BytesIO buffer
+    buffer = io.BytesIO()
+    download = file_client.download_file()
+    buffer.write(download.readall())
+    buffer.seek(0)
+
+    return buffer
+
+
+def get_azure_file_size(container_name: str, file_path: str) -> int:
+    """Get the size of an Azure Data Lake file without downloading it.
+
+    Args:
+        container_name: Azure filesystem (container) name.
+        file_path: File path within the filesystem.
+
+    Returns:
+        File size in bytes.
+    """
+    datalake_service_client = get_azure_datalake_service_client()
+    file_system_client = datalake_service_client.get_file_system_client(file_system=container_name)
+    file_client = file_system_client.get_file_client(file_path)
+    properties = file_client.get_file_properties()
+    return properties.size
+
+
+def list_azure_directory(container_name: str, prefix: str) -> List[Tuple[str, int]]:
+    """List files in an Azure Data Lake Storage directory.
+
+    Args:
+        container_name: Azure filesystem (container) name.
+        prefix: Directory prefix.
+
+    Returns:
+        List of (filename, size) tuples.
+    """
+    datalake_service_client = get_azure_datalake_service_client()
+    file_system_client = datalake_service_client.get_file_system_client(file_system=container_name)
+
+    # Ensure prefix ends with / to indicate a directory
+    if prefix and not prefix.endswith('/'):
+        prefix = prefix + '/'
+
+    # List paths with the prefix
+    file_list = []
+    paths = file_system_client.get_paths(path=prefix.rstrip('/'))
+    for path in paths:
+        if not path.is_directory:
+            file_list.append((path.name, path.content_length))
+
+    return file_list

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/storage/base.py

@@ -9,7 +9,7 @@ def parse_cloud_path(path: str) -> Tuple[str, str, str]:
     """Parse a cloud storage path into service, bucket/container, and object components.
 
     Args:
-        path: Cloud storage URL (gcs://, s3://, az://, or abfss://).
+        path: Cloud storage URL (gcs://, s3://, or abfss://).
 
     Returns:
         Tuple of (service, bucket, object_path).

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat/streaming/filesystems.py

@@ -29,7 +29,8 @@ def get_pyarrow_filesystem(
     aws_profile: Optional[str] = None,
     gcp_project: Optional[str] = None,
     gcp_credentials: Optional[str] = None,
-    azure_account: Optional[str] = None
+    azure_account: Optional[str] = None,
+    azure_access_key: Optional[str] = None
 ) -> Tuple[Any, str]:
     """Create a PyArrow filesystem for native cloud access.
 
@@ -42,6 +43,7 @@ def get_pyarrow_filesystem(
         gcp_project: GCP project ID for GCS.
         gcp_credentials: Path to GCP service account JSON for GCS.
         azure_account: Azure storage account name.
+        azure_access_key: Azure storage account access key.
 
     Returns:
         Tuple of (filesystem, path_prefix) where path_prefix is used
@@ -62,7 +64,7 @@ def get_pyarrow_filesystem(
     elif service == 'gcs':
         return _get_gcs_filesystem(gcp_project, gcp_credentials)
     elif service == 'azure':
-        return _get_azure_filesystem(azure_account)
+        return _get_azure_filesystem(azure_account, azure_access_key)
     else:
         raise ValueError(f"Unsupported service for PyArrow filesystem: {service}")
 
@@ -140,12 +142,14 @@ def _get_gcs_filesystem(
 
 
 def _get_azure_filesystem(
-    azure_account: Optional[str] = None
+    azure_account: Optional[str] = None,
+    azure_access_key: Optional[str] = None
 ) -> Tuple[Any, str]:
-    """Create PyArrow AzureFileSystem.
+    """Create PyArrow AzureFileSystem for ADLS Gen2.
 
     Args:
         azure_account: Azure storage account name.
+        azure_access_key: Azure storage account access key.
 
     Returns:
         Tuple of (AzureFileSystem, path_prefix).
@@ -156,16 +160,11 @@ def _get_azure_filesystem(
 
     if azure_account:
         kwargs['account_name'] = azure_account
-    else:
-        # Try to get from environment
-        account_url = os.environ.get('AZURE_STORAGE_ACCOUNT_URL')
-        if account_url:
-            # Extract account name from URL
-            # Format: https://<account>.blob.core.windows.net
-            import re
-            match = re.match(r'https://([^.]+)\.blob\.core\.windows\.net', account_url)
-            if match:
-                kwargs['account_name'] = match.group(1)
+
+    # Check for access key (parameter or environment variable)
+    access_key = azure_access_key or os.environ.get('AZURE_STORAGE_ACCESS_KEY')
+    if access_key:
+        kwargs['account_key'] = access_key
 
     filesystem = pa_fs.AzureFileSystem(**kwargs)
     return filesystem, ''

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.3.1
-Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
+Version: 0.3.3
+Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Data Lake Storage Gen2 from your terminal
 Home-page: https://github.com/jonathansudhakar1/cloudcat
 Author: Jonathan Sudhakar
 Author-email: jonathan@example.com
@@ -9,7 +9,7 @@ Project-URL: Homepage, https://github.com/jonathansudhakar1/cloudcat
 Project-URL: Documentation, https://cloudcatcli.com
 Project-URL: Bug Reports, https://github.com/jonathansudhakar1/cloudcat/issues
 Project-URL: Source, https://github.com/jonathansudhakar1/cloudcat
-Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,azure-blob,data-engineering,etl,spark,bigquery,databricks,kafka,hive
+Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,adls,datalake,data-engineering,etl,spark,bigquery,databricks,kafka,hive
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
@@ -38,7 +38,7 @@ Requires-Dist: tabulate>=0.8.9
 Requires-Dist: colorama>=0.4.4
 Requires-Dist: google-cloud-storage>=2.0.0
 Requires-Dist: boto3>=1.18.0
-Requires-Dist: azure-storage-blob>=12.0.0
+Requires-Dist: azure-storage-file-datalake>=12.0.0
 Requires-Dist: azure-identity>=1.0.0
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=5.0.0; extra == "parquet"

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ cloudcat/compression.py
 cloudcat/config.py
 cloudcat/filtering.py
 cloudcat/formatters.py
+cloudcat/progress.py
 cloudcat.egg-info/PKG-INFO
 cloudcat.egg-info/SOURCES.txt
 cloudcat.egg-info/dependency_links.txt

{cloudcat-0.3.1 → cloudcat-0.3.3}/cloudcat.egg-info/requires.txt

@@ -4,7 +4,7 @@ tabulate>=0.8.9
 colorama>=0.4.4
 google-cloud-storage>=2.0.0
 boto3>=1.18.0
-azure-storage-blob>=12.0.0
+azure-storage-file-datalake>=12.0.0
 azure-identity>=1.0.0
 
 [all]

{cloudcat-0.3.1 → cloudcat-0.3.3}/setup.py

@@ -30,7 +30,7 @@ setup(
         # Cloud providers (included by default)
         "google-cloud-storage>=2.0.0",
         "boto3>=1.18.0",
-        "azure-storage-blob>=12.0.0",
+        "azure-storage-file-datalake>=12.0.0",
         "azure-identity>=1.0.0",
     ],
     tests_require=[
@@ -60,10 +60,10 @@ setup(
     },
     author="Jonathan Sudhakar",
     author_email="jonathan@example.com",
-    description="Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal",
+    description="Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Data Lake Storage Gen2 from your terminal",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    keywords="cloud, gcs, s3, azure, cli, storage, data, parquet, csv, json, avro, orc, google-cloud, aws, azure-blob, data-engineering, etl, spark, bigquery, databricks, kafka, hive",
+    keywords="cloud, gcs, s3, azure, cli, storage, data, parquet, csv, json, avro, orc, google-cloud, aws, adls, datalake, data-engineering, etl, spark, bigquery, databricks, kafka, hive",
     project_urls={
         "Homepage": "https://github.com/jonathansudhakar1/cloudcat",
         "Documentation": "https://cloudcatcli.com",

cloudcat-0.3.1/cloudcat/storage/azure.py

@@ -1,125 +0,0 @@
-"""Azure Blob Storage client and operations."""
-
-import io
-import os
-import sys
-from typing import List, Tuple
-
-from colorama import Fore, Style
-
-from ..config import cloud_config
-
-# Try to import Azure client
-try:
-    from azure.storage.blob import BlobServiceClient, ContainerClient
-    HAS_AZURE = True
-except ImportError:
-    BlobServiceClient = None
-    ContainerClient = None
-    HAS_AZURE = False
-
-
-def get_azure_blob_service_client():
-    """Get an Azure BlobServiceClient with optional account configuration.
-
-    Returns:
-        azure.storage.blob.BlobServiceClient instance.
-
-    Raises:
-        SystemExit: If azure-storage-blob is not installed.
-        ValueError: If Azure credentials are not configured.
-    """
-    if not HAS_AZURE:
-        sys.stderr.write(
-            Fore.RED + "Error: azure-storage-blob package is required for Azure access.\n" +
-            "Install it with: pip install azure-storage-blob\n" + Style.RESET_ALL
-        )
-        sys.exit(1)
-
-    # Check for explicit account override
-    if cloud_config.azure_account:
-        account_url = f"https://{cloud_config.azure_account}.blob.core.windows.net"
-        from azure.identity import DefaultAzureCredential
-        credential = DefaultAzureCredential()
-        return BlobServiceClient(account_url=account_url, credential=credential)
-
-    # Fall back to environment variables
-    connection_string = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
-    account_url = os.environ.get('AZURE_STORAGE_ACCOUNT_URL')
-
-    if connection_string:
-        return BlobServiceClient.from_connection_string(connection_string)
-    elif account_url:
-        from azure.identity import DefaultAzureCredential
-        credential = DefaultAzureCredential()
-        return BlobServiceClient(account_url=account_url, credential=credential)
-    else:
-        raise ValueError(
-            "Azure credentials not found. Use --account, or set "
-            "AZURE_STORAGE_CONNECTION_STRING or AZURE_STORAGE_ACCOUNT_URL environment variable."
-        )
-
-
-def get_azure_stream(container_name: str, blob_name: str) -> io.BytesIO:
-    """Get a file stream from Azure Blob Storage.
-
-    Args:
-        container_name: Azure container name.
-        blob_name: Blob path within the container.
-
-    Returns:
-        BytesIO buffer containing the file content.
-    """
-    blob_service_client = get_azure_blob_service_client()
-    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
-
-    # Download blob to a BytesIO buffer
-    buffer = io.BytesIO()
-    download_stream = blob_client.download_blob()
-    buffer.write(download_stream.readall())
-    buffer.seek(0)
-
-    return buffer
-
-
-def get_azure_file_size(container_name: str, blob_name: str) -> int:
-    """Get the size of an Azure blob without downloading it.
-
-    Args:
-        container_name: Azure container name.
-        blob_name: Blob path within the container.
-
-    Returns:
-        File size in bytes.
-    """
-    blob_service_client = get_azure_blob_service_client()
-    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
-    properties = blob_client.get_blob_properties()
-    return properties.size
-
-
-def list_azure_directory(container_name: str, prefix: str) -> List[Tuple[str, int]]:
-    """List files in an Azure Blob Storage container directory.
-
-    Args:
-        container_name: Azure container name.
-        prefix: Directory prefix.
-
-    Returns:
-        List of (filename, size) tuples.
-    """
-    blob_service_client = get_azure_blob_service_client()
-    container_client = blob_service_client.get_container_client(container_name)
-
-    # Ensure prefix ends with / to indicate a directory
-    if prefix and not prefix.endswith('/'):
-        prefix = prefix + '/'
-
-    # List blobs with the prefix
-    file_list = []
-    blobs = container_client.list_blobs(name_starts_with=prefix)
-    for blob in blobs:
-        if not blob.name.endswith('/'):
-            file_list.append((blob.name, blob.size))
-
-    return file_list