cloudcat 0.3.0__tar.gz → 0.3.1__tar.gz

{cloudcat-0.3.0 → cloudcat-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.3.0
+Version: 0.3.1
 Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
 Home-page: https://github.com/jonathansudhakar1/cloudcat
 Author: Jonathan Sudhakar

{cloudcat-0.3.0 → cloudcat-0.3.1}/cloudcat/__init__.py

@@ -2,4 +2,4 @@
 #!/usr/bin/env python
 """cloudcat - A CLI utility to read and display files from cloud storage."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{cloudcat-0.3.0 → cloudcat-0.3.1}/cloudcat/cli.py

@@ -538,7 +538,8 @@ def get_record_count(
     bucket: str,
     object_path: str,
     input_format: str,
-    delimiter: Optional[str] = None
+    delimiter: Optional[str] = None,
+    quiet: bool = False
 ):
     """Get record count from a file.
 
@@ -548,6 +549,7 @@ def get_record_count(
         object_path: Object path.
         input_format: Data format.
         delimiter: CSV delimiter.
+        quiet: If True, suppress progress messages.
 
     Returns:
         Record count (int) or "Unknown" on failure.
@@ -582,7 +584,8 @@ def get_record_count(
                 pass  # Ignore cleanup errors
     else:
         # For CSV and JSON, we need to count the rows
-        click.echo(Fore.YELLOW + "Counting records (this might take a while for large files)..." + Style.RESET_ALL)
+        if not quiet:
+            click.echo(Fore.YELLOW + "Counting records (this might take a while for large files)..." + Style.RESET_ALL)
 
         stream = get_stream(service, bucket, object_path)
         if compression:
@@ -665,6 +668,42 @@ def get_record_count(
         return "Unknown"
 
 
+def get_record_count_multiple_files(
+    service: str,
+    bucket: str,
+    file_list: List[Tuple[str, int]],
+    input_format: str,
+    delimiter: Optional[str] = None
+):
+    """Get total record count across multiple files.
+
+    Args:
+        service: Cloud service identifier.
+        bucket: Bucket or container name.
+        file_list: List of (filename, size) tuples.
+        input_format: Data format.
+        delimiter: CSV delimiter.
+
+    Returns:
+        Total record count (int) or "Unknown" on failure.
+    """
+    click.echo(Fore.YELLOW + f"Counting records across {len(file_list)} files..." + Style.RESET_ALL)
+    total_count = 0
+
+    for file_name, file_size in file_list:
+        try:
+            count = get_record_count(service, bucket, file_name, input_format, delimiter, quiet=True)
+            if isinstance(count, int):
+                total_count += count
+                click.echo(Fore.BLUE + f"  {file_name}: {count:,} records" + Style.RESET_ALL)
+            else:
+                click.echo(Fore.YELLOW + f"  {file_name}: {count}" + Style.RESET_ALL)
+        except Exception as e:
+            click.echo(Fore.YELLOW + f"  {file_name}: Error - {str(e)}" + Style.RESET_ALL)
+
+    return total_count
+
+
 @click.command()
 @click.version_option(version=__version__, prog_name='cloudcat')
 @click.option('--path', '-p', required=True, help='Path to the file or directory (gcs://, s3://, or az://)')
@@ -787,6 +826,7 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
 
         # Initialize streaming stats
         streaming_stats = None
+        multi_file_list = None  # For directory reads with --count
 
         # Handle directory paths based on multi-file-mode
         if is_directory:
@@ -819,7 +859,7 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
                 file_list = get_files_for_multiread(service, bucket, object_path, input_format, max_size_mb)
 
                 # Read data from multiple files
-                df, full_schema, total_record_count = read_data_from_multiple_files(
+                df, full_schema, rows_in_files = read_data_from_multiple_files(
                     service, bucket, file_list, input_format, num_rows, columns, delimiter, offset
                 )
 
@@ -827,6 +867,11 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
                 total_size = sum(f[1] for f in file_list)
                 streaming_stats = StreamingStats(file_size=total_size, bytes_read=total_size, format_type=input_format)
 
+                # Store file_list for --count to use later
+                multi_file_list = file_list
+                # total_record_count will be computed later if --count is specified
+                total_record_count = None
+
                 # Update object_path for display/logging purposes
                 object_path = f"{object_path} ({len(file_list)} files)"
         else:
@@ -860,8 +905,13 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
             if count:
                 try:
                     if total_record_count is None:
-                        total_record_count = get_record_count(service, bucket, object_path, input_format, delimiter)
-                    click.echo(Fore.CYAN + f"Total records: {total_record_count}" + Style.RESET_ALL)
+                        if multi_file_list:
+                            total_record_count = get_record_count_multiple_files(
+                                service, bucket, multi_file_list, input_format, delimiter
+                            )
+                        else:
+                            total_record_count = get_record_count(service, bucket, object_path, input_format, delimiter)
+                    click.echo(Fore.CYAN + f"Total records: {total_record_count:,}" + Style.RESET_ALL)
                 except Exception as e:
                     click.echo(Fore.YELLOW + f"Could not count records: {str(e)}" + Style.RESET_ALL)
             return
@@ -884,8 +934,15 @@ def main(path, output_format, input_format, columns, num_rows, offset, where, sc
         if count:
             try:
                 if total_record_count is None:
-                    total_record_count = get_record_count(service, bucket, object_path, input_format, delimiter)
-                click.echo(Fore.CYAN + f"\nTotal records: {total_record_count}" + Style.RESET_ALL)
+                    if multi_file_list:
+                        # Count across all files in the directory
+                        total_record_count = get_record_count_multiple_files(
+                            service, bucket, multi_file_list, input_format, delimiter
+                        )
+                    else:
+                        # Single file count
+                        total_record_count = get_record_count(service, bucket, object_path, input_format, delimiter)
+                click.echo(Fore.CYAN + f"\nTotal records: {total_record_count:,}" + Style.RESET_ALL)
             except Exception as e:
                 click.echo(Fore.YELLOW + f"\nCould not count records: {str(e)}" + Style.RESET_ALL)
 

{cloudcat-0.3.0 → cloudcat-0.3.1}/cloudcat/compression.py

@@ -33,6 +33,10 @@ except ImportError:
 def detect_compression(path: str) -> Optional[str]:
     """Detect compression type from file extension.
 
+    For Parquet and ORC files, snappy is an internal codec handled by PyArrow,
+    not external compression. Files like .parquet.snappy should be read directly
+    by PyArrow without external decompression.
+
     Args:
         path: File path to check for compression extension.
 
@@ -40,6 +44,12 @@ def detect_compression(path: str) -> Optional[str]:
         Compression type string ('gzip', 'zstd', 'lz4', 'snappy', 'bz2') or None.
     """
     path_lower = path.lower()
+
+    # For Parquet and ORC, snappy is an internal codec - PyArrow handles it
+    # Files like .parquet.snappy or .orc.snappy don't need external decompression
+    if path_lower.endswith('.parquet.snappy') or path_lower.endswith('.orc.snappy'):
+        return None
+
     if path_lower.endswith('.gz') or path_lower.endswith('.gzip'):
         return 'gzip'
     elif path_lower.endswith('.zst') or path_lower.endswith('.zstd'):
@@ -101,6 +111,9 @@ def decompress_stream(stream: Union[BinaryIO, bytes], compression: str) -> io.By
 def strip_compression_extension(path: str) -> str:
     """Remove compression extension from path to get the actual file extension.
 
+    For Parquet and ORC files with .snappy extension (e.g., .parquet.snappy),
+    snappy is an internal codec, so we strip .snappy to get .parquet.
+
     Args:
         path: File path that may have a compression extension.
 
@@ -108,6 +121,14 @@ def strip_compression_extension(path: str) -> str:
         Path with compression extension removed.
     """
     path_lower = path.lower()
+
+    # For .parquet.snappy and .orc.snappy, strip .snappy to get the format
+    # (snappy is internal codec, not external compression)
+    if path_lower.endswith('.parquet.snappy'):
+        return path[:-len('.snappy')]
+    if path_lower.endswith('.orc.snappy'):
+        return path[:-len('.snappy')]
+
     for ext in COMPRESSION_EXTENSIONS:
         if path_lower.endswith(ext):
             return path[:-len(ext)]

{cloudcat-0.3.0 → cloudcat-0.3.1}/cloudcat.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.3.0
+Version: 0.3.1
 Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
 Home-page: https://github.com/jonathansudhakar1/cloudcat
 Author: Jonathan Sudhakar

{cloudcat-0.3.0 → cloudcat-0.3.1}/tests/test_compression.py

@@ -32,7 +32,18 @@ class TestCompressionDetection:
 
     def test_snappy_detection(self):
         assert detect_compression("data.csv.snappy") == "snappy"
-        assert detect_compression("path/to/file.PARQUET.SNAPPY") == "snappy"
+        assert detect_compression("data.json.snappy") == "snappy"
+
+    def test_parquet_snappy_is_internal_codec(self):
+        # For Parquet files, .snappy is an internal codec handled by PyArrow
+        # Not external compression - PyArrow reads these directly
+        assert detect_compression("path/to/file.parquet.snappy") is None
+        assert detect_compression("data.PARQUET.SNAPPY") is None
+
+    def test_orc_snappy_is_internal_codec(self):
+        # For ORC files, .snappy is an internal codec handled by PyArrow
+        assert detect_compression("path/to/file.orc.snappy") is None
+        assert detect_compression("data.ORC.SNAPPY") is None
 
     def test_bz2_detection(self):
         assert detect_compression("data.csv.bz2") == "bz2"
@@ -66,6 +77,18 @@ class TestStripCompressionExtension:
 
     def test_strip_snappy(self):
         assert strip_compression_extension("data.csv.snappy") == "data.csv"
+        assert strip_compression_extension("data.json.snappy") == "data.json"
+
+    def test_strip_parquet_snappy(self):
+        # For .parquet.snappy, strip .snappy to get .parquet format
+        # (snappy is internal codec, not external compression)
+        assert strip_compression_extension("data.parquet.snappy") == "data.parquet"
+        assert strip_compression_extension("path/to/file.PARQUET.SNAPPY") == "path/to/file.PARQUET"
+
+    def test_strip_orc_snappy(self):
+        # For .orc.snappy, strip .snappy to get .orc format
+        assert strip_compression_extension("data.orc.snappy") == "data.orc"
+        assert strip_compression_extension("path/to/file.ORC.SNAPPY") == "path/to/file.ORC"
 
     def test_strip_bz2(self):
         assert strip_compression_extension("data.csv.bz2") == "data.csv"