tab-cli 0.1.1__tar.gz → 0.1.2__tar.gz

{tab_cli-0.1.1 → tab_cli-0.1.2}/CHANGELOG.md

@@ -1,3 +1,5 @@
+ - 0.1.2:
+   - Bugfix on reading directories.
  - 0.1.1:
    - Better credential handling for Azure Blob Storage and Google Cloud Storage.
  - 0.1.0: Initial release

{tab_cli-0.1.1 → tab_cli-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tab-cli
-Version: 0.1.1
+Version: 0.1.2
 Summary: A CLI tool for tabular data
 Author-email: Tongfei Chen <tongfei@pm.me>
 License-File: LICENSE
@@ -18,7 +18,9 @@ Requires-Dist: adlfs>=2025.1.0; extra == 'azure'
 Requires-Dist: azure-identity>=1.10.0; extra == 'azure'
 Provides-Extra: gcs
 Requires-Dist: gcsfs>=2025.1.0; extra == 'gcs'
+Requires-Dist: google-auth>=2.27.0; extra == 'gcs'
 Provides-Extra: s3
+Requires-Dist: boto3>=1.28.0; extra == 's3'
 Requires-Dist: s3fs>=2025.1.0; extra == 's3'
 Description-Content-Type: text/markdown
 

{tab_cli-0.1.1 → tab_cli-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tab-cli"
-version = "0.1.1"
+version = "0.1.2"
 description = "A CLI tool for tabular data"
 authors = [{name = "Tongfei Chen", email = "tongfei@pm.me"}]
 readme = "README.md"
@@ -18,8 +18,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-s3 = ["s3fs>=2025.1.0"]
-gcs = ["gcsfs>=2025.1.0"]
+s3 = ["s3fs>=2025.1.0", "boto3>=1.28.0"]
+gcs = ["gcsfs>=2025.1.0", "google-auth>=2.27.0"]
 azure = ["adlfs>=2025.1.0", "azure-identity>=1.10.0"]
 
 [dependency-groups]

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/formats/parquet.py

@@ -31,22 +31,8 @@ class ParquetFormat(FormatHandler):
         return pl.scan_parquet(url, storage_options=storage_options).select(pl.len()).collect().item()
 
     def extra_summary(self, url: str) -> dict[str, str | int | float] | None:
-        import pyarrow.parquet as pq
-
-        pf = pq.ParquetFile(url)
-        metadata = pf.metadata
-
-        extra: dict[str, str | int | float] = {}
-        codecs: set[str] = set()
-        for rg_idx in range(metadata.num_row_groups):
-            rg = metadata.row_group(rg_idx)
-            for col_idx in range(rg.num_columns):
-                col = rg.column(col_idx)
-                codecs.add(col.compression)
-        extra["Row groups"] = metadata.num_row_groups
-        if codecs:
-            extra["Compression"] = ", ".join(sorted(codecs))
-        return extra
+        # TODO: Parquet metadata
+        pass
 
     def write(self, lf: pl.LazyFrame) -> Iterable[bytes]:
         output = BytesIO()

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/handlers/__init__.py

@@ -28,6 +28,13 @@ def _get_extension(path: str) -> str:
     return os.path.splitext(basename)[1][1:].lower()
 
 
+def _is_data_file(path: str) -> bool:
+    """Check if a file is a data file (not metadata/marker file)."""
+    basename = os.path.basename(path.rstrip("/"))
+    # Skip hidden files (e.g., .crc files) or Spark/Hadoop marker and metadata files
+    return not (basename.startswith(".") or basename.startswith("_"))
+
+
 def infer_reader(path: str, format: str | None = None) -> TableReader:
     """Infer the reader for a file.
 
@@ -48,13 +55,15 @@ def infer_reader(path: str, format: str | None = None) -> TableReader:
 
     # Infer format from path
     if backend.is_directory(path):
-        # Get extension from first file in directory
+        # Get extension from first data file in directory (skip metadata files)
         for file_info in backend.list_files(path, ""):
+            if not _is_data_file(file_info.url):
+                continue
             extension = _get_extension(file_info.url)
             if extension:
                 break
         else:
-            raise ValueError(f"No files found in directory: {path}")
+            raise ValueError(f"No data files found in directory: {path}")
     else:
         extension = _get_extension(path)
 

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/handlers/base.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 
 import polars as pl
 from rich import box
-from rich.progress import Progress
+from rich.progress import Progress, track
 from rich.table import Table
 
 from tab_cli.formats.base import FormatHandler
@@ -55,7 +55,7 @@ class TableSummary:
         table = Table(
             show_header=False,
             box=box.SIMPLE_HEAD,
-            row_styles=["", _ALT_ROW_STYLE],
+            row_styles=[_ALT_ROW_STYLE_0, _ALT_ROW_STYLE_1],
         )
         table.add_column(style=_KEY_STYLE)
         table.add_column(style=_VAL_STYLE)
@@ -162,7 +162,7 @@ class TableReader:
         extra_numeric: dict[str, float] = {}
         extra_strings: dict[str, set[str]] = {}
 
-        for file_info in files:
+        for file_info in track(files):
             file_size += file_info.size
             polars_uri = self.backend.normalize_for_polars(file_info.url)
             storage_options = self.backend.storage_options(file_info.url)

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/storage/aws.py

@@ -1,10 +1,9 @@
 import os
 from enum import Enum
 
-from typing import BinaryIO, Iterator
 from loguru import logger
 
-from tab_cli.storage.base import StorageBackend, FileInfo
+from tab_cli.storage.fsspec import CloudFsspecBackend
 from tab_cli.url_parser import parse_url
 
 
@@ -14,7 +13,7 @@ class AwsAuthMethod(Enum):
     ANONYMOUS = 3        # Public buckets
 
 
-class AwsBackend(StorageBackend):
+class AwsBackend(CloudFsspecBackend):
     """Storage backend for AWS S3.
 
     URL format: s3://bucket/path
@@ -31,6 +30,8 @@ class AwsBackend(StorageBackend):
     3. Anonymous access (for public buckets, if requested)
     """
 
+    protocol = "s3"
+
     def __init__(self, anon: bool = False) -> None:
         """Initialize the AWS S3 storage backend.
 
@@ -187,37 +188,3 @@ class AwsBackend(StorageBackend):
         except Exception:
             pass
         return None
-
-    def _to_internal(self, url: str) -> str:
-        """Convert URL to internal path for s3fs operations."""
-        parsed = parse_url(url)
-        return f"{parsed.bucket}/{parsed.path}"
-
-    def _to_uri(self, internal_path: str) -> str:
-        """Convert internal path back to s3:// URL."""
-        return f"s3://{internal_path}"
-
-    def open(self, url: str) -> BinaryIO:
-        return self.fs.open(self._to_internal(url), "rb")
-
-    def list_files(self, url: str, extension: str) -> Iterator[FileInfo]:
-        internal_path = self._to_internal(url)
-        pattern = f"{internal_path}/**/*{extension}"
-        for path in sorted(self.fs.glob(pattern)):
-            info = self.fs.info(path)
-            yield FileInfo(url=self._to_uri(path), size=info["size"])
-
-    def size(self, url: str) -> int:
-        return self.fs.size(self._to_internal(url))
-
-    def is_directory(self, url: str) -> bool:
-        path = self._to_internal(url)
-        try:
-            info = self.fs.info(path)
-            return info.get("type") == "directory"
-        except FileNotFoundError:
-            try:
-                contents = self.fs.ls(path, detail=False)
-                return len(contents) > 0
-            except Exception:
-                return False

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/storage/az.py

@@ -1,10 +1,9 @@
 import os
 from enum import Enum
 
-from typing import Any, BinaryIO, Iterator
 from loguru import logger
 
-from tab_cli.storage import StorageBackend, FileInfo
+from tab_cli.storage.fsspec import CloudFsspecBackend
 from tab_cli.url_parser import parse_url
 
 
@@ -16,7 +15,7 @@ class AzAuthMethod(Enum):
     AZURE_CLI = 5
 
 
-class AzBackend(StorageBackend):
+class AzBackend(CloudFsspecBackend):
     """Storage backend for Azure Blob Storage with configurable URL interpretation.
 
     When az_url_authority_is_account=True:
@@ -27,6 +26,8 @@ class AzBackend(StorageBackend):
         - az://container/path - authority is the container name
     """
 
+    protocol = "az"
+
     def __init__(self, account: str | None, container: str | None, az_url_authority_is_account: bool = False) -> None:
         """Initialize the Azure Blob Storage backend.
 
@@ -97,6 +98,7 @@ class AzBackend(StorageBackend):
                 logger.debug("azure-identity not installed, skipping Azure AD / RBAC authentication")
             except Exception as e:
                 logger.debug(f"Azure AD / RBAC authentication failed: {e}")
+                self.fs = None  # Reset fs so we can try CLI fallback
 
         if self.fs is None:
             # 5. Fallback to fetching account key via Azure CLI
@@ -202,48 +204,9 @@ class AzBackend(StorageBackend):
                 "azure_use_azure_cli": "true",
             }
 
-    def _to_internal(self, url: str) -> str:
-        """Convert URL to (filesystem, internal_path) for adlfs operations."""
-        parsed = parse_url(url)
-        return f"{parsed.bucket}/{parsed.path}"
-
-    def _to_uri(self, account: str, internal_path: str) -> str:
+    def _to_uri(self, internal_path: str) -> str:
         """Convert internal path back to az:// URL."""
         if self.url_authority_is_account:
-            return f"az://{account}/{internal_path}"
+            return f"az://{self.account}/{internal_path}"
         else:
             return f"az://{internal_path}"
-
-    def open(self, url: str) -> BinaryIO:
-        return self.fs.open(self._to_internal(url), "rb")
-
-    def list_files(self, url: str, extension: str) -> Iterator[FileInfo]:
-        internal_path = self._to_internal(url)
-        pattern = f"{internal_path}/**/*{extension}"
-        for path in sorted(self.fs.glob(pattern)):
-            info = self.fs.info(path)
-            yield FileInfo(url=self._to_uri(self.account, path), size=info["size"])
-
-    def size(self, url: str) -> int:
-        return self.fs.size(self._to_internal(url))
-
-    def is_directory(self, url: str) -> bool:
-        path = self._to_internal(url)
-        try:
-            info = self.fs.info(path)
-            return info.get("type") == "directory"
-        except FileNotFoundError:
-            try:
-                contents = self.fs.ls(path, detail=False)
-                return len(contents) > 0
-            except Exception:
-                return False
-
-    def __del__(self):
-        try:
-            # Check if fs exists and has a close method
-            if hasattr(self, 'fs') and self.fs is not None:
-                self.fs.close()
-        except Exception:
-            # Silently fail as the interpreter is likely in mid-teardown
-            pass

tab_cli-0.1.2/tab_cli/storage/fsspec.py

@@ -0,0 +1,104 @@
+"""Cloud storage backend using fsspec.
+
+Fallback for any fsspec-supported protocol not handled by dedicated backends.
+
+Note: The following have dedicated backends:
+      - az://, abfs://, abfss:// -> AzBackend
+      - gs:// -> GcsBackend
+      - s3:// -> S3Backend
+
+The appropriate protocol handler package must be installed separately.
+"""
+
+from typing import BinaryIO, Iterator, Any
+import fsspec
+from loguru import logger
+
+from tab_cli.storage.base import FileInfo, StorageBackend
+from tab_cli.url_parser import parse_url
+
+
+class FsspecBackend(StorageBackend):
+    """Base class for fsspec-based storage backends.
+
+    This class provides common implementations for fsspec-compatible filesystems.
+    Subclasses can override _to_internal and _to_uri to customize path handling.
+    """
+
+    fs: fsspec.AbstractFileSystem
+    protocol: str
+
+    def __init__(self, protocol: str) -> None:
+        self.protocol = protocol
+
+        try:
+            self.fs = fsspec.filesystem(protocol)
+        except (ImportError, ValueError) as e:
+            raise ImportError(f"No handler found for {protocol}:// URLs") from e
+
+    def _to_internal(self, url: str) -> str:
+        """Convert URL to internal path for fsspec operations.
+
+        Default implementation returns the URL as-is.
+        Cloud backends override this to return bucket/path format.
+        """
+        return url
+
+    def _to_uri(self, internal_path: str) -> str:
+        """Convert internal path back to a URL.
+
+        Default implementation prefixes with protocol if needed.
+        """
+        if internal_path.startswith(f"{self.protocol}://"):
+            return internal_path
+        return f"{self.protocol}://{internal_path}"
+
+    def open(self, url: str) -> BinaryIO:
+        return self.fs.open(self._to_internal(url), "rb")
+
+    def list_files(self, url: str, extension: str) -> Iterator[FileInfo]:
+        internal_path = self._to_internal(url)
+        pattern = f"{internal_path}/**/*{extension}"
+        files = self.fs.glob(pattern)
+        logger.debug(f"{len(files)} files found.")
+        for path in sorted(files):
+            info = self.fs.info(path)
+            yield FileInfo(url=self._to_uri(path), size=info["size"])
+
+    def size(self, url: str) -> int:
+        return self.fs.size(self._to_internal(url))
+
+    def is_directory(self, url: str) -> bool:
+        path = self._to_internal(url)
+        try:
+            info = self.fs.info(path)
+            return info.get("type") == "directory"
+        except FileNotFoundError:
+            try:
+                contents = self.fs.ls(path, detail=False)
+                return len(contents) > 0
+            except Exception:
+                return False
+
+    def storage_options(self, url: str) -> dict[str, Any] | None:
+        return None
+
+
+class CloudFsspecBackend(FsspecBackend):
+    """Base class for cloud storage backends (S3, Azure, GCS).
+
+    Cloud backends use bucket/path format internally instead of full URLs.
+    """
+
+    def __init__(self) -> None:
+        # Subclasses must set self.fs and self.protocol before calling methods
+        pass
+
+    def _to_internal(self, url: str) -> str:
+        """Convert URL to internal path (bucket/path) for cloud fsspec operations."""
+        parsed = parse_url(url.rstrip("/"))  # strip trailing slash to avoid empty path segments
+        return f"{parsed.bucket}/{parsed.path}"
+
+    def _to_uri(self, internal_path: str) -> str:
+        """Convert internal path back to URL. Subclasses must implement."""
+        return f"{self.protocol}://{internal_path}"

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/storage/gcloud.py

@@ -2,10 +2,9 @@ import os
 from enum import Enum
 from pathlib import Path
 
-from typing import BinaryIO, Iterator
 from loguru import logger
 
-from tab_cli.storage.base import StorageBackend, FileInfo
+from tab_cli.storage.fsspec import CloudFsspecBackend
 from tab_cli.url_parser import parse_url
 
 
@@ -16,7 +15,7 @@ class GcloudAuthMethod(Enum):
     GOOGLE_DEFAULT = 4   # google.auth.default()
 
 
-class GcloudBackend(StorageBackend):
+class GcloudBackend(CloudFsspecBackend):
     """Storage backend for Google Cloud Storage.
 
     URL format: gs://bucket/path
@@ -28,6 +27,8 @@ class GcloudBackend(StorageBackend):
     4. gcloud CLI login credentials (gcloud auth print-access-token)
     """
 
+    protocol = "gs"
+
     def __init__(self) -> None:
         """Initialize the Google Cloud Storage backend."""
         try:
@@ -170,46 +171,3 @@ class GcloudBackend(StorageBackend):
             return {
                 "token": "google_default",
             }
-
-    def _to_internal(self, url: str) -> str:
-        """Convert URL to internal path for gcsfs operations."""
-        parsed = parse_url(url)
-        return f"{parsed.bucket}/{parsed.path}"
-
-    def _to_uri(self, internal_path: str) -> str:
-        """Convert internal path back to gs:// URL."""
-        return f"gs://{internal_path}"
-
-    def open(self, url: str) -> BinaryIO:
-        return self.fs.open(self._to_internal(url), "rb")
-
-    def list_files(self, url: str, extension: str) -> Iterator[FileInfo]:
-        internal_path = self._to_internal(url)
-        pattern = f"{internal_path}/**/*{extension}"
-        for path in sorted(self.fs.glob(pattern)):
-            info = self.fs.info(path)
-            yield FileInfo(url=self._to_uri(path), size=info["size"])
-
-    def size(self, url: str) -> int:
-        return self.fs.size(self._to_internal(url))
-
-    def is_directory(self, url: str) -> bool:
-        path = self._to_internal(url)
-        try:
-            info = self.fs.info(path)
-            return info.get("type") == "directory"
-        except FileNotFoundError:
-            try:
-                contents = self.fs.ls(path, detail=False)
-                return len(contents) > 0
-            except Exception:
-                return False
-
-    def __del__(self):
-        try:
-            if hasattr(self, 'fs') and self.fs is not None:
-                # gcsfs doesn't have a close method, but clear session if possible
-                if hasattr(self.fs, 'session') and self.fs.session:
-                    pass  # aiohttp session cleanup is handled by gcsfs
-        except Exception:
-            pass

{tab_cli-0.1.1 → tab_cli-0.1.2}/tab_cli/url_parser.py

@@ -40,6 +40,8 @@ def parse_url(url: str) -> ParsedUrl:
     Returns:
         ParsedUrl with scheme, bucket, account, and path components.
     """
+    # Strips trailing slashes to avoid empty path segments
+    url = url.rstrip("/")
     parsed = urlparse(url)
     scheme = parsed.scheme.lower() if parsed.scheme else ""
 

tab_cli-0.1.1/tab_cli/storage/fsspec.py

@@ -1,60 +0,0 @@
-"""Cloud storage backend using fsspec.
-
-Fallback for any fsspec-supported protocol not handled by dedicated backends.
-
-Note: The following have dedicated backends:
-      - az://, abfs://, abfss:// -> AzBackend
-      - gs:// -> GcsBackend
-      - s3:// -> S3Backend
-
-The appropriate protocol handler package must be installed separately.
-"""
-
-from typing import BinaryIO, Iterator, Any
-import fsspec
-
-from tab_cli.storage.base import FileInfo, StorageBackend
-
-# Package hints for common protocols
-_PACKAGE_HINTS: dict[str, str] = {}
-
-
-class FsspecBackend(StorageBackend):
-
-    def __init__(self, protocol: str) -> None:
-        self._protocol = protocol
-
-        try:
-            self._fs: fsspec.AbstractFileSystem = fsspec.filesystem(protocol)
-        except (ImportError, ValueError) as e:
-            pkg = _PACKAGE_HINTS.get(protocol)
-            if pkg is not None:
-                raise ImportError(f"Package '{pkg}' is required for {protocol}:// URLs. Install with: pip install {pkg}") from e
-            raise ImportError(f"No handler found for {protocol}:// URLs") from e
-
-    def open(self, url: str) -> BinaryIO:
-        return self._fs.open(url, "rb")
-
-    def list_files(self, url: str, extension: str) -> Iterator[FileInfo]:
-        pattern = f"{url}/**/*{extension}"
-        for path in sorted(self._fs.glob(pattern)):
-            info = self._fs.info(path)
-            full_uri = f"{self._protocol}://{path}" if not path.startswith(f"{self._protocol}://") else path
-            yield FileInfo(url=full_uri, size=info["size"])
-
-    def size(self, url: str) -> int:
-        return self._fs.size(url)
-
-    def is_directory(self, url: str) -> bool:
-        try:
-            info = self._fs.info(url)
-            return info.get("type") == "directory"
-        except FileNotFoundError:
-            try:
-                contents = self._fs.ls(url, detail=False)
-                return len(contents) > 0
-            except Exception:
-                return False
-
-    def storage_options(self, url: str) -> dict[str, Any] | None:
-        return None