cloudcat 0.6.1__tar.gz → 0.6.2__tar.gz

{cloudcat-0.6.1 → cloudcat-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.6.1
+Version: 0.6.2
 Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Data Lake Storage Gen2 from your terminal
 Author: Jonathan Sudhakar
 License: MIT
@@ -554,6 +554,10 @@ Options:
 
   --config-profile TEXT        Named profile from ~/.config/cloudcat/config.toml
 
+  --install-completion [bash|zsh|fish]
+                               Enable completion with one command (adds one
+                               line to your shell rc file; idempotent)
+
   --completion [bash|zsh|fish] Print the shell completion script and exit.
                                Once enabled, TAB completes PATH too: local
                                files natively, s3://<TAB> lists buckets,

{cloudcat-0.6.1 → cloudcat-0.6.2}/README.md

@@ -472,6 +472,10 @@ Options:
 
   --config-profile TEXT        Named profile from ~/.config/cloudcat/config.toml
 
+  --install-completion [bash|zsh|fish]
+                               Enable completion with one command (adds one
+                               line to your shell rc file; idempotent)
+
   --completion [bash|zsh|fish] Print the shell completion script and exit.
                                Once enabled, TAB completes PATH too: local
                                files natively, s3://<TAB> lists buckets,

{cloudcat-0.6.1 → cloudcat-0.6.2}/cloudcat/__init__.py

@@ -2,4 +2,4 @@
 #!/usr/bin/env python
 """cloudcat - A CLI utility to read and display files from cloud storage."""
 
-__version__ = "0.6.1"
+__version__ = "0.6.2"

{cloudcat-0.6.1 → cloudcat-0.6.2}/cloudcat/cli.py

@@ -202,6 +202,35 @@ def get_files_for_multiread(
     return selected_files
 
 
+_ICEBERG_NESTED_RE = re.compile(r'(.*)/metadata/(?:v\d+|\d+-[^/]*)\.metadata\.json$')
+
+
+def _find_nested_tables(files: List[Tuple[str, int]]) -> dict:
+    """Find lakehouse table roots nested inside a directory listing.
+
+    Returns {root_path: format}. Used to guide users who point cloudcat at
+    a folder OF tables rather than at a table — merging data files across
+    different tables would silently produce wrong results.
+    """
+    tables = {}
+    for name, _size in files:
+        if '/_delta_log/' in f'/{name}':
+            root = name.split('/_delta_log/')[0]
+            if root:  # empty means the listing prefix IS the table root
+                tables.setdefault(root, 'delta')
+            continue
+        match = _ICEBERG_NESTED_RE.match(name)
+        if match and match.group(1):
+            tables.setdefault(match.group(1), 'iceberg')
+    return tables
+
+
+def _has_known_extension(name: str) -> bool:
+    """True if the filename matches any recognized data-format extension."""
+    return any(re.search(regex, name, re.IGNORECASE)
+               for regex in FORMAT_EXTENSION_MAP.values())
+
+
 def find_first_non_empty_file(
     service: str,
     bucket: str,
@@ -239,7 +268,40 @@ def find_first_non_empty_file(
         info(Fore.YELLOW + f"No files matching format '{input_format}' found in "
              f"{service}://{bucket}/{prefix}. Using first available file." + Style.RESET_ALL)
 
-    # Use the first non-empty, non-metadata file
+    # No explicit format: the selection drives format INFERENCE, so only
+    # consider files whose extension we can actually infer from.
+    if not input_format:
+        nested_tables = _find_nested_tables(non_empty_files)
+        if nested_tables:
+            listing = '\n'.join(
+                f"  {fmt}: {root}/" for root, fmt in sorted(nested_tables.items())
+            )
+            first_root = sorted(nested_tables)[0]
+            raise ValueError(
+                "this directory contains lakehouse table(s), not a single dataset:\n"
+                f"{listing}\n"
+                f"Point at a table root, e.g.: cloudcat {first_root}/ "
+                "(or pass --input-format to force reading loose files)"
+            )
+
+        for file_name, file_size in non_empty_files:
+            if any(re.search(pattern, file_name) for pattern in SKIP_PATTERNS):
+                continue
+            if not _has_known_extension(file_name):
+                continue  # e.g. catalog.db, .lock — can't infer a format from it
+            if not quiet:
+                info(Fore.BLUE + f"Selected file: {file_name} ({file_size} bytes)" + Style.RESET_ALL)
+            return file_name, file_size
+
+        sample = ', '.join(name.rsplit('/', 1)[-1] for name, _ in non_empty_files[:5])
+        raise ValueError(
+            f"no files with a recognized data format in {service}://{bucket}/{prefix} "
+            f"(found: {sample}). Formats are inferred from extensions "
+            "(.csv/.json/.parquet/.avro/.orc/.txt, optionally compressed); "
+            "pass --input-format to force one."
+        )
+
+    # Explicit format requested but unmatched: first non-metadata file.
     for file_name, file_size in non_empty_files:
         if not any(re.search(pattern, file_name) for pattern in SKIP_PATTERNS):
             if not quiet:
@@ -850,6 +912,44 @@ def _print_completion(ctx, param, value):
     ctx.exit()
 
 
+_COMPLETION_SNIPPETS = {
+    'zsh': ('~/.zshrc', 'eval "$(cloudcat --completion zsh)"'),
+    'bash': ('~/.bashrc',
+             'COMP_WORDBREAKS=${COMP_WORDBREAKS//:/}  # let URLs like s3:// complete\n'
+             'eval "$(cloudcat --completion bash)"'),
+    'fish': ('~/.config/fish/config.fish', 'cloudcat --completion fish | source'),
+}
+
+_COMPLETION_MARKER = '# added by cloudcat --install-completion'
+
+
+def _install_completion(ctx, param, value):
+    """Eager callback: enable shell completion in the user's rc file.
+
+    Idempotent — a marker comment prevents duplicate blocks on re-runs.
+    """
+    if not value or ctx.resilient_parsing:
+        return
+    rc_path, snippet = _COMPLETION_SNIPPETS[value]
+    rc_file = os.path.expanduser(rc_path)
+
+    existing = ''
+    if os.path.exists(rc_file):
+        with open(rc_file, 'r', encoding='utf-8', errors='replace') as f:
+            existing = f.read()
+    if _COMPLETION_MARKER in existing:
+        click.echo(f"Completion is already set up in {rc_file}")
+        ctx.exit()
+
+    os.makedirs(os.path.dirname(rc_file), exist_ok=True)
+    block = f"\n{_COMPLETION_MARKER}\n{snippet}\n"
+    with open(rc_file, 'a', encoding='utf-8') as f:
+        f.write(block)
+    click.echo(f"Added cloudcat completion to {rc_file} — restart your shell "
+               f"(or run: source {rc_path}) and try: cloudcat s3://<TAB>")
+    ctx.exit()
+
+
 # Where each agent expects personal skills (name -> path builder).
 _SKILL_DESTINATIONS = {
     'claude': lambda: os.path.join(os.path.expanduser('~'), '.claude', 'skills', 'cloudcat', 'SKILL.md'),
@@ -962,6 +1062,9 @@ def _render_data(df, output_format: str) -> str:
 @click.option('--completion', type=click.Choice(['bash', 'zsh', 'fish']), is_eager=True,
               expose_value=False, callback=_print_completion,
               help='Print the shell completion script and exit (e.g. eval "$(cloudcat --completion zsh)")')
+@click.option('--install-completion', type=click.Choice(['bash', 'zsh', 'fish']), is_eager=True,
+              expose_value=False, callback=_install_completion,
+              help='Enable shell completion by adding one line to your shell rc file (idempotent)')
 @click.option('--install-skill', type=click.Choice(['claude', 'claude-project', 'codex', 'print']),
               is_eager=True, expose_value=False, callback=_install_skill,
               help='Install the bundled AI-agent skill: claude (~/.claude/skills), '

{cloudcat-0.6.1 → cloudcat-0.6.2}/cloudcat/completion.py

@@ -15,7 +15,51 @@ Behavior mirrors `ls` completion, scheme-aware:
 
 from typing import List, Optional, Tuple
 
-from click.shell_completion import CompletionItem
+import os
+
+from click.shell_completion import CompletionItem, ZshComplete, add_completion_class
+
+
+def _debug(message: str) -> None:
+    """Optionally record why a completion produced nothing.
+
+    Completion swallows every error so it can never break the shell — which
+    makes "TAB does nothing" undiagnosable. Set CLOUDCAT_COMPLETE_DEBUG=1 and
+    failures are appended to ~/.cache/cloudcat/completion.log instead.
+    """
+    if not os.environ.get('CLOUDCAT_COMPLETE_DEBUG'):
+        return
+    try:
+        import datetime
+        cache_dir = os.path.join(
+            os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')), 'cloudcat')
+        os.makedirs(cache_dir, exist_ok=True)
+        with open(os.path.join(cache_dir, 'completion.log'), 'a', encoding='utf-8') as f:
+            f.write(f"{datetime.datetime.now().isoformat()} {message}\n")
+    except Exception:
+        pass
+
+
+class CloudcatZshComplete(ZshComplete):
+    """Click's zsh completer, hardened for real-world .zshrc files.
+
+    The stock script calls compdef unconditionally; if the user evals it
+    before compinit has run, registration fails silently and TAB "does
+    nothing". This variant bootstraps compinit when needed.
+    """
+
+    name = 'zsh'
+    source_template = ZshComplete.source_template.replace(
+        "    compdef %(complete_func)s %(prog_name)s",
+        """    if ! typeset -f compdef >/dev/null 2>&1; then
+        autoload -Uz compinit && compinit -u
+    fi
+    compdef %(complete_func)s %(prog_name)s""",
+    )
+
+
+# Replace the registered zsh completer with the hardened variant.
+add_completion_class(CloudcatZshComplete)
 
 # Never return more candidates than a shell menu can usefully show.
 LIMIT = 100
@@ -36,7 +80,10 @@ def _s3_client(profile: Optional[str]):
     config = Config(connect_timeout=CONNECT_TIMEOUT, read_timeout=READ_TIMEOUT,
                     retries={'max_attempts': 0})
     session = boto3.Session(profile_name=profile) if profile else boto3.Session()
-    return session.client('s3', config=config)
+    # Without a configured region some boto3 versions raise NoRegionError at
+    # client creation; bucket listing is region-agnostic, so default it.
+    region = session.region_name or 'us-east-1'
+    return session.client('s3', region_name=region, config=config)
 
 
 def _list_s3_buckets(profile: Optional[str]) -> List[str]:
@@ -196,4 +243,6 @@ def complete_path(ctx, param, incomplete: str) -> List[CompletionItem]:
         return items
     except Exception:
         # Completion must never break the shell: no candidates beats a stack.
+        import traceback
+        _debug(f"complete_path({incomplete!r}) failed:\n{traceback.format_exc()}")
         return []

{cloudcat-0.6.1 → cloudcat-0.6.2}/cloudcat.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudcat
-Version: 0.6.1
+Version: 0.6.2
 Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Data Lake Storage Gen2 from your terminal
 Author: Jonathan Sudhakar
 License: MIT
@@ -554,6 +554,10 @@ Options:
 
   --config-profile TEXT        Named profile from ~/.config/cloudcat/config.toml
 
+  --install-completion [bash|zsh|fish]
+                               Enable completion with one command (adds one
+                               line to your shell rc file; idempotent)
+
   --completion [bash|zsh|fish] Print the shell completion script and exit.
                                Once enabled, TAB completes PATH too: local
                                files natively, s3://<TAB> lists buckets,

{cloudcat-0.6.1 → cloudcat-0.6.2}/cloudcat.egg-info/SOURCES.txt

@@ -62,4 +62,5 @@ tests/test_streaming_stats.py
 tests/test_tables.py
 tests/test_tracking.py
 tests/test_v05_features.py
+tests/test_v062_fixes.py
 tests/test_visual_output.py

cloudcat-0.6.2/tests/test_v062_fixes.py

@@ -0,0 +1,155 @@
+"""Tests for the 0.6.2 fixes: mixed-directory inference and completion setup."""
+
+import os
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from cloudcat.cli import main, _find_nested_tables
+import cloudcat.completion as completion
+
+runner = CliRunner()
+
+
+# --------------------------------------------------------------------------
+# directory inference (bug 1)
+# --------------------------------------------------------------------------
+
+class TestNestedTableDetection:
+    def test_finds_delta_and_iceberg_roots(self):
+        files = [
+            ("/lake/orders_delta/_delta_log/000.json", 10),
+            ("/lake/orders_delta/part-0.parquet", 10),
+            ("/lake/shop/orders/metadata/00001-abc.metadata.json", 10),
+            ("/lake/shop/orders/data/f.parquet", 10),
+            ("/lake/catalog.db", 10),
+        ]
+        tables = _find_nested_tables(files)
+        assert tables == {
+            "/lake/orders_delta": "delta",
+            "/lake/shop/orders": "iceberg",
+        }
+
+    def test_hadoop_style_iceberg_metadata(self):
+        files = [("/w/t/metadata/v3.metadata.json", 5)]
+        assert _find_nested_tables(files) == {"/w/t": "iceberg"}
+
+    def test_plain_files_are_not_tables(self):
+        files = [("/data/a.parquet", 5), ("/data/metadata/notes.txt", 5)]
+        assert _find_nested_tables(files) == {}
+
+
+class TestMixedDirectoryCli:
+    def test_directory_of_tables_gets_guidance(self, tmp_path):
+        deltalake = pytest.importorskip("deltalake")
+        import pandas as pd
+        from deltalake import write_deltalake
+        write_deltalake(str(tmp_path / "orders_delta"),
+                        pd.DataFrame({"a": [1, 2]}))
+        # a second, fake iceberg table (detection only needs the marker file)
+        meta = tmp_path / "shop" / "orders" / "metadata"
+        meta.mkdir(parents=True)
+        (meta / "00001-abc.metadata.json").write_text("{}")
+        (tmp_path / "catalog.db").write_text("sqlite")
+
+        res = runner.invoke(main, [str(tmp_path) + "/", "--no-color"])
+        assert res.exit_code == 1
+        assert "contains lakehouse table(s)" in res.output
+        assert "orders_delta" in res.output
+        assert "shop/orders" in res.output
+        assert "Point at a table root" in res.output
+
+    def test_unknown_extensions_are_skipped_for_inference(self, tmp_path):
+        (tmp_path / "catalog.db").write_text("junk")
+        (tmp_path / "data.csv").write_text("a\n1\n2\n")
+        res = runner.invoke(main, [str(tmp_path) + "/", "--no-color",
+                                   "-s", "dont_show", "-o", "csv"])
+        assert res.exit_code == 0
+        assert "1" in res.stdout and "2" in res.stdout
+
+    def test_only_unknown_extensions_gives_clear_error(self, tmp_path):
+        (tmp_path / "catalog.db").write_text("junk")
+        (tmp_path / "state.lock").write_text("x")
+        res = runner.invoke(main, [str(tmp_path) + "/", "--no-color"])
+        assert res.exit_code == 1
+        assert "no files with a recognized data format" in res.output
+        assert "catalog.db" in res.output
+        assert "--input-format" in res.output
+
+    def test_explicit_format_still_forces_reading(self, tmp_path):
+        (tmp_path / "catalog.db").write_text("junk")
+        (tmp_path / "data.csv").write_text("a\n1\n")
+        res = runner.invoke(main, [str(tmp_path) + "/", "--no-color", "-i", "csv",
+                                   "-s", "dont_show", "-o", "csv"])
+        assert res.exit_code == 0
+
+
+# --------------------------------------------------------------------------
+# completion hardening (bug 2)
+# --------------------------------------------------------------------------
+
+class TestS3RegionFallback:
+    def test_client_gets_a_region_when_none_configured(self, monkeypatch, tmp_path):
+        for var in ("AWS_DEFAULT_REGION", "AWS_REGION", "AWS_PROFILE"):
+            monkeypatch.delenv(var, raising=False)
+        monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "none"))
+        monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(tmp_path / "none"))
+        client = completion._s3_client(None)
+        assert client.meta.region_name == "us-east-1"
+
+
+class TestZshHardening:
+    def test_emitted_script_bootstraps_compinit(self):
+        res = runner.invoke(main, ["--completion", "zsh"])
+        assert res.exit_code == 0
+        assert "typeset -f compdef" in res.stdout
+        assert "compinit" in res.stdout
+
+
+class TestInstallCompletion:
+    def test_zsh_install_is_idempotent(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("HOME", str(tmp_path))
+        first = runner.invoke(main, ["--install-completion", "zsh"])
+        second = runner.invoke(main, ["--install-completion", "zsh"])
+        assert first.exit_code == 0 and second.exit_code == 0
+        content = (tmp_path / ".zshrc").read_text()
+        assert content.count('eval "$(cloudcat --completion zsh)"') == 1
+        assert "already set up" in second.output
+
+    def test_bash_install_includes_wordbreaks_fix(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("HOME", str(tmp_path))
+        res = runner.invoke(main, ["--install-completion", "bash"])
+        assert res.exit_code == 0
+        content = (tmp_path / ".bashrc").read_text()
+        assert "COMP_WORDBREAKS" in content
+        assert 'eval "$(cloudcat --completion bash)"' in content
+
+    def test_fish_install_creates_config_dir(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("HOME", str(tmp_path))
+        res = runner.invoke(main, ["--install-completion", "fish"])
+        assert res.exit_code == 0
+        assert "cloudcat --completion fish | source" in \
+            (tmp_path / ".config" / "fish" / "config.fish").read_text()
+
+
+class TestCompletionDebugLog:
+    def test_failures_are_logged_when_enabled(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("CLOUDCAT_COMPLETE_DEBUG", "1")
+        monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path))
+        from click.shell_completion import ShellComplete
+        with patch.object(completion, "_list_s3_buckets", side_effect=RuntimeError("boom")):
+            sc = ShellComplete(main, {}, "cloudcat", "_CLOUDCAT_COMPLETE")
+            assert sc.get_completions([], "s3://") == []
+        log = tmp_path / "cloudcat" / "completion.log"
+        assert log.exists()
+        assert "boom" in log.read_text()
+
+    def test_silent_by_default(self, tmp_path, monkeypatch):
+        monkeypatch.delenv("CLOUDCAT_COMPLETE_DEBUG", raising=False)
+        monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path))
+        from click.shell_completion import ShellComplete
+        with patch.object(completion, "_list_s3_buckets", side_effect=RuntimeError("boom")):
+            sc = ShellComplete(main, {}, "cloudcat", "_CLOUDCAT_COMPLETE")
+            sc.get_completions([], "s3://")
+        assert not (tmp_path / "cloudcat" / "completion.log").exists()