chatsbom 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

chatsbom/commands/collect.py

@@ -159,6 +159,8 @@ class Storage:
         self.filepath = filepath
         self.visited_ids: set[int] = set()
         self.min_stars_seen: float = float('inf')
+        # Ensure parent directories exist
+        os.makedirs(os.path.dirname(self.filepath), exist_ok=True)
         self._load_existing()
 
     def _load_existing(self):

chatsbom/commands/convert.py

@@ -134,10 +134,10 @@ def convert_project(project_dir: Path, output_format: str, overwrite: bool) -> C
 
 def main(
     input_dir: str = typer.Option(
-        'data', help='Root data directory',
+        'data/sbom', help='Root data directory',
     ),
     concurrency: int = typer.Option(
-        4, help='Number of concurrent syft processes',
+        16, help='Number of concurrent syft processes',
     ),
     output_format: str = typer.Option(
         'json', '--format', help='Syft output format (json, spdx-json, cyclonedx-json)',

chatsbom/commands/download.py

@@ -163,10 +163,10 @@ def load_targets(jsonl_path: str) -> list[dict]:
 
 def main(
     input_file: str | None = typer.Option(
-        None, help='Input JSONL file path (default: {language}.jsonl)',
+        None, help='Input JSONL file path (default: data/github/{language}.jsonl)',
     ),
     output_dir: str = typer.Option(
-        'data', help='Download destination directory',
+        'data/sbom', help='Download destination directory',
     ),
     language: Language | None = typer.Option(
         None, help='Target Language (default: all)',
@@ -220,7 +220,7 @@ def main(
             if input_file:
                 target_file = input_file
             else:
-                target_file = f"{lang}.jsonl"
+                target_file = f"data/github/{lang}.jsonl"
 
             # Check if file exists, if not, skip efficiently
             if not os.path.exists(target_file):

chatsbom/commands/index.py

@@ -119,8 +119,8 @@ def scan_artifacts(meta_context: dict[str, Any]) -> list[list[Any]]:
     repo = meta_context['repo']
     repo_id = meta_context['id']
 
-    # Expected path: data/{language}/{owner}/{repo}/**/sbom.json
-    base_dir = Path('data') / language / owner / repo
+    # Expected path: data/sbom/{language}/{owner}/{repo}/**/sbom.json
+    base_dir = Path('data/sbom') / language / owner / repo
     if not base_dir.exists():
         return []
 
@@ -302,7 +302,7 @@ def main(
     else:
         langs_to_process = language if language else list(Language)
         for lang in langs_to_process:
-            f = Path(f"{lang.value}.jsonl")
+            f = Path(f"data/github/{lang.value}.jsonl")
             if f.exists():
                 files_to_process.append(f)
             else:

chatsbom/core/clickhouse.py

@@ -1,4 +1,6 @@
 """ClickHouse connection utilities."""
+import socket
+
 import clickhouse_connect
 import typer
 from rich.console import Console
@@ -14,85 +16,129 @@ def check_clickhouse_connection(
     require_database: bool = True,
 ) -> bool:
     """
-    Check ClickHouse connection and optionally verify database existence.
-
-    Args:
-        host: ClickHouse host
-        port: ClickHouse HTTP port
-        user: ClickHouse username
-        password: ClickHouse password
-        database: Database to check
-        console: Rich console for output (creates one if None)
-        require_database: If True, check that the database exists
-
-    Returns:
-        True if connection (and database) check passed, False otherwise
-
-    Raises:
-        typer.Exit: If connection fails
+    Check ClickHouse connection with multi-step validation.
+
+    Steps:
+        1. Network - is the server reachable?
+        2. Authentication - are credentials valid?
+        3. Database - does it exist and is it accessible?
+        4. Tables - do required tables exist?
     """
-    if console is None:
-        console = Console()
+    console = console or Console()
+
+    if not _check_network(host, port, console):
+        raise typer.Exit(1)
+
+    if not _check_auth(host, port, user, password, console):
+        raise typer.Exit(1)
+
+    if not require_database:
+        return True
+
+    if not _check_database(host, port, user, password, database, console):
+        raise typer.Exit(1)
+
+    if not _check_tables(host, port, user, password, database, console):
+        raise typer.Exit(1)
+
+    return True
 
-    # Step 1: Test basic connectivity
+
+def _check_network(host: str, port: int, console: Console) -> bool:
+    """Step 1: Check network connectivity."""
+    try:
+        with socket.create_connection((host, port), timeout=5):
+            return True
+    except TimeoutError:
+        console.print(
+            f'[bold red]Error:[/] Connection to [cyan]{host}:{port}[/] timed out.\n\n'
+            '[green]Solution:[/] [cyan]docker compose up -d[/]',
+        )
+    except OSError as e:
+        console.print(
+            f'[bold red]Error:[/] Cannot reach [cyan]{host}:{port}[/]\n'
+            f'[dim]{e}[/dim]\n\n'
+            '[green]Solution:[/] [cyan]docker compose up -d[/]',
+        )
+    return False
+
+
+def _check_auth(host: str, port: int, user: str, password: str, console: Console) -> bool:
+    """Step 2: Check authentication."""
     try:
         client = clickhouse_connect.get_client(
-            host=host,
-            port=port,
-            username=user,
-            password=password,
-            database='default',
+            host=host, port=port, username=user, password=password, database='default',
         )
         client.query('SELECT 1')
+        return True
     except Exception as e:
-        console.print(
-            f'[bold red]Error:[/] Failed to connect to ClickHouse at '
-            f'[cyan]{host}:{port}[/]\n\n'
-            f'Details: {e}\n\n'
-            'Please ensure ClickHouse is running:\n\n'
-            '[bold]Option 1:[/] Use docker compose\n'
-            '  [cyan]docker compose up -d[/]\n\n'
-            '[bold]Option 2:[/] Use docker run\n'
-            '  Step 1: Start ClickHouse\n'
-            '  [cyan]docker run --rm -d --name clickhouse \\\n'
-            '    -p 8123:8123 -p 9000:9000 \\\n'
-            '    -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \\\n'
-            '    -v ./database:/var/lib/clickhouse \\\n'
-            '    clickhouse/clickhouse-server[/]\n\n'
-            '  Step 2: Create admin and guest users\n'
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"CREATE USER IF NOT EXISTS admin IDENTIFIED BY 'admin'\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"CREATE USER IF NOT EXISTS guest IDENTIFIED BY 'guest'\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"GRANT CURRENT GRANTS ON *.* TO admin WITH GRANT OPTION\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"GRANT SELECT ON *.* TO guest\"[/]\n",
-        )
-        raise typer.Exit(1)
+        err = str(e).lower()
+        if any(x in err for x in ['authentication', 'password', 'denied', 'incorrect']):
+            console.print(
+                f'[bold red]Error:[/] Authentication failed for [cyan]{user}[/]\n\n'
+                '[green]Solution:[/] Create user:\n'
+                f'  [cyan]docker exec clickhouse clickhouse-client -q \\\n'
+                f'    "CREATE USER IF NOT EXISTS {user} IDENTIFIED BY \'<password>\'"[/]',
+            )
+        else:
+            console.print(f'[bold red]Error:[/] Auth failed: [dim]{e}[/dim]')
+        return False
+
 
-    # Step 2: Check database exists (if required)
-    if require_database:
-        try:
-            result = client.query(
-                'SELECT name FROM system.databases WHERE name = {db:String}',
-                parameters={'db': database},
+def _check_database(
+    host: str, port: int, user: str, password: str, database: str, console: Console,
+) -> bool:
+    """Step 3: Check database access."""
+    try:
+        client = clickhouse_connect.get_client(
+            host=host, port=port, username=user, password=password, database=database,
+        )
+        client.query('SELECT 1')
+        return True
+    except Exception as e:
+        err = str(e).lower()
+        if 'unknown database' in err:
+            console.print(
+                f'[bold red]Error:[/] Database [cyan]{database}[/] does not exist.\n\n'
+                '[green]Solution:[/] [cyan]chatsbom index --language go[/]',
             )
-            if not result.result_rows:
-                console.print(
-                    f'[bold red]Error:[/] Database [cyan]{database}[/] '
-                    'does not exist.\n\n'
-                    'Please run the import command first to create and '
-                    'populate the database:\n\n'
-                    '  [cyan]chatsbom index[/]',
-                )
-                raise typer.Exit(1)
-        except typer.Exit:
-            raise
-        except Exception as e:
+        elif any(x in err for x in ['access', 'denied', 'grant', 'not allowed']):
             console.print(
-                f'[bold red]Error:[/] Failed to check database: {e}',
+                f'[bold red]Error:[/] User [cyan]{user}[/] cannot access [cyan]{database}[/]\n\n'
+                '[green]Solution:[/] Grant access:\n'
+                f'  [cyan]docker exec clickhouse clickhouse-client -q \\\n'
+                f'    "GRANT SELECT ON {database}.* TO {user}"[/]\n\n'
+                '[dim]Or update config/clickhouse/users.d/guest.xml[/dim]',
             )
-            raise typer.Exit(1)
+        else:
+            console.print(
+                f'[bold red]Error:[/] Cannot access [cyan]{database}[/]: [dim]{e}[/dim]',
+            )
+        return False
 
-    return True
+
+def _check_tables(
+    host: str, port: int, user: str, password: str, database: str, console: Console,
+) -> bool:
+    """Step 4: Check required tables exist."""
+    required = {'repositories', 'artifacts'}
+
+    try:
+        client = clickhouse_connect.get_client(
+            host=host, port=port, username=user, password=password, database=database,
+        )
+        result = client.query('SHOW TABLES')
+        existing = {row[0] for row in result.result_rows}
+
+        if missing := required - existing:
+            console.print(
+                f'[bold red]Error:[/] Missing tables: [cyan]{", ".join(sorted(missing))}[/]\n\n'
+                '[green]Solution:[/] [cyan]chatsbom index --language go[/]',
+            )
+            return False
+        return True
+    except Exception as e:
+        console.print(
+            f'[bold red]Error:[/] Cannot check tables: [dim]{e}[/dim]',
+        )
+        return False

chatsbom/core/client.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from pathlib import Path
 
 import requests
 import requests_cache
@@ -10,7 +11,7 @@ logger = structlog.get_logger('client')
 
 
 def get_http_client(
-    cache_name: str = 'http_cache',
+    cache_name: str = 'data/http/cache.sqlite3',
     expire_after: int = 86400,  # 24 hours
     retries: int = 3,
     pool_size: int = 50,
@@ -19,6 +20,10 @@ def get_http_client(
     Returns a requests session with caching and retry logic.
     """
 
+    # Ensure the data directory exists
+    cache_path = Path(cache_name)
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
     # Configure Caching
     # We want to cache 200 OK and 404 Not Found (negative caching)
     session = requests_cache.CachedSession(

chatsbom/core/config.py

@@ -11,12 +11,12 @@ class PathConfig:
     """File path configuration."""
 
     # Base directories
-    data_dir: Path = field(default_factory=lambda: Path('data'))
+    data_dir: Path = field(default_factory=lambda: Path('data/sbom'))
     output_dir: Path = field(default_factory=lambda: Path('.'))
 
     # File naming conventions
     sbom_filename: str = 'sbom.json'
-    repo_list_pattern: str = '{language}.jsonl'
+    repo_list_pattern: str = 'data/github/{language}.jsonl'
 
     def get_repo_list_path(self, language: str) -> Path:
         """Get the path for repository list file."""

chatsbom/core/validation.py

@@ -64,7 +64,7 @@ def validate_download_structure(data_dir: Path, language: str) -> bool:
     Validate download directory structure.
 
     Expected structure:
-    data/{language}/{owner}/{repo}/{branch}/[files]
+    data/sbom/{language}/{owner}/{repo}/{branch}/[files]
 
     Returns:
         True if valid

{chatsbom-0.2.8.dist-info → chatsbom-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatsbom
-Version: 0.2.8
+Version: 0.2.9
 Summary: ChatSBOM - Talk to your Supply Chain. Chat with SBOMs.
 Requires-Python: >=3.12
 Requires-Dist: claude-agent-sdk>=0.1.0
@@ -95,6 +95,7 @@ ChatSBOM follows a clean, modular architecture with high cohesion and low coupli
 collect → download → convert → index → status/query/chat
    ↓         ↓          ↓         ↓
  .jsonl    files/    sbom.json  database
+(github/)  (sbom/)   (sbom/)    (clickhouse/)
 ```
 
 ### Core Modules

{chatsbom-0.2.8.dist-info → chatsbom-0.2.9.dist-info}/RECORD

@@ -3,23 +3,23 @@ chatsbom/__main__.py,sha256=QVs4sO1YUqJt642_Gw4wyiDBwQ5YeL4I9v_xEDkTPdM,1228
 chatsbom/__version__.py,sha256=JodF7HbCCwUPFhWltShEeNCAWHPTwRcPvSm7qWfMTCQ,411
 chatsbom/commands/__init__.py,sha256=vOL_QwoyFQhEIfs-rf0LVieAjStfkqlhqkjaZ_1p7gQ,15
 chatsbom/commands/chat.py,sha256=jmToyhbg_yoKgX-yRf60Gl3u4oy00A1BzQDO0ktVoi8,12835
-chatsbom/commands/collect.py,sha256=ApweQztO889o4X4lBrxlIwdYorDUSiE01M4-XMO0HUM,16536
-chatsbom/commands/convert.py,sha256=etc6s_tT4VAvjOCZs8dnHppbTYEeAHI6kHJikkUWD64,8249
-chatsbom/commands/download.py,sha256=I9TwEXO1zrh9-DwvUUMh4iZ1W5ZCXbtE4jcJiK2KqnY,10218
-chatsbom/commands/index.py,sha256=DlK5FtwOFoJqy9gA6mYvfuVQLL9S15KRrH7_Qw7l0Fk,10633
+chatsbom/commands/collect.py,sha256=dPTX0o9thBeVhiGVRJWrRYQSxo5gZZ72EI2UVCuvbH8,16645
+chatsbom/commands/convert.py,sha256=qIERQed1Lky2XZ0cjA8N9G3BHIChj_0-ogxUBWD1nyU,8255
+chatsbom/commands/download.py,sha256=4UCyQTaj2NBKBhyvsCkSuezW0kc6VFHQTI4hZLRMxug,10247
+chatsbom/commands/index.py,sha256=ycUBCdi01-l5fzgW0S67nhVdkpPyBiagmRUFy0VpQ-Q,10655
 chatsbom/commands/query.py,sha256=bIlz4vXClbcOkoHCLnjqmfcXpMrDzEPGNhmltZe5SHk,5709
 chatsbom/commands/status.py,sha256=AsesBz3F_Mz30n3By2ZMQ0P1kUJPEJkyBHqh3mx1qFA,8553
 chatsbom/core/__init__.py,sha256=x8qFrlBTgFn7TkmBN11ysPCUzRWBY350c5AN4u-3dWE,17
-chatsbom/core/clickhouse.py,sha256=sw4gWiaC2BE4QJlVWDsoDULKFB6YslB0W1FM630IY94,3530
-chatsbom/core/client.py,sha256=GSnOOAxoSUKU1pLXPX4W3cM5G4Iv7fQnknW-YkcX43Q,1400
-chatsbom/core/config.py,sha256=K4vDe387iVwwMY9aTDbDLlUPVOHCKLcA5ZMK000MQgk,3992
+chatsbom/core/clickhouse.py,sha256=rk2Hs6KfuAD2hYq-N6YV_Ufzrf49JtXGUiNk97JLXdk,4973
+chatsbom/core/client.py,sha256=gSiRbf-PEClu9KVoWNxS4L6WQlRuPpg-dyfBa57iSK4,1569
+chatsbom/core/config.py,sha256=wMD9oja8CBUNqwwOHUTFzbvS0bPzN6Fq7Hbi__ZsE-8,4009
 chatsbom/core/repository.py,sha256=0LqBwlal5PxkhnAcTwtP0eLIMNKCjAYQNKIybjUTAPs,9837
 chatsbom/core/schema.py,sha256=oZUNNjS3UerZtIMo9-KOL-2E3XwS8FT9JNQStWq4tYE,1267
-chatsbom/core/validation.py,sha256=v4boiht4hK73nuZNYpzLTkZKsNSoFPMUDIX-8sP2E9g,3927
+chatsbom/core/validation.py,sha256=m1123AzUFSVFAuqJO6y5j4Q3DWPCKj70ApBbI5yJ9rg,3932
 chatsbom/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chatsbom/models/framework.py,sha256=Whlsbnn64YwftMifMEMVvhsWADC9uupBV4QN1i1BoNQ,2833
 chatsbom/models/language.py,sha256=RgbBQZfG630uNZ7w-BtB1LZ5B5Q_nEExGrwUadfpRqE,3601
-chatsbom-0.2.8.dist-info/METADATA,sha256=ySw9ndHHKi56MpsCgDkF-C735wy6oME9-JDAlabA9F0,4115
-chatsbom-0.2.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-chatsbom-0.2.8.dist-info/entry_points.txt,sha256=906Ig6u2FwWk3ftNi4mth03N1NRgP4-B2p9kpppcqWA,51
-chatsbom-0.2.8.dist-info/RECORD,,
+chatsbom-0.2.9.dist-info/METADATA,sha256=zuW5BOx3dige_d1dKTNtr1XnS23PLw-1bz2ofPewYz8,4161
+chatsbom-0.2.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+chatsbom-0.2.9.dist-info/entry_points.txt,sha256=906Ig6u2FwWk3ftNi4mth03N1NRgP4-B2p9kpppcqWA,51
+chatsbom-0.2.9.dist-info/RECORD,,