chatsbom 0.2.8__tar.gz → 0.2.9__tar.gz

{chatsbom-0.2.8 → chatsbom-0.2.9}/.gitignore

@@ -221,11 +221,5 @@ __marimo__/
 *.sqlite-shm
 *.sqlite-wal
 
-# Data folder
+# Runtime data (sbom files, github repo lists, clickhouse storage)
 /data/
-
-# JSONL folder
-/*.jsonl
-
-# Database folder
-/database/

{chatsbom-0.2.8 → chatsbom-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatsbom
-Version: 0.2.8
+Version: 0.2.9
 Summary: ChatSBOM - Talk to your Supply Chain. Chat with SBOMs.
 Requires-Python: >=3.12
 Requires-Dist: claude-agent-sdk>=0.1.0
@@ -95,6 +95,7 @@ ChatSBOM follows a clean, modular architecture with high cohesion and low coupli
 collect → download → convert → index → status/query/chat
    ↓         ↓          ↓         ↓
  .jsonl    files/    sbom.json  database
+(github/)  (sbom/)   (sbom/)    (clickhouse/)
 ```
 
 ### Core Modules

{chatsbom-0.2.8 → chatsbom-0.2.9}/README.md

@@ -73,6 +73,7 @@ ChatSBOM follows a clean, modular architecture with high cohesion and low coupli
 collect → download → convert → index → status/query/chat
    ↓         ↓          ↓         ↓
  .jsonl    files/    sbom.json  database
+(github/)  (sbom/)   (sbom/)    (clickhouse/)
 ```
 
 ### Core Modules

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/commands/collect.py

@@ -159,6 +159,8 @@ class Storage:
         self.filepath = filepath
         self.visited_ids: set[int] = set()
         self.min_stars_seen: float = float('inf')
+        # Ensure parent directories exist
+        os.makedirs(os.path.dirname(self.filepath), exist_ok=True)
         self._load_existing()
 
     def _load_existing(self):

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/commands/convert.py

@@ -134,10 +134,10 @@ def convert_project(project_dir: Path, output_format: str, overwrite: bool) -> C
 
 def main(
     input_dir: str = typer.Option(
-        'data', help='Root data directory',
+        'data/sbom', help='Root data directory',
     ),
     concurrency: int = typer.Option(
-        4, help='Number of concurrent syft processes',
+        16, help='Number of concurrent syft processes',
     ),
     output_format: str = typer.Option(
         'json', '--format', help='Syft output format (json, spdx-json, cyclonedx-json)',

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/commands/download.py

@@ -163,10 +163,10 @@ def load_targets(jsonl_path: str) -> list[dict]:
 
 def main(
     input_file: str | None = typer.Option(
-        None, help='Input JSONL file path (default: {language}.jsonl)',
+        None, help='Input JSONL file path (default: data/github/{language}.jsonl)',
     ),
     output_dir: str = typer.Option(
-        'data', help='Download destination directory',
+        'data/sbom', help='Download destination directory',
     ),
     language: Language | None = typer.Option(
         None, help='Target Language (default: all)',
@@ -220,7 +220,7 @@ def main(
             if input_file:
                 target_file = input_file
             else:
-                target_file = f"{lang}.jsonl"
+                target_file = f"data/github/{lang}.jsonl"
 
             # Check if file exists, if not, skip efficiently
             if not os.path.exists(target_file):

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/commands/index.py

@@ -119,8 +119,8 @@ def scan_artifacts(meta_context: dict[str, Any]) -> list[list[Any]]:
     repo = meta_context['repo']
     repo_id = meta_context['id']
 
-    # Expected path: data/{language}/{owner}/{repo}/**/sbom.json
-    base_dir = Path('data') / language / owner / repo
+    # Expected path: data/sbom/{language}/{owner}/{repo}/**/sbom.json
+    base_dir = Path('data/sbom') / language / owner / repo
     if not base_dir.exists():
         return []
 
@@ -302,7 +302,7 @@ def main(
     else:
         langs_to_process = language if language else list(Language)
         for lang in langs_to_process:
-            f = Path(f"{lang.value}.jsonl")
+            f = Path(f"data/github/{lang.value}.jsonl")
             if f.exists():
                 files_to_process.append(f)
             else:

chatsbom-0.2.9/chatsbom/core/clickhouse.py

@@ -0,0 +1,144 @@
+"""ClickHouse connection utilities."""
+import socket
+
+import clickhouse_connect
+import typer
+from rich.console import Console
+
+
+def check_clickhouse_connection(
+    host: str,
+    port: int,
+    user: str,
+    password: str,
+    database: str = 'chatsbom',
+    console: Console | None = None,
+    require_database: bool = True,
+) -> bool:
+    """
+    Check ClickHouse connection with multi-step validation.
+
+    Steps:
+        1. Network - is the server reachable?
+        2. Authentication - are credentials valid?
+        3. Database - does it exist and is it accessible?
+        4. Tables - do required tables exist?
+    """
+    console = console or Console()
+
+    if not _check_network(host, port, console):
+        raise typer.Exit(1)
+
+    if not _check_auth(host, port, user, password, console):
+        raise typer.Exit(1)
+
+    if not require_database:
+        return True
+
+    if not _check_database(host, port, user, password, database, console):
+        raise typer.Exit(1)
+
+    if not _check_tables(host, port, user, password, database, console):
+        raise typer.Exit(1)
+
+    return True
+
+
+def _check_network(host: str, port: int, console: Console) -> bool:
+    """Step 1: Check network connectivity."""
+    try:
+        with socket.create_connection((host, port), timeout=5):
+            return True
+    except TimeoutError:
+        console.print(
+            f'[bold red]Error:[/] Connection to [cyan]{host}:{port}[/] timed out.\n\n'
+            '[green]Solution:[/] [cyan]docker compose up -d[/]',
+        )
+    except OSError as e:
+        console.print(
+            f'[bold red]Error:[/] Cannot reach [cyan]{host}:{port}[/]\n'
+            f'[dim]{e}[/dim]\n\n'
+            '[green]Solution:[/] [cyan]docker compose up -d[/]',
+        )
+    return False
+
+
+def _check_auth(host: str, port: int, user: str, password: str, console: Console) -> bool:
+    """Step 2: Check authentication."""
+    try:
+        client = clickhouse_connect.get_client(
+            host=host, port=port, username=user, password=password, database='default',
+        )
+        client.query('SELECT 1')
+        return True
+    except Exception as e:
+        err = str(e).lower()
+        if any(x in err for x in ['authentication', 'password', 'denied', 'incorrect']):
+            console.print(
+                f'[bold red]Error:[/] Authentication failed for [cyan]{user}[/]\n\n'
+                '[green]Solution:[/] Create user:\n'
+                f'  [cyan]docker exec clickhouse clickhouse-client -q \\\n'
+                f'    "CREATE USER IF NOT EXISTS {user} IDENTIFIED BY \'<password>\'"[/]',
+            )
+        else:
+            console.print(f'[bold red]Error:[/] Auth failed: [dim]{e}[/dim]')
+        return False
+
+
+def _check_database(
+    host: str, port: int, user: str, password: str, database: str, console: Console,
+) -> bool:
+    """Step 3: Check database access."""
+    try:
+        client = clickhouse_connect.get_client(
+            host=host, port=port, username=user, password=password, database=database,
+        )
+        client.query('SELECT 1')
+        return True
+    except Exception as e:
+        err = str(e).lower()
+        if 'unknown database' in err:
+            console.print(
+                f'[bold red]Error:[/] Database [cyan]{database}[/] does not exist.\n\n'
+                '[green]Solution:[/] [cyan]chatsbom index --language go[/]',
+            )
+        elif any(x in err for x in ['access', 'denied', 'grant', 'not allowed']):
+            console.print(
+                f'[bold red]Error:[/] User [cyan]{user}[/] cannot access [cyan]{database}[/]\n\n'
+                '[green]Solution:[/] Grant access:\n'
+                f'  [cyan]docker exec clickhouse clickhouse-client -q \\\n'
+                f'    "GRANT SELECT ON {database}.* TO {user}"[/]\n\n'
+                '[dim]Or update config/clickhouse/users.d/guest.xml[/dim]',
+            )
+        else:
+            console.print(
+                f'[bold red]Error:[/] Cannot access [cyan]{database}[/]: [dim]{e}[/dim]',
+            )
+        return False
+
+
+def _check_tables(
+    host: str, port: int, user: str, password: str, database: str, console: Console,
+) -> bool:
+    """Step 4: Check required tables exist."""
+    required = {'repositories', 'artifacts'}
+
+    try:
+        client = clickhouse_connect.get_client(
+            host=host, port=port, username=user, password=password, database=database,
+        )
+        result = client.query('SHOW TABLES')
+        existing = {row[0] for row in result.result_rows}
+
+        if missing := required - existing:
+            console.print(
+                f'[bold red]Error:[/] Missing tables: [cyan]{", ".join(sorted(missing))}[/]\n\n'
+                '[green]Solution:[/] [cyan]chatsbom index --language go[/]',
+            )
+            return False
+        return True
+    except Exception as e:
+        console.print(
+            f'[bold red]Error:[/] Cannot check tables: [dim]{e}[/dim]',
+        )
+        return False

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/core/client.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from pathlib import Path
 
 import requests
 import requests_cache
@@ -10,7 +11,7 @@ logger = structlog.get_logger('client')
 
 
 def get_http_client(
-    cache_name: str = 'http_cache',
+    cache_name: str = 'data/http/cache.sqlite3',
     expire_after: int = 86400,  # 24 hours
     retries: int = 3,
     pool_size: int = 50,
@@ -19,6 +20,10 @@ def get_http_client(
     Returns a requests session with caching and retry logic.
     """
 
+    # Ensure the data directory exists
+    cache_path = Path(cache_name)
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
     # Configure Caching
     # We want to cache 200 OK and 404 Not Found (negative caching)
     session = requests_cache.CachedSession(

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/core/config.py

@@ -11,12 +11,12 @@ class PathConfig:
     """File path configuration."""
 
     # Base directories
-    data_dir: Path = field(default_factory=lambda: Path('data'))
+    data_dir: Path = field(default_factory=lambda: Path('data/sbom'))
     output_dir: Path = field(default_factory=lambda: Path('.'))
 
     # File naming conventions
     sbom_filename: str = 'sbom.json'
-    repo_list_pattern: str = '{language}.jsonl'
+    repo_list_pattern: str = 'data/github/{language}.jsonl'
 
     def get_repo_list_path(self, language: str) -> Path:
         """Get the path for repository list file."""

{chatsbom-0.2.8 → chatsbom-0.2.9}/chatsbom/core/validation.py

@@ -64,7 +64,7 @@ def validate_download_structure(data_dir: Path, language: str) -> bool:
     Validate download directory structure.
 
     Expected structure:
-    data/{language}/{owner}/{repo}/{branch}/[files]
+    data/sbom/{language}/{owner}/{repo}/{branch}/[files]
 
     Returns:
         True if valid

{chatsbom-0.2.8/config → chatsbom-0.2.9/config/clickhouse}/users.d/guest.xml

@@ -4,7 +4,7 @@
             <password>guest</password>
             <profile>readonly</profile>
             <allow_databases>
-                <database>sbom</database>
+                <database>chatsbom</database>
             </allow_databases>
         </guest>
     </users>

{chatsbom-0.2.8 → chatsbom-0.2.9}/docker-compose.yaml

@@ -5,8 +5,8 @@ services:
       - "8123:8123"
       - "9000:9000"
     volumes:
-      - ./database:/var/lib/clickhouse
-      - ./config/users.d:/etc/clickhouse-server/users.d
+      - ./data/clickhouse:/var/lib/clickhouse
+      - ./config/clickhouse/users.d:/etc/clickhouse-server/users.d
     environment:
       CLICKHOUSE_DB: sbom
     ulimits:

{chatsbom-0.2.8 → chatsbom-0.2.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "chatsbom"
-version = "0.2.8"
+version = "0.2.9"
 description = "ChatSBOM - Talk to your Supply Chain. Chat with SBOMs."
 readme = "README.md"
 requires-python = ">=3.12"

{chatsbom-0.2.8 → chatsbom-0.2.9}/test.sh

@@ -1,20 +1,20 @@
 # 1. Collect repository links from GitHub (e.g., top Go repos)
-uv run python -m chatsbom collect --language go --min-stars 10000
+uv run python -m chatsbom collect --min-stars 1000
 
 # 2. Download dependency files
-uv run python -m chatsbom download --language go
+uv run python -m chatsbom download
 
 # 3. Convert to standard SBOM format
-uv run python -m chatsbom convert --language go
+uv run python -m chatsbom convert
 
 # 4. Index SBOM data into database
-uv run python -m chatsbom index --language go
+uv run python -m chatsbom index
 
 # 5. Show database statistics
 uv run python -m chatsbom status
 
 # 6. Query dependencies
-uv run python -m chatsbom query gin --language go
+uv run python -m chatsbom query gin
 
 # 7. Launch AI chat interface
 uv run python -m chatsbom chat

{chatsbom-0.2.8 → chatsbom-0.2.9}/uv.lock

@@ -179,7 +179,7 @@ wheels = [
 
 [[package]]
 name = "chatsbom"
-version = "0.2.7"
+version = "0.2.8"
 source = { editable = "." }
 dependencies = [
     { name = "claude-agent-sdk" },

chatsbom-0.2.8/chatsbom/core/clickhouse.py

@@ -1,98 +0,0 @@
-"""ClickHouse connection utilities."""
-import clickhouse_connect
-import typer
-from rich.console import Console
-
-
-def check_clickhouse_connection(
-    host: str,
-    port: int,
-    user: str,
-    password: str,
-    database: str = 'chatsbom',
-    console: Console | None = None,
-    require_database: bool = True,
-) -> bool:
-    """
-    Check ClickHouse connection and optionally verify database existence.
-
-    Args:
-        host: ClickHouse host
-        port: ClickHouse HTTP port
-        user: ClickHouse username
-        password: ClickHouse password
-        database: Database to check
-        console: Rich console for output (creates one if None)
-        require_database: If True, check that the database exists
-
-    Returns:
-        True if connection (and database) check passed, False otherwise
-
-    Raises:
-        typer.Exit: If connection fails
-    """
-    if console is None:
-        console = Console()
-
-    # Step 1: Test basic connectivity
-    try:
-        client = clickhouse_connect.get_client(
-            host=host,
-            port=port,
-            username=user,
-            password=password,
-            database='default',
-        )
-        client.query('SELECT 1')
-    except Exception as e:
-        console.print(
-            f'[bold red]Error:[/] Failed to connect to ClickHouse at '
-            f'[cyan]{host}:{port}[/]\n\n'
-            f'Details: {e}\n\n'
-            'Please ensure ClickHouse is running:\n\n'
-            '[bold]Option 1:[/] Use docker compose\n'
-            '  [cyan]docker compose up -d[/]\n\n'
-            '[bold]Option 2:[/] Use docker run\n'
-            '  Step 1: Start ClickHouse\n'
-            '  [cyan]docker run --rm -d --name clickhouse \\\n'
-            '    -p 8123:8123 -p 9000:9000 \\\n'
-            '    -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \\\n'
-            '    -v ./database:/var/lib/clickhouse \\\n'
-            '    clickhouse/clickhouse-server[/]\n\n'
-            '  Step 2: Create admin and guest users\n'
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"CREATE USER IF NOT EXISTS admin IDENTIFIED BY 'admin'\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"CREATE USER IF NOT EXISTS guest IDENTIFIED BY 'guest'\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"GRANT CURRENT GRANTS ON *.* TO admin WITH GRANT OPTION\"[/]\n"
-            '  [cyan]docker exec -it clickhouse clickhouse-client -q \\\n'
-            "    \"GRANT SELECT ON *.* TO guest\"[/]\n",
-        )
-        raise typer.Exit(1)
-
-    # Step 2: Check database exists (if required)
-    if require_database:
-        try:
-            result = client.query(
-                'SELECT name FROM system.databases WHERE name = {db:String}',
-                parameters={'db': database},
-            )
-            if not result.result_rows:
-                console.print(
-                    f'[bold red]Error:[/] Database [cyan]{database}[/] '
-                    'does not exist.\n\n'
-                    'Please run the import command first to create and '
-                    'populate the database:\n\n'
-                    '  [cyan]chatsbom index[/]',
-                )
-                raise typer.Exit(1)
-        except typer.Exit:
-            raise
-        except Exception as e:
-            console.print(
-                f'[bold red]Error:[/] Failed to check database: {e}',
-            )
-            raise typer.Exit(1)
-
-    return True