scrypted-tuya 0.1.2-beta → 2.1.0

package/db_to_md.py

@@ -0,0 +1,484 @@
+#!/usr/bin/env python3
+"""Convert Gemini antigravity conversation .db files to Markdown."""
+
+import sqlite3
+import re
+import json
+import sys
+import os
+
+
+def parse_protobuf(data: bytes) -> dict:
+    result = {}
+    pos = 0
+    while pos < len(data):
+        if pos >= len(data):
+            break
+        byte = data[pos]
+        field_tag = byte >> 3
+        wire_type = byte & 0x7
+        pos += 1
+        if wire_type == 0:
+            value = 0; shift = 0
+            while pos < len(data):
+                b = data[pos]
+                value |= (b & 0x7f) << shift
+                pos += 1
+                if (b & 0x80) == 0: break
+                shift += 7
+            result[field_tag] = value
+        elif wire_type == 2:
+            if pos >= len(data): break
+            length = 0; shift = 0
+            while pos < len(data):
+                b = data[pos]
+                length |= (b & 0x7f) << shift
+                pos += 1
+                if (b & 0x80) == 0: break
+                shift += 7
+            value = data[pos:pos + length]
+            pos += length
+            result[field_tag] = value
+        else:
+            break
+    return result
+
+
+def extract_text_from_bytes(data: bytes) -> list:
+    if not data: return []
+    return [t.decode('utf-8', errors='replace') for t in re.findall(b'[\x20-\x7e]{4,}', data)]
+
+
+def extract_text_segments(data: bytes) -> list:
+    if not data: return []
+    return [t.decode('utf-8', errors='replace') for t in re.findall(b'[^\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\xff]{5,}', data)]
+
+
+def extract_json_from_bytes(data: bytes):
+    if not data: return None
+    json_match = re.search(b'\{[^\}]{5,}[^\}]*\}', data)
+    if json_match:
+        try:
+            return json.loads(json_match.group().decode('utf-8'))
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            pass
+    return None
+
+
+def parse_step_payload(payload: bytes) -> dict:
+    if not payload: return {}
+    result = {}
+    parsed = parse_protobuf(payload)
+    for field_tag, value in parsed.items():
+        if field_tag == 5 and isinstance(value, bytes):
+            nested = parse_protobuf(value)
+            result['nested'] = nested
+            if 4 in nested and isinstance(nested[4], bytes):
+                field4_text = extract_text_from_bytes(nested[4])
+                result['tool_names'] = field4_text
+                result['json'] = extract_json_from_bytes(nested[4])
+                if result.get('json'):
+                    j = result['json']
+                    result['tool_action'] = j.get('toolAction', '')
+                    result['tool_summary'] = j.get('toolSummary', '')
+                    result['command_line'] = j.get('CommandLine', '')
+                    result['cwd'] = j.get('Cwd', '')
+                    result['pattern'] = j.get('Pattern', '')
+                    result['search_directory'] = j.get('SearchDirectory', '')
+                    result['directory_path'] = j.get('DirectoryPath', '')
+                    result['excludes'] = j.get('Excludes', [])
+            if 9 in nested and isinstance(nested[9], bytes):
+                result['session_info'] = extract_text_from_bytes(nested[9])
+        elif field_tag == 4 and isinstance(value, bytes):
+            result['field4_text'] = extract_text_from_bytes(value)
+            result['json'] = extract_json_from_bytes(value)
+    return result
+
+
+def parse_metadata(metadata: bytes) -> dict:
+    if not metadata: return {}
+    result = {}
+    result['all_text'] = extract_text_from_bytes(metadata)
+    call_match = re.search(b'call_[a-zA-Z0-9_]+', metadata)
+    if call_match:
+        result['call_id'] = call_match.group().decode('utf-8')
+    result['json'] = extract_json_from_bytes(metadata)
+    return result
+
+
+def parse_executor_metadata(data: bytes) -> dict:
+    result = {}
+    parsed = parse_protobuf(data)
+    for field_tag, value in parsed.items():
+        if field_tag == 10 and isinstance(value, bytes):
+            nested = parse_protobuf(value)
+            for ft2, fv2 in nested.items():
+                if ft2 == 1 and isinstance(fv2, bytes):
+                    result['content_segments'] = extract_text_segments(fv2)
+                    result['json'] = extract_json_from_bytes(fv2)
+                elif ft2 == 3 and isinstance(fv2, bytes):
+                    result['tool_definitions'] = extract_text_segments(fv2)
+                elif ft2 == 4 and isinstance(fv2, bytes):
+                    result['system_prompt'] = extract_text_segments(fv2)
+    return result
+
+
+STEP_TYPE_NAMES = {
+    14: 'user', 15: 'assistant', 17: 'system',
+    23: 'assistant', 101: 'assistant', 132: 'tool_call',
+}
+STATUS_NAMES = {3: 'completed', 5: 'completed', 6: 'error', 7: 'error'}
+
+
+def extract_paren_content(text, prefix):
+    """Extract content after prefix up to matching closing paren."""
+    if not text.startswith(prefix):
+        return None
+    start = len(prefix)
+    depth = 1
+    pos = start
+    while pos < len(text) and depth > 0:
+        if text[pos] == '(':
+            depth += 1
+        elif text[pos] == ')':
+            depth -= 1
+        pos += 1
+    return text[start:pos - 1]
+
+
+def convert_db(db_path: str, output_path: str = None):
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+
+    trajectory_meta = conn.execute('SELECT * FROM trajectory_meta LIMIT 1').fetchone()
+    if trajectory_meta:
+        trajectory_id, cascade_id = trajectory_meta['trajectory_id'], trajectory_meta['cascade_id']
+        trajectory_type, source = trajectory_meta['trajectory_type'], trajectory_meta['source']
+    else:
+        trajectory_id = cascade_id = trajectory_type = source = None
+
+    steps = conn.execute('SELECT idx, step_type, status, step_payload, metadata FROM steps ORDER BY idx').fetchall()
+
+    # Load executor_metadata - use only first entry to avoid duplication
+    executor_content = []
+    em_cur = conn.execute('SELECT data FROM executor_metadata ORDER BY idx LIMIT 1')
+    em_row = em_cur.fetchone()
+    if em_row:
+        em_result = parse_executor_metadata(em_row[0])
+        if 'content_segments' in em_result:
+            executor_content = em_result['content_segments']
+
+    # Get unique content segments (deduplicate)
+    seen = set()
+    unique_segments = []
+    for s in executor_content:
+        s = s.strip()
+        if len(s) > 3 and s not in seen:
+            seen.add(s)
+            unique_segments.append(s)
+
+    lines = []
+    lines.append('# Conversation Export')
+    lines.append('')
+
+    if trajectory_meta:
+        lines.append('## Trajectory Metadata')
+        lines.append('')
+        lines.append(f'- **Trajectory ID**: `{trajectory_id}`')
+        lines.append(f'- **Cascade ID**: `{cascade_id}`')
+        lines.append(f'- **Trajectory Type**: {trajectory_type}')
+        lines.append(f'- **Source**: {source}')
+        lines.append(f'- **Total Steps**: {len(steps)}')
+        lines.append('')
+    lines.append('---')
+    lines.append('')
+
+    # --- Executor Metadata: Command Summary ---
+    if unique_segments:
+        lines.append('## Commands & Actions Summary')
+        lines.append('')
+        lines.append('Extracted from executor metadata:')
+        lines.append('')
+
+        for text in unique_segments:
+            text = text.strip()
+            if len(text) < 4: continue
+
+            # Skip configuration/setting lines
+            skip_prefixes = [
+                'enable-', 'jetski-', 'antigravity_', 'teamwork_',
+                'message_continue_check', 'terminal_step_check',
+                'max_generator_invocations_check', 'empty_output_continuation_check',
+                'force_invocation', 'no_tool_call_check', 'battle_mode_tool_args',
+                'request_artifact_feedback_stop', 'idle_subagent_guard',
+                'augmented_intent', 'subagent_reminder', 'running_tasks_reminder',
+                'bash_command_reminder', 'message_delivery', 'read_url_content',
+                'replace_file_content', 'manage_subagents', 'communication_style',
+                'terminal_sandbox', 'planning_mode_artifacts', 'conversation_transcript',
+                'json-hooks-enabled', 'enable-file-diff-accumulator',
+                'enable-customization-skills', 'use-component-rewrite',
+                'enable-markdown-agents', 'agy-customizations',
+                'antigravity_guide', 'generative_ui', 'jetski-autonomous-mode',
+                'enable-cost-accumulator', 'enable-url-artifacts',
+                'enable-background-task-accumulator', 'enable-permissioned-github',
+                'enable-skill-accumulator', 'enable-teamwork-subagent',
+                'enable-owl-slash-command', 'enable-generative-hooks',
+                'teamwork_preview', 'DeepInvestigator', 'user_information',
+                '#timeout_long_running_search_command',
+                'request_artifact_feedback_stop',
+            ]
+
+            should_skip = False
+            for prefix in skip_prefixes:
+                if text.startswith(prefix):
+                    should_skip = True
+                    break
+            if should_skip:
+                continue
+
+            # System messages
+            if text.startswith('As IDE feedback'):
+                lines.append('### 💡 System Prompt')
+                lines.append('')
+                lines.append(text[:1000])
+                lines.append('')
+            elif text.startswith('planning_mode:'):
+                lines.append('### 📋 Planning Mode')
+                lines.append('')
+                lines.append(text[:500])
+                lines.append('')
+            elif text.startswith('gemini-'):
+                lines.append(f'### 🤖 Model')
+                lines.append('')
+                lines.append(text)
+                lines.append('')
+            elif text.startswith('bash_command_reminder'):
+                lines.append('### ⚠️ Bash Command Reminder')
+                lines.append('')
+                lines.append(text[:500])
+                lines.append('')
+            elif text.startswith('running_tasks_reminder'):
+                lines.append('### 📋 Running Tasks Reminder')
+                lines.append('')
+                lines.append(text[:500])
+                lines.append('')
+            elif text.startswith('unsandboxed('):
+                content = extract_paren_content(text, 'unsandboxed(')
+                if content:
+                    lines.append('### 🔓 Unsandboxed Command')
+                    lines.append('')
+                    if content.startswith('cat <<') or '{' in content[:20]:
+                        lines.append('```')
+                        lines.append(content[:3000])
+                        lines.append('```')
+                    else:
+                        lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('gunsandboxed('):
+                content = extract_paren_content(text, 'gunsandboxed(')
+                if content:
+                    lines.append('### 🔫 Gunsandboxed Command')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('command('):
+                content = extract_paren_content(text, 'command(')
+                if content:
+                    lines.append('### 💬 Command')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('read_file('):
+                content = extract_paren_content(text, 'read_file(')
+                if content:
+                    lines.append('### 📄 Read File')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('write_file('):
+                content = extract_paren_content(text, 'write_file(')
+                if content:
+                    lines.append('### ✏️ Write File')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('read_url('):
+                content = extract_paren_content(text, 'read_url(')
+                if content:
+                    lines.append('### 🔗 Read URL')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('run_command('):
+                content = extract_paren_content(text, 'run_command(')
+                if content:
+                    lines.append('### 🏃 Run Command')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('find_by_name('):
+                content = extract_paren_content(text, 'find_by_name(')
+                if content:
+                    lines.append('### 🔍 Find by Name')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('list_dir('):
+                content = extract_paren_content(text, 'list_dir(')
+                if content:
+                    lines.append('### 📁 List Directory')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif text.startswith('$') or text.startswith('!') or text.startswith('#') or text.startswith('%') or text.startswith('=') or text.startswith('>'):
+                # Result markers - skip
+                pass
+            elif text.startswith('mcp('):
+                content = extract_paren_content(text, 'mcp(')
+                if content:
+                    lines.append('### 🧩 MCP Call')
+                    lines.append('')
+                    lines.append(f'```\n{content}\n```')
+                    lines.append('')
+            elif len(text) > 20 and ('{' in text[:50] or 'EOF' in text or 'package.json' in text or 'tsconfig' in text):
+                lines.append('### 📝 Configuration')
+                lines.append('')
+                lines.append('```')
+                lines.append(text[:3000])
+                lines.append('```')
+                lines.append('')
+            elif len(text) > 15:
+                lines.append(f'### {text[:100]}')
+                lines.append('')
+
+        lines.append('---')
+        lines.append('')
+
+    # --- Tool Calls from steps table ---
+    tool_calls = []
+    assistant_msgs = []
+    user_msgs = []
+    other_steps = []
+
+    for step in steps:
+        step_type = step['step_type']
+        status = step['status']
+        idx = step['idx']
+        step_name = STEP_TYPE_NAMES.get(step_type, f'unknown({step_type})')
+        status_name = STATUS_NAMES.get(status, str(status))
+
+        payload_info = parse_step_payload(step['step_payload']) if step['step_payload'] else {}
+        meta_info = parse_metadata(step['metadata']) if step['metadata'] else {}
+
+        step_data = {
+            'idx': idx, 'step_type': step_type, 'step_name': step_name,
+            'status': status, 'status_name': status_name, **meta_info, **payload_info,
+        }
+
+        if step_type == 132: tool_calls.append(step_data)
+        elif step_type in (15, 23, 101): assistant_msgs.append(step_data)
+        elif step_type == 14: user_msgs.append(step_data)
+        elif step_type == 17: other_steps.append(step_data)
+        else: other_steps.append(step_data)
+
+    if tool_calls:
+        lines.append('## Tool Calls (Detailed)')
+        lines.append('')
+        lines.append(f'Total: {len(tool_calls)}')
+        lines.append('')
+
+        for tc in tool_calls:
+            idx = tc['idx']
+            call_id = tc.get('call_id', '')
+            all_text = tc.get('all_text', [])
+            json_data = tc.get('json', {}) or tc.get('json') or {}
+
+            tool_name = ''
+            for i, t in enumerate(all_text):
+                if t.startswith('call_') and i + 1 < len(all_text):
+                    tool_name = all_text[i + 1]
+                    break
+
+            lines.append(f'### Tool Call #{idx}')
+            lines.append('')
+            lines.append(f'- **Call ID**: `{call_id}`')
+            lines.append(f'- **Tool**: `{tool_name}`')
+            lines.append(f'- **Status**: {tc["status_name"]}')
+
+            if json_data:
+                lines.append(f'- **Action**: {json_data.get("toolAction", "")}')
+                lines.append(f'- **Summary**: {json_data.get("toolSummary", "")}')
+                lines.append(f'- **Parameters**:')
+                lines.append('')
+                lines.append('```json')
+                lines.append(json.dumps(json_data, indent=2))
+                lines.append('```')
+            else:
+                json_match = re.search(r'\{[^\}]{5,}[^\}]*\}', ' '.join(all_text))
+                if json_match:
+                    try:
+                        j = json.loads(json_match.group())
+                        lines.append(f'- **Parameters**:')
+                        lines.append('')
+                        lines.append('```json')
+                        lines.append(json.dumps(j, indent=2))
+                        lines.append('```')
+                    except: pass
+
+            if json_data:
+                for key in ['CommandLine', 'Cwd', 'Pattern', 'SearchDirectory', 'DirectoryPath', 'Excludes']:
+                    val = json_data.get(key)
+                    if val is not None:
+                        if key == 'Excludes':
+                            lines.append(f'- **{key}**: `{", ".join(val)}`')
+                        else:
+                            lines.append(f'- **{key}**: `{val}`')
+
+            session_info = tc.get('session_info', [])
+            if session_info:
+                lines.append(f'- **Session**: `{session_info[0]}`')
+            lines.append('')
+
+    lines.append('## Summary')
+    lines.append('')
+    lines.append(f'- **Total Steps**: {len(steps)}')
+    lines.append(f'- **User Messages**: {len(user_msgs)}')
+    lines.append(f'- **Assistant Messages**: {len(assistant_msgs)}')
+    lines.append(f'- **Tool Calls**: {len(tool_calls)}')
+    lines.append(f'- **System Steps**: {len(other_steps)}')
+    lines.append('')
+
+    markdown_content = '\n'.join(lines)
+
+    if output_path is None:
+        output_path = 'conversation.md'
+
+    with open(output_path, 'w', encoding='utf-8') as f:
+        f.write(markdown_content)
+
+    print(f'Converted {len(steps)} steps to {output_path}')
+    print(f'  Tool calls: {len(tool_calls)}')
+    print(f'  Assistant messages: {len(assistant_msgs)}')
+    print(f'  User messages: {len(user_msgs)}')
+    print(f'  Other: {len(other_steps)}')
+    print(f'  Unique command segments: {len(unique_segments)}')
+
+    conn.close()
+
+
+if __name__ == '__main__':
+    db_path = sys.argv[1] if len(sys.argv) > 1 else None
+    output_path = sys.argv[2] if len(sys.argv) > 2 else None
+
+    if db_path is None:
+        db_path = os.path.join(
+            os.path.expanduser('~'),
+            '.gemini', 'antigravity', 'conversations',
+            '094b96f4-9751-4978-b473-18144c70a431.db'
+        )
+
+    if not os.path.exists(db_path):
+        print(f'Error: Database not found at {db_path}', file=sys.stderr)
+        sys.exit(1)
+
+    convert_db(db_path, output_path)