convoviz 0.2.12__py3-none-any.whl → 0.4.7__py3-none-any.whl

convoviz/interactive.py

@@ -1,5 +1,6 @@
 """Interactive configuration prompts using questionary."""
 
+import logging
 from pathlib import Path
 from typing import Literal, Protocol, cast
 
@@ -7,10 +8,16 @@ from questionary import Choice, Style, checkbox, select
 from questionary import path as qst_path
 from questionary import text as qst_text
 
-from convoviz.config import ConvovizConfig, get_default_config
+from convoviz.config import ConvovizConfig, OutputKind, YAMLConfig, get_default_config
 from convoviz.io.loaders import find_latest_zip, validate_zip
 from convoviz.utils import colormaps, default_font_path, font_names, font_path, validate_header
 
+OUTPUT_TITLES = {
+    OutputKind.MARKDOWN: "Markdown conversations",
+    OutputKind.GRAPHS: "Graphs (usage analytics)",
+    OutputKind.WORDCLOUDS: "Word clouds",
+}
+
 CUSTOM_STYLE = Style(
     [
         ("qmark", "fg:#34eb9b bold"),
@@ -26,6 +33,9 @@ CUSTOM_STYLE = Style(
     ]
 )
 
+logger = logging.getLogger(__name__)
+
+
 class _QuestionaryPrompt[T](Protocol):
     def ask(self) -> T | None: ...
 
@@ -73,6 +83,7 @@ def run_interactive_config(initial_config: ConvovizConfig | None = None) -> Conv
         Updated configuration based on user input
     """
     config = initial_config or get_default_config()
+    logger.info("Starting interactive configuration")
 
     # Set sensible defaults if not already set
     if not config.input_path:
@@ -87,7 +98,7 @@ def run_interactive_config(initial_config: ConvovizConfig | None = None) -> Conv
     input_default = str(config.input_path) if config.input_path else ""
     input_result: str = _ask_or_cancel(
         qst_path(
-            "Enter the path to the export ZIP, conversations JSON, or extracted directory:",
+            "Enter the path to the export ZIP:",  # , conversations JSON, or extracted directory:",
             default=input_default,
             validate=_validate_input_path,
             style=CUSTOM_STYLE,
@@ -96,6 +107,7 @@ def run_interactive_config(initial_config: ConvovizConfig | None = None) -> Conv
 
     if input_result:
         config.input_path = Path(input_result)
+    logger.debug(f"User selected input: {config.input_path}")
 
     # Prompt for output folder
     output_result: str = _ask_or_cancel(
@@ -108,140 +120,132 @@ def run_interactive_config(initial_config: ConvovizConfig | None = None) -> Conv
 
     if output_result:
         config.output_folder = Path(output_result)
-
-    # Prompt for author headers
-    headers = config.message.author_headers
-    for role in ["system", "user", "assistant", "tool"]:
-        current = getattr(headers, role)
-        result: str = _ask_or_cancel(
-            qst_text(
-                f"Enter the message header for '{role}':",
-                default=current,
-                validate=lambda t: validate_header(t)
-                or "Must be a valid markdown header (e.g., # Title)",
-                style=CUSTOM_STYLE,
-            )
+    logger.debug(f"User selected output: {config.output_folder}")
+
+    # Prompt for outputs to generate
+    output_choices = [
+        Choice(
+            title=OUTPUT_TITLES.get(kind, kind.value.title()),
+            value=kind,
+            checked=kind in config.outputs,
         )
-        if result:
-            setattr(headers, role, result)
-
-    # Prompt for LaTeX delimiters
-    latex_result = cast(
-        Literal["default", "dollars"],
-        _ask_or_cancel(
-            select(
-                "Select the LaTeX math delimiters:",
-                choices=["default", "dollars"],
-                default=config.conversation.markdown.latex_delimiters,
-                style=CUSTOM_STYLE,
-            )
-        ),
-    )
-
-    if latex_result:
-        config.conversation.markdown.latex_delimiters = latex_result
-
-    # Prompt for markdown flavor
-    flavor_result = cast(
-        Literal["standard", "obsidian"],
-        _ask_or_cancel(
-            select(
-                "Select the markdown flavor:",
-                choices=["standard", "obsidian"],
-                default=config.conversation.markdown.flavor,
-                style=CUSTOM_STYLE,
-            )
-        ),
-    )
-
-    if flavor_result:
-        config.conversation.markdown.flavor = flavor_result
-
-    # Prompt for YAML headers
-    yaml_config = config.conversation.yaml
-    yaml_choices = [
-        Choice(title=field, checked=getattr(yaml_config, field))
-        for field in [
-            "title",
-            "tags",
-            "chat_link",
-            "create_time",
-            "update_time",
-            "model",
-            "used_plugins",
-            "message_count",
-            "content_types",
-            "custom_instructions",
-        ]
+        for kind in OutputKind
     ]
 
-    selected: list[str] = _ask_or_cancel(
+    selected_outputs: list[OutputKind] = _ask_or_cancel(
         checkbox(
-            "Select YAML metadata headers to include:",
-            choices=yaml_choices,
+            "Select outputs to generate:",
+            choices=output_choices,
             style=CUSTOM_STYLE,
         )
     )
 
-    selected_set = set(selected)
-    for field_name in [
-        "title",
-        "tags",
-        "chat_link",
-        "create_time",
-        "update_time",
-        "model",
-        "used_plugins",
-        "message_count",
-        "content_types",
-        "custom_instructions",
-    ]:
-        setattr(yaml_config, field_name, field_name in selected_set)
-
-    # Prompt for font
-    available_fonts = font_names()
-    if available_fonts:
-        current_font = (
-            config.wordcloud.font_path.stem if config.wordcloud.font_path else available_fonts[0]
-        )
-        font_result: str = _ask_or_cancel(
-            select(
-                "Select the font for word clouds:",
-                choices=available_fonts,
-                default=current_font if current_font in available_fonts else available_fonts[0],
-                style=CUSTOM_STYLE,
+    config.outputs = set(selected_outputs) if selected_outputs else set()
+    logger.debug(f"User selected outputs: {config.outputs}")
+
+    # Prompt for markdown settings (only if markdown output is selected)
+    if OutputKind.MARKDOWN in config.outputs:
+        # Prompt for author headers
+        headers = config.message.author_headers
+        for role in ["user", "assistant"]:
+            current = getattr(headers, role)
+            result: str = _ask_or_cancel(
+                qst_text(
+                    f"Enter the message header for '{role}':",
+                    default=current,
+                    validate=lambda t: validate_header(t)
+                    or "Must be a valid markdown header (e.g., # Title)",
+                    style=CUSTOM_STYLE,
+                )
             )
+            if result:
+                setattr(headers, role, result)
+        logger.debug(f"User selected headers: {headers}")
+
+        # Prompt for markdown flavor
+        flavor_result = cast(
+            Literal["standard", "obsidian"],
+            _ask_or_cancel(
+                select(
+                    "Select the markdown flavor:",
+                    choices=["standard", "obsidian"],
+                    default=config.conversation.markdown.flavor,
+                    style=CUSTOM_STYLE,
+                )
+            ),
         )
 
-        if font_result:
-            config.wordcloud.font_path = font_path(font_result)
-
-    # Prompt for colormap
-    available_colormaps = colormaps()
-    if available_colormaps:
-        colormap_result: str = _ask_or_cancel(
-            select(
-                "Select the color theme for word clouds:",
-                choices=available_colormaps,
-                default=config.wordcloud.colormap
-                if config.wordcloud.colormap in available_colormaps
-                else available_colormaps[0],
+        if flavor_result:
+            config.conversation.markdown.flavor = flavor_result
+        logger.debug(f"User selected flavor: {config.conversation.markdown.flavor}")
+
+        # Prompt for YAML headers
+        yaml_config = config.conversation.yaml
+        yaml_fields = list(YAMLConfig.model_fields.keys())
+        yaml_choices = [
+            Choice(title=field, checked=getattr(yaml_config, field)) for field in yaml_fields
+        ]
+
+        selected: list[str] = _ask_or_cancel(
+            checkbox(
+                "Select YAML metadata headers to include:",
+                choices=yaml_choices,
                 style=CUSTOM_STYLE,
             )
         )
 
-        if colormap_result:
-            config.wordcloud.colormap = colormap_result
+        selected_set = set(selected)
+        for field_name in yaml_fields:
+            setattr(yaml_config, field_name, field_name in selected_set)
+
+    # Prompt for wordcloud settings (only if wordclouds output is selected)
+    if OutputKind.WORDCLOUDS in config.outputs:
+        # Prompt for font
+        available_fonts = font_names()
+        if available_fonts:
+            current_font = (
+                config.wordcloud.font_path.stem
+                if config.wordcloud.font_path
+                else available_fonts[0]
+            )
+            font_result: str = _ask_or_cancel(
+                select(
+                    "Select the font for word clouds:",
+                    choices=available_fonts,
+                    default=current_font if current_font in available_fonts else available_fonts[0],
+                    style=CUSTOM_STYLE,
+                )
+            )
 
-    # Prompt for custom stopwords
-    stopwords_result: str = _ask_or_cancel(
-        qst_text(
-            "Enter custom stopwords (comma-separated):",
-            default=config.wordcloud.custom_stopwords,
-            style=CUSTOM_STYLE,
+            if font_result:
+                config.wordcloud.font_path = font_path(font_result)
+
+        # Prompt for colormap
+        available_colormaps = colormaps()
+        if available_colormaps:
+            colormap_result: str = _ask_or_cancel(
+                select(
+                    "Select the color theme for word clouds:",
+                    choices=available_colormaps,
+                    default=config.wordcloud.colormap
+                    if config.wordcloud.colormap in available_colormaps
+                    else available_colormaps[0],
+                    style=CUSTOM_STYLE,
+                )
+            )
+
+            if colormap_result:
+                config.wordcloud.colormap = colormap_result
+
+        # Prompt for custom stopwords
+        stopwords_result: str = _ask_or_cancel(
+            qst_text(
+                "Enter custom stopwords (comma-separated):",
+                default=config.wordcloud.custom_stopwords,
+                style=CUSTOM_STYLE,
+            )
         )
-    )
 
-    config.wordcloud.custom_stopwords = stopwords_result
+        config.wordcloud.custom_stopwords = stopwords_result
 
     return config

convoviz/io/assets.py

@@ -1,8 +1,11 @@
 "Asset management functions."
 
+import logging
 import shutil
 from pathlib import Path
 
+logger = logging.getLogger(__name__)
+
 
 def resolve_asset_path(source_dir: Path, asset_id: str) -> Path | None:
     """Find the actual file for a given asset ID in the source directory.
@@ -26,6 +29,7 @@ def resolve_asset_path(source_dir: Path, asset_id: str) -> Path | None:
     # 1. Try exact match
     exact_path = (source_dir / asset_id).resolve()
     if exact_path.exists() and exact_path.is_file() and exact_path.is_relative_to(source_dir):
+        logger.debug(f"Resolved asset (exact): {asset_id} -> {exact_path}")
         return exact_path
 
     # 2. Try prefix match in root
@@ -37,6 +41,7 @@ def resolve_asset_path(source_dir: Path, asset_id: str) -> Path | None:
             if p.is_file() and p.resolve().is_relative_to(source_dir)
         ]
         if files:
+            logger.debug(f"Resolved asset (prefix root): {asset_id} -> {files[0]}")
             return files[0]
     except Exception:
         pass
@@ -53,6 +58,7 @@ def resolve_asset_path(source_dir: Path, asset_id: str) -> Path | None:
                 if p.is_file() and p.resolve().is_relative_to(dalle_dir)
             ]
             if files:
+                logger.debug(f"Resolved asset (dalle): {asset_id} -> {files[0]}")
                 return files[0]
         except Exception:
             pass
@@ -69,6 +75,7 @@ def resolve_asset_path(source_dir: Path, asset_id: str) -> Path | None:
                     if p.is_file() and p.resolve().is_relative_to(user_dir)
                 ]
                 if files:
+                    logger.debug(f"Resolved asset (user dir): {asset_id} -> {files[0]}")
                     return files[0]
     except Exception:
         pass
@@ -92,7 +99,11 @@ def copy_asset(source_path: Path, dest_dir: Path) -> str:
     dest_path = assets_dir / source_path.name
 
     if not dest_path.exists():
-        shutil.copy2(source_path, dest_path)
+        try:
+            shutil.copy2(source_path, dest_path)
+            logger.debug(f"Copied asset: {source_path.name}")
+        except Exception as e:
+            logger.warning(f"Failed to copy asset {source_path}: {e}")
 
     # Return forward-slash path for Markdown compatibility even on Windows
     return f"assets/{source_path.name}"

convoviz/io/loaders.py

@@ -1,5 +1,6 @@
 """Loading functions for conversations and collections."""
 
+import logging
 from pathlib import Path, PurePosixPath
 from zipfile import ZipFile
 
@@ -8,6 +9,8 @@ from orjson import loads
 from convoviz.exceptions import InvalidZipError
 from convoviz.models import Conversation, ConversationCollection
 
+logger = logging.getLogger(__name__)
+
 
 def _is_safe_zip_member_name(name: str) -> bool:
     """Return True if a ZIP entry name is safe to extract.
@@ -46,6 +49,7 @@ def extract_archive(filepath: Path) -> Path:
     """
     folder = filepath.with_suffix("")
     folder.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Extracting archive: {filepath} to {folder}")
 
     with ZipFile(filepath) as zf:
         for member in zf.infolist():
@@ -115,6 +119,7 @@ def load_collection_from_json(filepath: Path | str) -> ConversationCollection:
         Loaded ConversationCollection object
     """
     filepath = Path(filepath)
+    logger.debug(f"Loading collection from JSON: {filepath}")
     with filepath.open(encoding="utf-8") as f:
         data = loads(f.read())
 

convoviz/io/writers.py

@@ -1,5 +1,6 @@
 """Writing functions for conversations and collections."""
 
+import logging
 from os import utime as os_utime
 from pathlib import Path
 from urllib.parse import quote
@@ -13,6 +14,8 @@ from convoviz.models import Conversation, ConversationCollection
 from convoviz.renderers import render_conversation
 from convoviz.utils import sanitize
 
+logger = logging.getLogger(__name__)
+
 # Month names for folder naming
 _MONTH_NAMES = [
     "January",
@@ -102,6 +105,7 @@ def save_conversation(
     markdown = render_conversation(conversation, config, headers, asset_resolver=asset_resolver)
     with final_path.open("w", encoding="utf-8") as f:
         f.write(markdown)
+    logger.debug(f"Saved conversation: {final_path}")
 
     # Set modification time
     timestamp = conversation.update_time.timestamp()
@@ -135,6 +139,7 @@ def _generate_year_index(year_dir: Path, year: str) -> None:
 
     index_path = year_dir / "_index.md"
     index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    logger.debug(f"Generated year index: {index_path}")
 
 
 def _generate_month_index(month_dir: Path, year: str, month: str) -> None:
@@ -146,9 +151,7 @@ def _generate_month_index(month_dir: Path, year: str, month: str) -> None:
         month: The month folder name (e.g., "03-March")
     """
     month_name = month.split("-", 1)[1] if "-" in month else month
-    files = sorted(
-        [f.name for f in month_dir.glob("*.md") if f.name != "_index.md"]
-    )
+    files = sorted([f.name for f in month_dir.glob("*.md") if f.name != "_index.md"])
 
     lines = [
         f"# {month_name} {year}",
@@ -164,6 +167,7 @@ def _generate_month_index(month_dir: Path, year: str, month: str) -> None:
 
     index_path = month_dir / "_index.md"
     index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    logger.debug(f"Generated month index: {index_path}")
 
 
 def save_collection(

convoviz/logging_config.py

@@ -0,0 +1,69 @@
+"""Logging configuration for convoviz."""
+
+import logging
+import tempfile
+from pathlib import Path
+
+from rich.logging import RichHandler
+
+
+def setup_logging(
+    verbosity: int = 0,
+    log_file: Path | None = None,
+) -> Path:
+    """Set up logging configuration.
+
+    Args:
+        verbosity: Level of verbosity (0=WARNING, 1=INFO, 2=DEBUG)
+        log_file: Path to log file. If None, a temporary file is created.
+
+    Returns:
+        Path to the log file used.
+    """
+    # clear existing handlers
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+
+    # Determine log level for console
+    if verbosity >= 2:
+        console_level = logging.DEBUG
+    elif verbosity >= 1:
+        console_level = logging.INFO
+    else:
+        console_level = logging.WARNING
+
+    # Console handler (Rich)
+    console_handler = RichHandler(
+        rich_tracebacks=True,
+        markup=True,
+        show_time=False,
+        show_path=False,
+    )
+    console_handler.setLevel(console_level)
+
+    # File handler
+    if log_file is None:
+        # Create temp file if not specified
+        with tempfile.NamedTemporaryFile(prefix="convoviz_", suffix=".log", delete=False) as tf:
+            log_file = Path(tf.name)
+
+    # Ensure parent dir exists
+    if not log_file.parent.exists():
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+
+    file_handler = logging.FileHandler(log_file, encoding="utf-8")
+    file_handler.setLevel(logging.DEBUG)  # Always log DEBUG to file
+    file_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    file_handler.setFormatter(file_formatter)
+
+    # Configure root logger
+    # We set root level to DEBUG so that the handlers can filter as they please
+    root_logger.setLevel(logging.DEBUG)
+    root_logger.addHandler(console_handler)
+    root_logger.addHandler(file_handler)
+
+    # Reduce noise from explicit libraries if necessary
+    logging.getLogger("matplotlib").setLevel(logging.WARNING)
+    logging.getLogger("PIL").setLevel(logging.WARNING)
+
+    return log_file

convoviz/models/conversation.py

@@ -24,6 +24,8 @@ class Conversation(BaseModel):
     mapping: dict[str, Node]
     moderation_results: list[Any] = Field(default_factory=list)
     current_node: str
+    is_starred: bool | None = None
+    voice: str | dict[str, Any] | None = None
     plugin_ids: list[str] | None = None
     conversation_id: str
     conversation_template_id: str | None = None
@@ -156,3 +158,19 @@ class Conversation(BaseModel):
     def year_start(self) -> datetime:
         """Get January 1st of the year this conversation was created."""
         return self.create_time.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
+
+    @property
+    def citation_map(self) -> dict[str, dict[str, str | None]]:
+        """Aggregate citation metadata from all messages in the conversation.
+
+        Traverses all nodes (including hidden ones) to collect embedded citation definitions
+        from tool outputs (e.g. search results).
+        """
+        aggregated_map = {}
+        for node in self.all_message_nodes:
+            if not node.message:
+                continue
+            # Extract citations from message parts
+            if hasattr(node.message, "internal_citation_map"):
+                aggregated_map.update(node.message.internal_citation_map)
+        return aggregated_map

convoviz/models/message.py

@@ -46,6 +46,9 @@ class MessageMetadata(BaseModel):
     is_user_system_message: bool | None = None
     is_visually_hidden_from_conversation: bool | None = None
     user_context_message_data: dict[str, Any] | None = None
+    citations: list[dict[str, Any]] | None = None
+    search_result_groups: list[dict[str, Any]] | None = None
+    content_references: list[dict[str, Any]] | None = None
 
     model_config = ConfigDict(protected_namespaces=())
 
@@ -179,11 +182,12 @@ class Message(BaseModel):
         1. It is empty (no text, no images).
         2. Explicitly marked as visually hidden.
         3. It is an internal system message (not custom instructions).
-        4. It is a browser tool output (intermediate search steps).
+        4. It is a browser tool output (intermediate search steps) UNLESS it is a tether_quote.
         5. It is an assistant message targeting a tool (internal call).
         6. It is code interpreter input (content_type="code").
-        7. It is browsing status (tether_browsing_display).
-        8. It is internal reasoning (thoughts, reasoning_recap from o1/o3).
+        7. It is browsing status, internal reasoning (o1/o3), or massive web scraps (sonic_webpage).
+        8. It is a redundant DALL-E textual status update.
+        9. It is from internal bio (memory) or web.run orchestration tools.
         """
         if self.is_empty:
             return True
@@ -197,10 +201,29 @@ class Message(BaseModel):
             # Only show if explicitly marked as user system message (Custom Instructions)
             return not self.metadata.is_user_system_message
 
-        # Hide browser tool outputs (intermediate search steps)
-        if self.author.role == "tool" and self.author.name == "browser":
+        # Hide sonic_webpage (massive scraped text) and system_error
+        if self.content.content_type in ("sonic_webpage", "system_error"):
             return True
 
+        if self.author.role == "tool":
+            # Hide memory updates (bio) and internal search orchestration (web.run)
+            if self.author.name in ("bio", "web.run"):
+                return True
+
+            # Hide browser tool outputs (intermediate search steps)
+            # EXCEPTION: tether_quote (citations) should remain visible
+            if self.author.name == "browser":
+                return self.content.content_type != "tether_quote"
+
+            # Hide DALL-E textual status ("DALL·E displayed 1 images...")
+            if (
+                self.author.name == "dalle.text2im"
+                and self.content.content_type == "text"
+                # Check if it doesn't have images (just in case they attach images to text logic)
+                and not self.images
+            ):
+                return True
+
         # Hide assistant messages targeting tools (e.g., search(...), code input)
         # recipient="all" or None means it's for the user; anything else is internal
         if self.author.role == "assistant" and self.recipient not in ("all", None):
@@ -216,3 +239,56 @@ class Message(BaseModel):
             "thoughts",
             "reasoning_recap",
         )
+
+    @property
+    def internal_citation_map(self) -> dict[str, dict[str, str | None]]:
+        """Extract a map of citation IDs to metadata from content parts.
+
+        Used for resolving embedded citations (e.g. citeturn0search18).
+        Key format: "turn{turn_index}search{ref_index}"
+        """
+        if not self.content.parts:
+            return {}
+
+        citation_mapping = {}
+
+        # Helper to process a single search result entry
+        def process_entry(entry: dict[str, Any]) -> None:
+            ref_id = entry.get("ref_id")
+            if not ref_id:
+                return
+            # Only care about search results for now
+            if ref_id.get("ref_type") != "search":
+                return
+
+            turn_idx = ref_id.get("turn_index")
+            ref_idx = ref_id.get("ref_index")
+
+            if turn_idx is not None and ref_idx is not None:
+                # turn_idx is int, ref_idx is int
+                key = f"turn{turn_idx}search{ref_idx}"
+                citation_mapping[key] = {
+                    "title": entry.get("title"),
+                    "url": entry.get("url"),
+                }
+
+        # 1. Extract from self.content.parts
+        if self.content and self.content.parts:
+            for part in self.content.parts:
+                if isinstance(part, dict):
+                    if part.get("type") == "search_result":
+                        process_entry(part)
+                    elif part.get("type") == "search_result_group":
+                        for entry in part.get("entries", []):
+                            process_entry(entry)
+
+        # 2. Extract from metadata.search_result_groups (if present)
+        if self.metadata and self.metadata.search_result_groups:
+            for group in self.metadata.search_result_groups:
+                if isinstance(group, dict):
+                    # Groups might have 'entries' or be flat?
+                    # Based on name 'groups', likely similar to part structure
+                    for entry in group.get("entries", []):
+                        process_entry(entry)
+
+        return citation_mapping