htmlgraph 0.9.3__py3-none-any.whl → 0.27.5__py3-none-any.whl

htmlgraph/parser.py

@@ -4,10 +4,10 @@ HTML parser wrapper using justhtml.
 Provides CSS selector-based querying and data extraction from HTML files.
 """
 
+import re
+from datetime import datetime
 from pathlib import Path
 from typing import Any
-from datetime import datetime
-import re
 
 from justhtml import JustHTML
 
@@ -22,7 +22,9 @@ class HtmlParser:
     - Graph structure parsing (nodes, edges)
     """
 
-    def __init__(self, html_content: str | None = None, filepath: Path | str | None = None):
+    def __init__(
+        self, html_content: str | None = None, filepath: Path | str | None = None
+    ):
         """
         Initialize parser with HTML content or file.
 
@@ -60,7 +62,8 @@ class HtmlParser:
         Returns:
             List of matching elements
         """
-        return self.html.query(selector)
+        result: list[Any] = self.html.query(selector)
+        return result
 
     def query_one(self, selector: str) -> Any | None:
         """
@@ -77,20 +80,23 @@ class HtmlParser:
 
     def get_article(self) -> Any | None:
         """Get the main article element (graph node root)."""
-        return self.query_one("article[id]")
+        results = self.query("article[id]")
+        return results[0] if results else None
 
     def get_node_id(self) -> str | None:
         """Extract node ID from article element."""
         article = self.get_article()
         if article:
-            return article.attrs.get("id")
+            result: str | None = article.attrs.get("id")
+            return result
         return None
 
     def get_data_attribute(self, element: Any, attr: str) -> str | None:
         """Get a data-* attribute value from an element."""
         if element is None:
             return None
-        return element.attrs.get(f"data-{attr}")
+        result: str | None = element.attrs.get(f"data-{attr}")
+        return result
 
     def get_all_data_attributes(self, element: Any) -> dict[str, str]:
         """Get all data-* attributes from an element."""
@@ -123,31 +129,70 @@ class HtmlParser:
         }
 
         # Standard attributes
-        for attr in ["type", "status", "priority", "agent-assigned", "track-id", "plan-task-id", "claimed-by-session"]:
+        for attr in [
+            "type",
+            "status",
+            "priority",
+            "agent-assigned",
+            "track-id",
+            "plan-task-id",
+            "claimed-by-session",
+            "spike-subtype",
+            "session-id",
+            "from-feature-id",
+            "to-feature-id",
+            "model-name",
+        ]:
             value = self.get_data_attribute(article, attr)
             if value:
                 key = attr.replace("-", "_")
                 metadata[key] = value
 
+        # Boolean attributes
+        auto_generated = self.get_data_attribute(article, "auto-generated")
+        if auto_generated:
+            metadata["auto_generated"] = auto_generated.lower() == "true"
+
+        # Pattern sequence (for pattern nodes)
+        sequence_attr = self.get_data_attribute(article, "sequence")
+        if sequence_attr:
+            try:
+                import json
+
+                metadata["sequence"] = json.loads(sequence_attr)
+            except (json.JSONDecodeError, ValueError):
+                # Invalid JSON, skip
+                pass
+
         # Timestamps (with fallbacks for session-specific attributes)
         claimed_at = self.get_data_attribute(article, "claimed-at")
         if claimed_at:
             try:
-                metadata["claimed_at"] = datetime.fromisoformat(claimed_at.replace("Z", "+00:00"))
+                metadata["claimed_at"] = datetime.fromisoformat(
+                    claimed_at.replace("Z", "+00:00")
+                )
             except ValueError:
                 metadata["claimed_at"] = claimed_at
 
-        created_value = self.get_data_attribute(article, "created") or self.get_data_attribute(article, "started-at")
+        created_value = self.get_data_attribute(
+            article, "created"
+        ) or self.get_data_attribute(article, "started-at")
         if created_value:
             try:
-                metadata["created"] = datetime.fromisoformat(created_value.replace("Z", "+00:00"))
+                metadata["created"] = datetime.fromisoformat(
+                    created_value.replace("Z", "+00:00")
+                )
             except ValueError:
                 metadata["created"] = created_value
 
-        updated_value = self.get_data_attribute(article, "updated") or self.get_data_attribute(article, "last-activity")
+        updated_value = self.get_data_attribute(
+            article, "updated"
+        ) or self.get_data_attribute(article, "last-activity")
         if updated_value:
             try:
-                metadata["updated"] = datetime.fromisoformat(updated_value.replace("Z", "+00:00"))
+                metadata["updated"] = datetime.fromisoformat(
+                    updated_value.replace("Z", "+00:00")
+                )
             except ValueError:
                 metadata["updated"] = updated_value
 
@@ -156,14 +201,18 @@ class HtmlParser:
     def get_title(self) -> str | None:
         """Get node title from h1 or title element."""
         # Try h1 in header first
-        h1 = self.query_one("article header h1")
+        h1_results = self.query("article header h1")
+        h1 = h1_results[0] if h1_results else None
         if h1:
-            return h1.to_text().strip()
+            text: str = h1.to_text().strip()
+            return text
 
         # Fall back to title element
-        title = self.query_one("title")
+        title_results = self.query("title")
+        title = title_results[0] if title_results else None
         if title:
-            return title.to_text().strip()
+            text2: str = title.to_text().strip()
+            return text2
 
         return None
 
@@ -179,7 +228,8 @@ class HtmlParser:
         """
         edges: dict[str, list[dict[str, Any]]] = {}
 
-        edge_nav = self.query_one("nav[data-graph-edges]")
+        edge_nav_results = self.query("nav[data-graph-edges]")
+        edge_nav = edge_nav_results[0] if edge_nav_results else None
         if not edge_nav:
             return edges
 
@@ -211,13 +261,18 @@ class HtmlParser:
                 since = link.attrs.get("data-since")
                 if since:
                     try:
-                        edge_data["since"] = datetime.fromisoformat(since.replace("Z", "+00:00"))
+                        edge_data["since"] = datetime.fromisoformat(
+                            since.replace("Z", "+00:00")
+                        )
                     except ValueError:
                         edge_data["since"] = since
 
                 # Any other data attributes as properties
                 for key, value in link.attrs.items():
-                    if key.startswith("data-") and key not in ["data-relationship", "data-since"]:
+                    if key.startswith("data-") and key not in [
+                        "data-relationship",
+                        "data-since",
+                    ]:
                         if "properties" not in edge_data:
                             edge_data["properties"] = {}
                         edge_data["properties"][key[5:]] = value
@@ -245,11 +300,13 @@ class HtmlParser:
             # Remove common status emojis
             text = re.sub(r"^[✅⏳❌🔄]\s*", "", text)
 
-            steps.append({
-                "description": text,
-                "completed": completed,
-                "agent": agent,
-            })
+            steps.append(
+                {
+                    "description": text,
+                    "completed": completed,
+                    "agent": agent,
+                }
+            )
 
         return steps
 
@@ -289,10 +346,10 @@ class HtmlParser:
         article = self.get_article()
         if article and self.get_data_attribute(article, "type") == "session":
             # Add event_count if present
-            event_count = article.attrs.get("data-event-count")
-            if event_count:
+            event_count_str: str | None = article.attrs.get("data-event-count")
+            if event_count_str:
                 try:
-                    properties["event_count"] = int(event_count)
+                    properties["event_count"] = int(event_count_str)  # type: ignore[assignment]
                 except (ValueError, TypeError):
                     pass
 
@@ -301,26 +358,85 @@ class HtmlParser:
             if agent:
                 properties["agent"] = agent
 
+            # Add transcript_id if present (for Claude Code transcript integration)
+            transcript_id = article.attrs.get("data-transcript-id")
+            if transcript_id:
+                properties["transcript_id"] = transcript_id
+
         return properties
 
     def get_content(self) -> str:
         """Extract main content from section[data-content]."""
-        content_section = self.query_one("section[data-content]")
+        content_section_results = self.query("section[data-content]")
+        content_section = (
+            content_section_results[0] if content_section_results else None
+        )
         if not content_section:
             return ""
 
         # Get text content excluding the h3 header
         text_parts = []
         for child in content_section.children:
-            if hasattr(child, 'name') and child.name == "h3":
+            if hasattr(child, "name") and child.name == "h3":
                 continue
-            if hasattr(child, 'to_text'):
+            if hasattr(child, "to_text"):
                 text = child.to_text().strip()
                 if text:
                     text_parts.append(text)
 
         return "\n".join(text_parts)
 
+    def get_findings(self) -> str | None:
+        """Extract findings from section[data-findings] (Spike-specific)."""
+        findings_section_results = self.query("section[data-findings]")
+        findings_section = (
+            findings_section_results[0] if findings_section_results else None
+        )
+        if not findings_section:
+            return None
+
+        # Look for findings-content div using full selector
+        content_div_results = self.query("section[data-findings] div.findings-content")
+        content_div = content_div_results[0] if content_div_results else None
+        if content_div:
+            text = content_div.to_text().strip()
+            return text if text else None
+
+        # Fallback: get all text excluding h3 header
+        text_parts = []
+        for child in findings_section.children:
+            if hasattr(child, "name") and child.name == "h3":
+                continue
+            if hasattr(child, "to_text"):
+                text = child.to_text().strip()
+                if text:
+                    text_parts.append(text)
+
+        result = "\n".join(text_parts)
+        return result if result else None
+
+    def get_decision(self) -> str | None:
+        """Extract decision from section[data-decision] (Spike-specific)."""
+        decision_section_results = self.query("section[data-decision]")
+        decision_section = (
+            decision_section_results[0] if decision_section_results else None
+        )
+        if not decision_section:
+            return None
+
+        # Get text content excluding the h3 header
+        text_parts = []
+        for child in decision_section.children:
+            if hasattr(child, "name") and child.name == "h3":
+                continue
+            if hasattr(child, "to_text"):
+                text = child.to_text().strip()
+                if text:
+                    text_parts.append(text)
+
+        result = "\n".join(text_parts)
+        return result if result else None
+
     def parse_full_node(self) -> dict[str, Any]:
         """
         Parse complete node data from HTML.
@@ -330,7 +446,7 @@ class HtmlParser:
         metadata = self.get_node_metadata()
         title = self.get_title()
 
-        return {
+        result = {
             **metadata,
             "title": title or metadata.get("id", "Untitled"),
             "edges": self.get_edges(),
@@ -339,6 +455,17 @@ class HtmlParser:
             "content": self.get_content(),
         }
 
+        # Add Spike-specific fields if present
+        findings = self.get_findings()
+        if findings is not None:
+            result["findings"] = findings
+
+        decision = self.get_decision()
+        if decision is not None:
+            result["decision"] = decision
+
+        return result
+
 
 def parse_html_file(filepath: Path | str) -> dict[str, Any]:
     """
@@ -355,9 +482,7 @@ def parse_html_file(filepath: Path | str) -> dict[str, Any]:
 
 
 def query_html_files(
-    directory: Path | str,
-    selector: str,
-    pattern: str = "*.html"
+    directory: Path | str, selector: str, pattern: str = "*.html"
 ) -> list[tuple[Path, list[Any]]]:
     """
     Query multiple HTML files with CSS selector.