cseq 0.0.1__py3-none-any.whl

cseq/html_bundle.py

@@ -0,0 +1,464 @@
+from __future__ import annotations
+
+from html import escape
+import json
+from pathlib import Path
+import re
+from typing import Iterable
+
+from .html import render_html, build_html_payload
+from .model import CpuEvidenceRecord
+from .runtime import TraceSession
+from .sequence import SequenceModel
+from .trace_analysis import MarkerSuggestion
+from .trace_diff import TraceDiffResult
+
+
+def write_html_bundle(
+    output_html: str | Path,
+    model: SequenceModel,
+    *,
+    runtime_session: TraceSession | None = None,
+    cpu_evidence: Iterable[CpuEvidenceRecord] = (),
+    explanations: dict[str, list[dict[str, object]]] | None = None,
+    trace_diff: TraceDiffResult | None = None,
+    marker_suggestions: list[MarkerSuggestion] | None = None,
+    viewport_size: int = 200,
+) -> tuple[Path, Path]:
+    """Write a Large Bundle as ``index.html + .cseqdata``.
+
+    HTTP(S) opening uses fetch(). Local ``file://`` opening can be blocked by
+    browser policy, so the shell falls back to a file chooser for the same
+    sidecar. No external server or CDN is required.
+    """
+    html_path = Path(output_html)
+    data_path = html_path.with_suffix(".cseqdata")
+
+    payload = build_html_payload(
+        model,
+        runtime_session=runtime_session,
+        cpu_evidence=cpu_evidence,
+        explanations=explanations,
+        trace_diff=trace_diff,
+        marker_suggestions=marker_suggestions,
+        viewport_size=viewport_size,
+    )
+    data_path.write_text(
+        json.dumps(payload, ensure_ascii=False, sort_keys=True),
+        encoding="utf-8",
+    )
+
+    # Build only a tiny shell. The large runtime payload never passes through
+    # a Portable-HTML string in Bundle mode.
+    portable = render_html(model, title="cseq sequence", viewport_size=viewport_size)
+    match = re.search(
+        r'<script type="application/json" id="cseq-data">(.*?)</script>',
+        portable,
+        re.S,
+    )
+    if not match:
+        raise ValueError("embedded cseq data not found")
+    meta = json.dumps({"sidecar": data_path.name}, ensure_ascii=False)
+    shell = portable[: match.start()] + (
+        f'<script type="application/json" id="cseq-bundle-meta">{meta}</script>'
+    ) + portable[match.end() :]
+
+    loader = r'''async function loadCseqBundle(){
+  const meta=JSON.parse(document.getElementById('cseq-bundle-meta').textContent);
+  try {
+    const response=await fetch(meta.sidecar,{cache:'no-store'});
+    if(!response.ok) throw new Error(`HTTP ${response.status}`);
+    return await response.json();
+  } catch(fetchError) {
+    return await new Promise((resolve,reject)=>{
+      const wrap=document.createElement('div');
+      wrap.className='panel-body';
+      const msg=document.createElement('span');
+      msg.textContent=`Sidecar ${meta.sidecar} を選択してください（file:// fallback） `;
+      const input=document.createElement('input');
+      input.type='file';
+      input.accept='.cseqdata,application/json';
+      input.onchange=async()=>{
+        try {
+          const file=input.files?.[0];
+          if(!file) return;
+          const parsed=JSON.parse(await file.text());
+          wrap.remove();
+          resolve(parsed);
+        } catch(e) { reject(e); }
+      };
+      wrap.append(msg,input);
+      document.body.prepend(wrap);
+    });
+  }
+}
+'''
+    source_line = "const data = JSON.parse(document.getElementById('cseq-data').textContent);"
+    if source_line not in shell:
+        raise ValueError("portable bootstrap line not found")
+    shell = shell.replace(
+        source_line,
+        loader + "\nasync function cseqBoot(){\nconst data = await loadCseqBundle();",
+        1,
+    )
+    close = "\n</script>\n</body>\n</html>"
+    if close not in shell:
+        raise ValueError("portable closing script not found")
+    shell = shell.replace(
+        close,
+        "\n}\ncseqBoot().catch(e=>{const pre=document.createElement('pre');pre.textContent='cseq bundle load failed: '+e;document.body.prepend(pre);});"
+        + close,
+        1,
+    )
+    shell = shell.replace(
+        "</header>",
+        f'<div class="meta">bundle data: {escape(data_path.name)}</div></header>',
+        1,
+    )
+    html_path.write_text(shell, encoding="utf-8")
+    return html_path, data_path
+
+
+def write_html_bundle_from_event_store(
+    output_html: str | Path,
+    model: SequenceModel,
+    event_store,
+    session: str,
+    *,
+    cpu_evidence: Iterable[CpuEvidenceRecord] = (),
+    explanations: dict[str, list[dict[str, object]]] | None = None,
+    trace_diff: TraceDiffResult | None = None,
+    marker_suggestions: list[MarkerSuggestion] | None = None,
+    viewport_size: int = 200,
+    event_batch_size: int = 5000,
+) -> tuple[Path, Path]:
+    """Stream a Large Bundle directly from EventStore.
+
+    Runtime events are never materialized as a Python list.  This is the
+    preferred path for million-event sessions already persisted in SQLite.
+    """
+    html_path = Path(output_html)
+    data_path = html_path.with_suffix('.cseqdata')
+    size = max(1, int(viewport_size))
+
+    payload = build_html_payload(
+        model,
+        runtime_session=None,
+        cpu_evidence=cpu_evidence,
+        explanations=explanations,
+        trace_diff=trace_diff,
+        marker_suggestions=marker_suggestions,
+        viewport_size=size,
+    )
+    payload.pop('runtime', None)
+    count = event_store.count(session)
+    payload['lod'] = 'summary' if count > 10000 else ('compact' if count > 1000 else 'detail')
+
+    tiles: list[dict[str, object]] = []
+    tile_events: list[dict[str, object]] = []
+    tile_start = 0
+
+    def event_row(e):
+        return {
+            'event_index': e.event_index,
+            'marker_id': e.marker_id,
+            'timestamp': e.timestamp_raw,
+            'cpu': e.cpu_hint,
+            'task': e.task_hint,
+            'clock_domain': e.clock_domain,
+            'message': e.message,
+            'payload': e.payload_dict(),
+        }
+
+    def flush_tile():
+        nonlocal tile_events, tile_start
+        if not tile_events:
+            return
+        tiles.append({
+            'tile_index': len(tiles),
+            'start_offset': tile_start,
+            'end_offset': tile_start + len(tile_events) - 1,
+            'start_event_index': tile_events[0]['event_index'],
+            'end_event_index': tile_events[-1]['event_index'],
+            'cpus': sorted({x['cpu'] for x in tile_events if x['cpu']}),
+            'tasks': sorted({x['task'] for x in tile_events if x['task']}),
+            'marker_count': len({x['marker_id'] for x in tile_events}),
+        })
+        tile_start += len(tile_events)
+        tile_events = []
+
+    # Write all non-runtime fields first, then stream the event array. JSON key
+    # order is not semantically significant.
+    with data_path.open('w', encoding='utf-8', newline='') as fh:
+        fh.write('{')
+        first_key = True
+        for key, value in payload.items():
+            if not first_key:
+                fh.write(',')
+            first_key = False
+            fh.write(json.dumps(str(key), ensure_ascii=False)); fh.write(':')
+            fh.write(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+        if not first_key:
+            fh.write(',')
+        fh.write('"runtime":{"session":')
+        fh.write(json.dumps(session, ensure_ascii=False))
+        fh.write(',"events":[')
+        first_event = True
+        for event in event_store.iter_events(session, batch_size=event_batch_size):
+            row = event_row(event)
+            if not first_event:
+                fh.write(',')
+            first_event = False
+            fh.write(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+            tile_events.append(row)
+            if len(tile_events) >= size:
+                flush_tile()
+        flush_tile()
+        fh.write('],"tiles":')
+        fh.write(json.dumps(tiles, ensure_ascii=False, sort_keys=True, separators=(',', ':')))
+        fh.write('}}')
+
+    # Reuse the standard bundle shell, but without rebuilding the large data.
+    portable = render_html(model, title='cseq sequence', viewport_size=size)
+    match = re.search(r'<script type="application/json" id="cseq-data">(.*?)</script>', portable, re.S)
+    if not match:
+        raise ValueError('embedded cseq data not found')
+    meta = json.dumps({'sidecar': data_path.name}, ensure_ascii=False)
+    shell = portable[:match.start()] + f'<script type="application/json" id="cseq-bundle-meta">{meta}</script>' + portable[match.end():]
+    loader = r'''async function loadCseqBundle(){
+  const meta=JSON.parse(document.getElementById('cseq-bundle-meta').textContent);
+  try {
+    const response=await fetch(meta.sidecar,{cache:'no-store'});
+    if(!response.ok) throw new Error(`HTTP ${response.status}`);
+    return await response.json();
+  } catch(fetchError) {
+    return await new Promise((resolve,reject)=>{
+      const wrap=document.createElement('div'); wrap.className='panel-body';
+      const msg=document.createElement('span'); msg.textContent=`Sidecar ${meta.sidecar} を選択してください（file:// fallback） `;
+      const input=document.createElement('input'); input.type='file'; input.accept='.cseqdata,application/json';
+      input.onchange=async()=>{ try { const file=input.files?.[0]; if(!file)return; const parsed=JSON.parse(await file.text()); wrap.remove(); resolve(parsed); } catch(e){ reject(e); } };
+      wrap.append(msg,input); document.body.prepend(wrap);
+    });
+  }
+}
+'''
+    source_line = "const data = JSON.parse(document.getElementById('cseq-data').textContent);"
+    shell = shell.replace(source_line, loader + "\nasync function cseqBoot(){\nconst data = await loadCseqBundle();", 1)
+    close = "\n</script>\n</body>\n</html>"
+    shell = shell.replace(close, "\n}\ncseqBoot().catch(e=>{const pre=document.createElement('pre');pre.textContent='cseq bundle load failed: '+e;document.body.prepend(pre);});" + close, 1)
+    shell = shell.replace('</header>', f'<div class="meta">bundle data: {escape(data_path.name)}</div></header>', 1)
+    html_path.write_text(shell, encoding='utf-8')
+    return html_path, data_path
+
+
+RANGE_INDEX_MAGIC = b"CSEQIDX2"
+RANGE_INDEX_HEADER_SIZE = 32
+RANGE_INDEX_ENTRY_SIZE = 16
+
+
+def compact_range_index_size(event_count: int, viewport_size: int) -> int:
+    size = max(1, int(viewport_size))
+    count = max(0, int(event_count))
+    tiles = (count + size - 1) // size
+    return RANGE_INDEX_HEADER_SIZE + tiles * RANGE_INDEX_ENTRY_SIZE
+
+
+def write_html_range_bundle_from_event_store(
+    output_html: str | Path,
+    model: SequenceModel,
+    event_store,
+    session: str,
+    *,
+    cpu_evidence: Iterable[CpuEvidenceRecord] = (),
+    explanations: dict[str, list[dict[str, object]]] | None = None,
+    trace_diff: TraceDiffResult | None = None,
+    marker_suggestions: list[MarkerSuggestion] | None = None,
+    viewport_size: int = 200,
+) -> tuple[Path, Path, Path]:
+    import struct
+
+    html_path = Path(output_html)
+    data_path = html_path.with_suffix('.cseqdata')
+    index_path = html_path.with_suffix('.cseqidx')
+    size = max(1, int(viewport_size))
+
+    def event_row(e):
+        return {
+            'event_index': e.event_index,
+            'marker_id': e.marker_id,
+            'timestamp': e.timestamp_raw,
+            'cpu': e.cpu_hint,
+            'task': e.task_hint,
+            'clock_domain': e.clock_domain,
+            'message': e.message,
+            'payload': e.payload_dict(),
+        }
+
+    entries: list[tuple[int, int]] = []
+    tile: list[dict[str, object]] = []
+    count = 0
+
+    def flush(fh):
+        nonlocal tile
+        if not tile:
+            return
+        start_pos = fh.tell()
+        raw = (json.dumps(tile, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + '\n').encode('utf-8')
+        fh.write(raw)
+        entries.append((start_pos, fh.tell() - 1))
+        tile = []
+
+    with data_path.open('wb') as fh:
+        for event in event_store.iter_events(session, batch_size=max(1000, size * 4)):
+            tile.append(event_row(event))
+            count += 1
+            if len(tile) >= size:
+                flush(fh)
+        flush(fh)
+
+    with index_path.open('wb') as fh:
+        fh.write(struct.pack('<8sQQQ', RANGE_INDEX_MAGIC, count, size, len(entries)))
+        for byte_start, byte_end in entries:
+            fh.write(struct.pack('<QQ', byte_start, byte_end))
+
+    meta_payload = {
+        'schema': 'cseq-range-bundle-v2',
+        'session': session,
+        'event_count': count,
+        'viewport_size': size,
+        'tile_count': len(entries),
+        'lod': 'summary' if count > 10000 else ('compact' if count > 1000 else 'detail'),
+        'data_file': data_path.name,
+        'index_file': index_path.name,
+        'index_header_size': RANGE_INDEX_HEADER_SIZE,
+        'index_entry_size': RANGE_INDEX_ENTRY_SIZE,
+        'cache_tiles': 8,
+    }
+
+    shell = render_html(
+        model,
+        title='cseq range bundle',
+        runtime_session=None,
+        cpu_evidence=cpu_evidence,
+        explanations=explanations,
+        trace_diff=trace_diff,
+        marker_suggestions=marker_suggestions,
+        viewport_size=size,
+    )
+    meta = json.dumps(meta_payload, ensure_ascii=False)
+    shell = shell.replace(
+        '</header>',
+        f'<div class="meta">range bundle v2: {escape(index_path.name)}</div></header>',
+        1,
+    )
+    range_loader = r'''
+<script type="application/json" id="cseq-range-meta">__META__</script>
+<script>
+(async()=>{
+  const timeline=document.getElementById('timeline');
+  const controls=document.getElementById('timeline-controls');
+  const meta=JSON.parse(document.getElementById('cseq-range-meta').textContent);
+  const tileCache=new Map();
+  function cacheGet(k){
+    if(!tileCache.has(k)) return null;
+    const v=tileCache.get(k); tileCache.delete(k); tileCache.set(k,v); return v;
+  }
+  function cachePut(k,v){
+    if(tileCache.has(k)) tileCache.delete(k);
+    tileCache.set(k,v);
+    while(tileCache.size>meta.cache_tiles) tileCache.delete(tileCache.keys().next().value);
+  }
+  async function fetchIndexEntry(tileNo){
+    const start=meta.index_header_size + tileNo*meta.index_entry_size;
+    const end=start + meta.index_entry_size - 1;
+    const r=await fetch(meta.index_file,{cache:'no-store',headers:{Range:`bytes=${start}-${end}`}});
+    if(!(r.status===206||r.status===200)) throw new Error(`index HTTP ${r.status}`);
+    const b=await r.arrayBuffer();
+    if(b.byteLength<meta.index_entry_size) throw new Error('short index entry');
+    const d=new DataView(b);
+    return {byte_start:Number(d.getBigUint64(0,true)),byte_end:Number(d.getBigUint64(8,true))};
+  }
+  async function fetchTile(tileNo){
+    const hit=cacheGet(tileNo); if(hit) return hit;
+    const entry=await fetchIndexEntry(tileNo);
+    const r=await fetch(meta.data_file,{cache:'no-store',headers:{Range:`bytes=${entry.byte_start}-${entry.byte_end}`}});
+    if(!(r.status===206||r.status===200)) throw new Error(`data HTTP ${r.status}`);
+    const events=JSON.parse((await r.text()).trim());
+    const startEvent=tileNo*meta.viewport_size;
+    const value={tile_index:tileNo,start_event_index:startEvent,end_event_index:Math.min(meta.event_count-1,startEvent+events.length-1),byte_start:entry.byte_start,byte_end:entry.byte_end,events};
+    cachePut(tileNo,value); return value;
+  }
+  function renderRows(value){
+    timeline.replaceChildren();
+    for(const e of value.events){
+      const row=document.createElement('div'); row.className='timeline-row';
+      for(const x of [e.event_index,e.timestamp||'?',e.cpu||'CPU?',e.task||'Task?',`${e.marker_id||''} ${e.message||''}`]){
+        const cell=document.createElement('div'); cell.textContent=String(x); row.appendChild(cell);
+      }
+      timeline.appendChild(row);
+    }
+    const m=document.createElement('div'); m.className='tile-summary';
+    m.textContent=`Range Tile ${value.tile_index+1}: events ${value.start_event_index}-${value.end_event_index} / bytes ${value.byte_start}-${value.byte_end}`;
+    timeline.prepend(m);
+  }
+  function prefetch(tileNo){
+    if(tileNo<0||tileNo>=meta.tile_count||tileCache.has(tileNo)) return;
+    fetchTile(tileNo).catch(()=>{});
+  }
+  async function loadTile(tileNo){
+    tileNo=Math.max(0,Math.min(meta.tile_count-1,Number(tileNo)||0));
+    const value=await fetchTile(tileNo); renderRows(value); input.value=String(tileNo+1);
+    prefetch(tileNo-1); prefetch(tileNo+1);
+  }
+  try{
+    const block=document.createElement('div'); block.className='path-controls';
+    const label=document.createElement('span'); label.textContent=`Range Bundle v2: ${meta.event_count} events / ${meta.tile_count} tiles`;
+    const input=document.createElement('input'); input.id='range-tile-input'; input.type='number'; input.min='1'; input.max=String(Math.max(1,meta.tile_count)); input.value='1'; input.style.width='8em';
+    const go=document.createElement('button'); go.textContent='移動';
+    const prev=document.createElement('button'); prev.textContent='前';
+    const next=document.createElement('button'); next.textContent='次';
+    block.append(label,prev,input,go,next); controls.replaceChildren(block);
+    go.onclick=()=>loadTile(Number(input.value)-1);
+    input.onkeydown=e=>{if(e.key==='Enter')loadTile(Number(input.value)-1);};
+    prev.onclick=()=>loadTile(Number(input.value)-2);
+    next.onclick=()=>loadTile(Number(input.value));
+    if(meta.tile_count) await loadTile(0);
+  }catch(e){
+    timeline.textContent=`Range Bundleを読み込めません。cseq serve等のHTTP配信で開いてください: ${e}`;
+  }
+})();
+</script>
+'''.replace('__META__', meta.replace('</','<\\/'))
+    shell = shell.replace('</body>', range_loader + '\n</body>', 1)
+    html_path.write_text(shell, encoding='utf-8')
+    return html_path, data_path, index_path
+
+def write_html_range_bundle(
+    output_html: str | Path,
+    model: SequenceModel,
+    *,
+    runtime_session: TraceSession,
+    cpu_evidence: Iterable[CpuEvidenceRecord] = (),
+    explanations: dict[str, list[dict[str, object]]] | None = None,
+    trace_diff: TraceDiffResult | None = None,
+    marker_suggestions: list[MarkerSuggestion] | None = None,
+    viewport_size: int = 200,
+) -> tuple[Path, Path, Path]:
+    """Range Bundle convenience wrapper for an in-memory TraceSession."""
+    import tempfile
+    from .event_store import EventStore
+    with tempfile.TemporaryDirectory(prefix='cseq-range-') as td:
+        db = Path(td) / 'events.db'
+        with EventStore(db) as store:
+            store.replace_session_events(runtime_session.name, iter(runtime_session.events), chunk_size=max(1000, viewport_size * 5), batch_size=max(1000, viewport_size * 5))
+            return write_html_range_bundle_from_event_store(
+                output_html,
+                model,
+                store,
+                runtime_session.name,
+                cpu_evidence=cpu_evidence,
+                explanations=explanations,
+                trace_diff=trace_diff,
+                marker_suggestions=marker_suggestions,
+                viewport_size=viewport_size,
+            )

cseq/hybrid_parser.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from .fast_parser import FastLexicalParser
+from .model import Diagnostic, FileParseArtifact, ParseStatus, SourceRange, TranslationUnit
+from .parser import ClangJsonParser, ParserBackend
+from .parser_environment import probe_parser_environment
+from .tree_sitter_parser import TreeSitterCParser
+
+
+class HybridParser(ParserBackend):
+    """Canonical Tree-sitter syntax with selective Clang semantic refinement.
+
+    When the locked Tree-sitter runtime is unavailable, the legacy lexical
+    parser remains a bounded fallback. Clang construction is lazy so syntax-only
+    work can still run on environments where semantic refinement is absent.
+    """
+
+    name = "hybrid-tree-sitter-clang-v3"
+
+    def __init__(self, fast: ParserBackend | None = None, semantic: ParserBackend | None = None) -> None:
+        self.fast = fast or self._default_fast()
+        self.semantic = semantic
+        self.fast_count = 0
+        self.semantic_count = 0
+        self.syntax_only_partial_count = 0
+
+    @staticmethod
+    def _default_fast() -> ParserBackend:
+        env = probe_parser_environment()
+        if env.canonical_fast_frontend_ready:
+            try:
+                return TreeSitterCParser()
+            except RuntimeError:
+                # Environment probing already exposes the mismatch. Keep the
+                # bounded legacy fallback rather than silently dropping parsing.
+                pass
+        return FastLexicalParser()
+
+    def _semantic_backend(self) -> ParserBackend | None:
+        if self.semantic is not None:
+            return self.semantic
+        try:
+            self.semantic = ClangJsonParser()
+        except RuntimeError:
+            return None
+        return self.semantic
+
+    def parse(self, translation_unit: TranslationUnit) -> FileParseArtifact:
+        prepared = self.fast.prepare(translation_unit)  # type: ignore[attr-defined]
+        if prepared.eligibility.eligible:
+            try:
+                artifact = self.fast.parse_prepared(translation_unit, prepared)  # type: ignore[attr-defined]
+            except Exception as exc:
+                semantic = self._semantic_backend()
+                if semantic is not None:
+                    artifact = semantic.parse(translation_unit)
+                    self.semantic_count += 1
+                    return artifact
+                return _opaque_parser_failure(translation_unit, self.fast.name, exc)
+            self.fast_count += 1
+            return artifact
+
+        semantic = self._semantic_backend()
+        if semantic is not None:
+            self.semantic_count += 1
+            return semantic.parse(translation_unit)
+
+        # Tree-sitter syntax remains useful evidence even when Clang refinement
+        # is unavailable. Return it explicitly as PARTIAL rather than dropping
+        # the TU or pretending semantic completeness.
+        if isinstance(self.fast, TreeSitterCParser):
+            artifact = self.fast.parse_prepared(translation_unit, prepared)
+            artifact.status = ParseStatus.PARTIAL
+            artifact.diagnostics.append(
+                Diagnostic(
+                    severity="warning",
+                    message=(
+                        "semantic refinement required but unavailable: "
+                        + ", ".join(prepared.eligibility.reasons)
+                    ),
+                )
+            )
+            artifact.parser_backend = "tree-sitter-c-v1+semantic-pending"
+            self.syntax_only_partial_count += 1
+            return artifact
+
+        # Legacy lexical parser cannot provide trustworthy syntax for an
+        # ineligible TU. Preserve the entire TU as explicit opaque evidence.
+        return FileParseArtifact(
+            translation_unit=translation_unit,
+            status=ParseStatus.OPAQUE_REGION,
+            diagnostics=[
+                Diagnostic(
+                    severity="error",
+                    message=(
+                        "semantic backend unavailable for lexical-ineligible TU: "
+                        + ", ".join(prepared.eligibility.reasons)
+                    ),
+                )
+            ],
+            opaque_regions=[SourceRange()],
+            parser_backend="lexical-fallback+semantic-unavailable",
+        )
+
+
+def _opaque_parser_failure(
+    translation_unit: TranslationUnit, backend_name: str, exc: Exception
+) -> FileParseArtifact:
+    return FileParseArtifact(
+        translation_unit=translation_unit,
+        status=ParseStatus.OPAQUE_REGION,
+        diagnostics=[
+            Diagnostic(
+                severity="error",
+                message=f"{backend_name} implementation failure with no semantic fallback: {type(exc).__name__}: {exc}",
+            )
+        ],
+        opaque_regions=[SourceRange()],
+        parser_backend=f"{backend_name}+failed",
+    )