cseq 0.0.1__py3-none-any.whl

cseq/project.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+from pathlib import Path
+from dataclasses import replace
+import json
+from concurrent.futures import ThreadPoolExecutor
+
+from .binary import BinaryImage, apply_binary_evidence, load_binary_image
+from .cache import (JsonCache, build_cache_key, deserialize_parse_artifact, serialize_parse_artifact,
+                    build_analysis_cache_key, build_analysis_cache_key_from_fingerprints, serialize_analysis_snapshot, restore_analysis_snapshot, serialize_parse_artifact_light)
+from .config import CseqConfig
+from .compile_db import CompilationDatabase
+from .cpu_rules import apply_cpu_rules
+from .linker import LinkerMapIndex, apply_linker_map, parse_linker_map
+from .dependencies import include_dependency_fingerprint
+from .dwarf import DwarfLineIndex, apply_dwarf_evidence, load_dwarf_lines
+from .model import ProjectIndex, TranslationUnit
+from .parser import ParserBackend
+from .hybrid_parser import HybridParser
+from .plugin import PluginManager, PluginRunResult
+from .scanner import scan_sources
+from .source_index import SourceFingerprintIndex
+from .sequence import resolve_direct_calls
+from .static_store import StaticSymbolStore
+from .valueflow import bind_direct_call_arguments, resolve_indirect_calls
+
+
+class Project:
+    def __init__(
+        self,
+        root: str | Path,
+        *,
+        parser: ParserBackend | None = None,
+        config: CseqConfig | str | Path | None = None,
+        configuration: str | None = None,
+        defines: tuple[str, ...] = (),
+        cache_dir: str | Path | None = None,
+        use_cache: bool = True,
+        compile_db: str | Path | None = None,
+        linker_map: str | Path | None = None,
+        binary: str | Path | None = None,
+        dwarf: str | Path | None = None,
+        plugins: PluginManager | None = None,
+        jobs: int = 1,
+        static_store: str | Path | None = None,
+    ) -> None:
+        self.root = Path(root).resolve()
+        self.parser = parser or HybridParser()
+        if isinstance(config, (str, Path)):
+            self.config = CseqConfig.load(config)
+        else:
+            self.config = config or CseqConfig.empty()
+        self.configuration = configuration
+        self.defines = self.config.defines_for(configuration, defines)
+        self.use_cache = use_cache
+        parser_cache_root = Path(cache_dir) if cache_dir is not None else (self.root / ".cseq" / "cache" / "parser")
+        self.cache = JsonCache(parser_cache_root) if use_cache else None
+        # When functions are persisted in StaticSymbolStore, keep a lightweight
+        # artifact sidecar without the giant function array. This lets warm runs
+        # restore calls/value-flow/guards without json-loading all symbols again.
+        # Lightweight non-function parse artifacts are stored atomically in the
+        # Static Symbol Store TU table, so symbol/artifact reuse cannot drift.
+        self.static_artifact_hits = 0
+        self.static_artifact_misses = 0
+        analysis_root = (parser_cache_root.parent / "analysis") if parser_cache_root.name == "parser" else (parser_cache_root / "_analysis")
+        self.analysis_cache = JsonCache(analysis_root) if use_cache else None
+        self.cache_hits = 0
+        self.cache_misses = 0
+        self.analysis_cache_hits = 0
+        self.analysis_cache_misses = 0
+        db_path = Path(compile_db).resolve() if compile_db else (self.root / "compile_commands.json")
+        self.compile_db = CompilationDatabase.load(db_path) if db_path.is_file() else None
+        self.linker_map: LinkerMapIndex | None = parse_linker_map(linker_map) if linker_map is not None else None
+        self.binary_image: BinaryImage | None = load_binary_image(binary) if binary is not None else None
+        self.dwarf_index: DwarfLineIndex | None = load_dwarf_lines(dwarf) if dwarf is not None else None
+        self.plugins = plugins if plugins is not None else PluginManager.discover(self.config.plugins_enabled)
+        self.jobs = max(1, int(jobs))
+        self.plugin_result = PluginRunResult(diagnostics=list(self.plugins.diagnostics))
+        self.fast_artifacts = 0
+        self.semantic_artifacts = 0
+        resolved_static_store = static_store
+        if isinstance(static_store, str) and static_store == "auto":
+            resolved_static_store = self.root / ".cseq" / "static" / "symbols.db"
+        self.static_symbol_store = StaticSymbolStore(resolved_static_store) if resolved_static_store is not None else None
+        self.static_store_path = Path(resolved_static_store).resolve() if resolved_static_store is not None else None
+        source_index_enabled = self.cache is not None or self.static_symbol_store is not None
+        self.source_index_path = self.root / ".cseq" / "source" / "index.db"
+        self.source_index = SourceFingerprintIndex(self.source_index_path) if source_index_enabled else None
+        self.static_store_hits = 0
+        self.static_store_misses = 0
+        self.static_store_removed_tus = 0
+        if self.static_symbol_store is not None:
+            self.static_symbol_store.ensure_project(self.root)
+        self._closed = False
+
+    def close(self) -> None:
+        """Release persistent SQLite handles owned by this Project.
+
+        Windows does not permit deleting an open SQLite database, so resource
+        lifetime must be explicit rather than relying on POSIX unlink behavior.
+        Calling close more than once is safe.
+        """
+        if self._closed:
+            return
+        self._closed = True
+        if self.source_index is not None:
+            try:
+                self.source_index.close()
+            except Exception:
+                pass
+        if self.static_symbol_store is not None:
+            try:
+                self.static_symbol_store.close()
+            except Exception:
+                pass
+
+    def __enter__(self) -> "Project":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        try:
+            self.close()
+        except Exception:
+            pass
+
+    def index(self) -> ProjectIndex:
+        files = scan_sources(self.root, fingerprint_index=self.source_index)
+        index = ProjectIndex(root=self.root, source_files=files, static_symbol_store=self.static_symbol_store)
+        config_define_args = tuple(f"-D{x}" for x in self.defines)
+        c_sources = [source for source in files if source.suffix == ".c"]
+        static_tu_state = (
+            self.static_symbol_store.translation_unit_state_raw()
+            if self.static_symbol_store is not None else {}
+        )
+
+        def parse_one(source):
+            compile_cmd = self.compile_db.lookup(source.path) if self.compile_db is not None else None
+            base_args = compile_cmd.arguments if compile_cmd is not None else ()
+            # Build the cheap TU identity first. Source text is opened only
+            # after persistent source/dependency and parser/static-store reuse
+            # checks miss. Warm unchanged TUs therefore do not open the .c body.
+            tu = TranslationUnit(
+                source=source,
+                arguments=tuple(base_args) + config_define_args,
+                working_directory=compile_cmd.directory if compile_cmd is not None else self.root,
+                configuration_name=self.configuration,
+                source_text=None,
+            )
+            # A TU fingerprint is required for safe Static Symbol Store reuse.
+            # Parser Cache already needs the same dependency walk, so compute it
+            # once and share it whenever either cache/store is enabled.
+            fingerprint = None
+            if self.cache is not None or self.static_symbol_store is not None:
+                dependency_hash = include_dependency_fingerprint(tu, self.source_index)
+                fingerprint = build_cache_key(
+                    tu, parser_backend=self.parser.name, include_dependency_hash=dependency_hash
+                )
+            state = static_tu_state.get(tu.source.project_relative_path)
+            store_reusable = bool(
+                state is not None and fingerprint is not None and state[0] == fingerprint
+            )
+            if store_reusable:
+                try:
+                    light = json.loads(state[1])
+                except (TypeError, json.JSONDecodeError):
+                    light = None
+                if isinstance(light, dict) and light:
+                    return deserialize_parse_artifact(
+                        light, tu, include_functions=False
+                    ), False, fingerprint, True, True
+            if self.cache is None:
+                try:
+                    source_text = source.path.read_text(encoding="utf-8", errors="replace")
+                except OSError:
+                    source_text = None
+                artifact = self.parser.parse(replace(tu, source_text=source_text))
+                return artifact, False, fingerprint, store_reusable, False
+            cached = self.cache.get(fingerprint)
+            if cached is not None:
+                return deserialize_parse_artifact(
+                    cached, tu, include_functions=not store_reusable
+                ), True, fingerprint, store_reusable, False
+            try:
+                source_text = source.path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                source_text = None
+            artifact = self.parser.parse(replace(tu, source_text=source_text))
+            payload = serialize_parse_artifact(artifact)
+            self.cache.put(fingerprint, payload)
+            return artifact, False, fingerprint, store_reusable, False
+
+        # Stream parse results into the ProjectIndex. In out-of-core mode,
+        # function metadata is written to SQLite immediately and released from
+        # each parse artifact before moving to the next TU. This prevents a
+        # temporary all-TU in-memory list from defeating the static store.
+        parser_fingerprints: list[str] = []
+        fast_count = 0
+        semantic_count = 0
+        static_dirty_tus = 0
+
+        def consume(result) -> None:
+            nonlocal fast_count, semantic_count, static_dirty_tus
+            artifact, cache_hit, parser_fingerprint, store_reusable, static_artifact_hit = result
+            if static_artifact_hit:
+                self.static_artifact_hits += 1
+            elif self.static_symbol_store is not None:
+                self.static_artifact_misses += 1
+            if parser_fingerprint is not None:
+                parser_fingerprints.append(parser_fingerprint)
+            if self.cache is not None:
+                if cache_hit:
+                    self.cache_hits += 1
+                else:
+                    self.cache_misses += 1
+            if artifact.parser_backend.startswith(("fast-lexical", "tree-sitter-c")):
+                fast_count += 1
+            elif artifact.parser_backend.startswith("clang-json"):
+                semantic_count += 1
+            else:
+                # Unknown parser strategies must not be silently classified as
+                # semantic evidence. Preserve conservative accounting.
+                semantic_count += 1
+            if self.static_symbol_store is not None:
+                source_path = artifact.translation_unit.source.project_relative_path
+                if store_reusable:
+                    self.static_store_hits += 1
+                else:
+                    self.static_store_misses += 1
+                    if parser_fingerprint is None:
+                        raise RuntimeError("static store requires a TU fingerprint")
+                    self.static_symbol_store.replace_translation_unit(
+                        source_path, parser_fingerprint, artifact.functions,
+                        serialize_parse_artifact_light(artifact),
+                    )
+                    static_dirty_tus += 1
+                    if static_dirty_tus >= 250:
+                        # Keep cold out-of-core builds in bounded SQLite
+                        # transactions. The expensive name index remains deferred
+                        # until the final commit.
+                        self.static_symbol_store.flush()
+                        static_dirty_tus = 0
+                artifact.functions = []
+            # source_text is an ephemeral per-TU I/O optimization only. Do not
+            # retain project source bodies through parse_artifacts.
+            if artifact.translation_unit.source_text is not None:
+                artifact.translation_unit = replace(artifact.translation_unit, source_text=None)
+            index.parse_artifacts.append(artifact)
+            index.callsites.extend(artifact.callsites)
+            index.value_slots.extend(artifact.value_slots)
+            index.flow_edges.extend(artifact.flow_edges)
+            index.guarded_regions.extend(artifact.guarded_regions)
+
+        if self.jobs > 1 and len(c_sources) > 1:
+            with ThreadPoolExecutor(max_workers=self.jobs, thread_name_prefix="cseq-parse") as pool:
+                for result in pool.map(parse_one, c_sources):
+                    consume(result)
+        else:
+            for source in c_sources:
+                consume(parse_one(source))
+
+        if self.static_symbol_store is not None:
+            current_c_sources = {s.project_relative_path for s in c_sources}
+            self.static_store_removed_tus = self.static_symbol_store.remove_missing_translation_units(
+                current_c_sources
+            )
+            self.static_symbol_store.commit()
+        analysis_key = None
+        restored = False
+        if self.analysis_cache is not None:
+            analysis_key = build_analysis_cache_key_from_fingerprints(parser_fingerprints, self.config, self.defines)
+            cached_analysis = self.analysis_cache.get(analysis_key)
+            if cached_analysis is not None:
+                restored = restore_analysis_snapshot(index, cached_analysis)
+        if restored:
+            self.analysis_cache_hits += 1
+        else:
+            if self.analysis_cache is not None:
+                self.analysis_cache_misses += 1
+            resolve_direct_calls(index)
+            bind_direct_call_arguments(index)
+            resolve_indirect_calls(index)
+            apply_cpu_rules(index, self.config, self.defines)
+            if self.analysis_cache is not None and analysis_key is not None:
+                self.analysis_cache.put(analysis_key, serialize_analysis_snapshot(index))
+        if self.linker_map is not None:
+            apply_linker_map(index, self.linker_map)
+            # Linker evidence can turn an ambiguous direct call into an exact one;
+            # bind its arguments now without mutating the cached source-only analysis.
+            bind_direct_call_arguments(index)
+        if self.binary_image is not None:
+            apply_binary_evidence(index, self.binary_image)
+        if self.dwarf_index is not None:
+            apply_dwarf_evidence(index, self.dwarf_index)
+        if self.plugins.plugins:
+            self.plugin_result = self.plugins.run_semantic_hints(functions=index.functions(), callsites=index.callsites)
+        else:
+            self.plugin_result = PluginRunResult(diagnostics=list(self.plugins.diagnostics))
+        self.fast_artifacts = fast_count
+        self.semantic_artifacts = semantic_count
+        if self.source_index is not None:
+            self.source_index.commit()
+        return index

cseq/query.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from .model import CallTargetKind, ProjectIndex
+from .trace_analysis import reconstruct_static_paths
+
+
+def query_calls(index: ProjectIndex, function_name: str) -> dict:
+    incoming = []
+    outgoing = []
+    functions = {f.qualified_id: f for f in index.functions()}
+    for c in index.callsites:
+        target_names = []
+        tids = set(c.target_function_ids)
+        if c.target_function_id:
+            tids.add(c.target_function_id)
+        for tid in tids:
+            f = functions.get(tid)
+            if f:
+                target_names.append(f.name)
+        if c.caller_name == function_name:
+            outgoing.append({
+                "raw": c.raw_text,
+                "resolution": c.target_kind.value,
+                "targets": sorted(set(target_names)),
+                "line": c.source_range.start_line,
+                "source": c.source_path,
+            })
+        if function_name in target_names:
+            incoming.append({
+                "caller": c.caller_name,
+                "raw": c.raw_text,
+                "resolution": c.target_kind.value,
+                "line": c.source_range.start_line,
+                "source": c.source_path,
+            })
+    return {"function": function_name, "incoming": incoming, "outgoing": outgoing}
+
+
+def query_unresolved(index: ProjectIndex) -> list[dict]:
+    unresolved = []
+    for c in index.callsites:
+        if c.target_kind in {CallTargetKind.UNKNOWN, CallTargetKind.INDIRECT_POSSIBLE, CallTargetKind.INTERNAL_POSSIBLE} or c.unknown_possible:
+            unresolved.append({
+                "caller": c.caller_name,
+                "raw": c.raw_text,
+                "resolution": c.target_kind.value,
+                "unknown_possible": c.unknown_possible,
+                "candidate_count": len(c.target_function_ids),
+                "source": c.source_path,
+                "line": c.source_range.start_line,
+            })
+    return unresolved
+
+
+def query_paths(index: ProjectIndex, start: str, end: str) -> dict:
+    result = reconstruct_static_paths(index, start, end)
+    return {
+        "start": start,
+        "end": end,
+        "state": result.state.value,
+        "paths": [list(p) for p in result.paths],
+    }

cseq/resources/README.md

@@ -0,0 +1,171 @@
+# cseq
+
+Cソースツリーから関数呼出し・分岐・コールバック・実行時markerを解析し、PlantUMLとインタラクティブHTMLを生成するシーケンス解析ツールです。
+
+初回リリースは **Windows + Python 3.11以上** を主対象とします。Tree-sitterを高速な構文フロントエンドとして使用し、型・macro・関数ポインタ等で意味解析が必要な翻訳単位だけClangへ送ります。
+
+## 主な機能
+
+- ディレクトリ以下の `.c` / `.h` を再帰走査
+- entry関数からのシーケンス生成
+- 同名`static`関数を区別
+- 外部未定義関数を内部calleeとして誤認しない
+- `#if/#ifdef`、macro、compiler extensionのfail-soft解析
+- 関数ポインタ、callback、dispatch tableの候補解決
+- if/else、loop、無限loop、switch/case/defaultをSequence IRへ保持
+- RTOS風の無限loopを有限表現
+- CPU group / build configurationを外部設定可能
+- runtime markerログの読込み、CPU/task evidence、静的経路との照合
+- marker自動注入: source直接書換え / Overlay（原本非変更）の両方式
+- PlantUML出力
+- HTML UI: 検索、viewport、LOD、Canvas表示、path query、trace diff、marker suggestion
+- 大きいruntime trace向けbundle/range出力
+- optional linker map / binary symbol / DWARF evidence
+
+## Windowsでの導入
+
+PowerShell例です。
+
+```powershell
+py -3.12 -m venv .venv
+.\.venv\Scripts\Activate.ps1
+python -m pip install --upgrade pip
+pip install "cseq[parser-tree-sitter]"
+```
+
+ローカルwheelから導入する場合:
+
+```powershell
+pip install .\cseq-0.0.1-py3-none-any.whl[parser-tree-sitter]
+```
+
+### Clang
+
+単純なC構文だけならTree-sitterで解析できますが、macro、関数ポインタ、型曖昧性、compiler extension等のselective semantic refinementにはClangを推奨します。
+
+```powershell
+winget install --id LLVM.LLVM --exact
+```
+
+確認:
+
+```powershell
+cseq doctor
+```
+
+`current_execution_mode` が `tree-sitter+clang` なら両backendを利用できます。
+
+## 最短利用例
+
+```powershell
+cseq analyze C:\path\to\project --entry main --json
+cseq plantuml C:\path\to\project --entry main --output sequence.puml
+cseq html C:\path\to\project --entry main --output sequence.html --mode auto
+```
+
+HTMLは小規模時はportable単一HTML、中規模以上ではbundle/rangeへ切替できます。
+
+```powershell
+cseq serve . --port 8000
+```
+
+range/bundleをブラウザで確認する際はローカルHTTP配信を推奨します。
+
+## プロジェクト設定
+
+```powershell
+cseq config init C:\path\to\project
+cseq config validate C:\path\to\project\cseq.toml
+```
+
+`cseq.toml`でentry、compile database、CPU group、build configuration等をプロジェクト固有名に合わせて設定します。cseq本体は特定企業・CPU名称を前提にしません。
+
+`compile_commands.json` がある場合はClang semantic解析の精度向上に利用できます。
+
+## Runtime marker
+
+### 直接書換え
+
+```powershell
+cseq marker apply .\src\worker.c --contains "rx=%d" --id RX01 --manifest .\rx01.json
+```
+
+manifestにbefore/after hashを保存し、安全条件が成立するときだけundoできます。
+
+```powershell
+cseq marker undo .\src\worker.c --manifest .\rx01.json
+```
+
+### Overlay（原本非変更）
+
+```powershell
+cseq marker apply .\src\worker.c `
+  --contains "rx=%d" `
+  --id RX01 `
+  --manifest .\rx01-overlay.json `
+  --overlay-output .\cseq-overlay\worker.c
+```
+
+元sourceは変更されず、marker注入済みコピーだけを生成します。
+
+## Runtime log
+
+marker形式:
+
+```text
+[[CSEQ:RX01]] message...
+```
+
+ログ形式は固定ではなくpatternで指定できます。
+
+```powershell
+cseq trace import run.log `
+  --pattern "[{cpu}] {time} {message}" `
+  --time-format "%H:%M:%S.%f" `
+  --json
+```
+
+CPU間で時刻が隣接しているだけでは因果関係とはみなしません。marker、payload、静的path等のevidenceを分離して扱います。
+
+## HTML UI
+
+```powershell
+cseq html C:\path\to\project `
+  --entry main `
+  --runtime-log run.log `
+  --runtime-pattern "[{cpu}] {time} {message}" `
+  --runtime-time-format "%H:%M:%S.%f" `
+  --output sequence.html `
+  --mode auto
+```
+
+UIは大量eventを全DOM化せず、viewport/LOD/Canvas/range bundleを使います。初回リリースでは深い性能チューニングより、汎用性・正しさ・必要UIを優先します。
+
+## 設計上の原則
+
+- Silent Dropしない。解析不能領域はPARTIAL/OPAQUE等で明示する。
+- static possible と runtime observed を混同しない。
+- CPUのstatic/build/runtime evidenceを分離する。
+- cross-CPU timestamp adjacencyだけで因果を断定しない。
+- Tree-sitterとClangは代替関係ではなく、Fast Syntax + Selective Semantic Refinementとして併用する。
+
+## 開発時検証
+
+```powershell
+python -m pytest -q
+```
+
+Windows実機で全テスト、Tree-sitter/Clang統合acceptance、clean wheel installを確認してからrelease候補を作成します。
+
+## 同梱ドキュメントの出力
+
+インストール済みwheelから、README・設計書（清書版）・実装/検証成績書を1コマンドで取り出せます。
+
+```powershell
+cseq docs readme --output README.md
+cseq docs design --output シーケンス解析_設計書.md
+cseq docs report --output 実装・検証成績書.html
+cseq docs all --output-dir .\cseq-docs
+```
+
+`--output`を省略するとREADME/設計書/成績書の内容を標準出力します。`docs all` は3点を一括で出力します。