codeanalyzer-python 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

codeanalyzer/__main__.py

@@ -1,11 +1,16 @@
-from contextlib import nullcontext
-import sys
-import typer
-from typing import Optional, Annotated
 from pathlib import Path
-from codeanalyzer.utils import _set_log_level
-from codeanalyzer.utils import logger
+from typing import Annotated, Optional
+from enum import Enum
+
+import typer
+
 from codeanalyzer.core import AnalyzerCore
+from codeanalyzer.utils import _set_log_level, logger
+
+
+class OutputFormat(str, Enum):
+    JSON = "json"
+    MSGPACK = "msgpack"
 
 
 def main(
@@ -16,6 +21,15 @@ def main(
         Optional[Path],
         typer.Option("-o", "--output", help="Output directory for artifacts."),
     ] = None,
+    format: Annotated[
+        OutputFormat,
+        typer.Option(
+            "-f",
+            "--format",
+            help="Output format: json or msgpack.",
+            case_sensitive=False,
+        ),
+    ] = OutputFormat.JSON,
     analysis_level: Annotated[
         int,
         typer.Option("-a", "--analysis-level", help="1: symbol table, 2: call graph."),
@@ -57,16 +71,42 @@ def main(
         input, analysis_level, using_codeql, rebuild_analysis, cache_dir, clear_cache
     ) as analyzer:
         artifacts = analyzer.analyze()
-        print_stream = sys.stdout
-        stream_context = nullcontext(print_stream)
 
-        if output is not None:
+        # Handle output based on format
+        if output is None:
+            # Output to stdout (only for JSON)
+            if format == OutputFormat.JSON:
+                print(artifacts.model_dump_json(separators=(",", ":")))
+            else:
+                logger.error(
+                    f"Format '{format.value}' requires an output directory (use -o/--output)"
+                )
+                raise typer.Exit(code=1)
+        else:
+            # Output to file
             output.mkdir(parents=True, exist_ok=True)
-            output_file = output / "analysis.json"
-            stream_context = output_file.open("w")
+            _write_output(artifacts, output, format)
+
+
+def _write_output(artifacts, output_dir: Path, format: OutputFormat):
+    """Write artifacts to file in the specified format."""
+    if format == OutputFormat.JSON:
+        output_file = output_dir / "analysis.json"
+        # Use Pydantic's json() with separators for compact output
+        json_str = artifacts.model_dump_json(indent=None)
+        with output_file.open("w") as f:
+            f.write(json_str)
+        logger.info(f"Analysis saved to {output_file}")
 
-        with stream_context as f:
-            print(artifacts.model_dump_json(indent=4), file=f)
+    elif format == OutputFormat.MSGPACK:
+        output_file = output_dir / "analysis.msgpack"
+        msgpack_data = artifacts.to_msgpack_bytes()
+        with output_file.open("wb") as f:
+            f.write(msgpack_data)
+        logger.info(f"Analysis saved to {output_file}")
+        logger.info(
+            f"Compression ratio: {artifacts.get_compression_ratio():.1%} of JSON size"
+        )
 
 
 app = typer.Typer(

codeanalyzer/core.py

@@ -1,19 +1,16 @@
 import hashlib
 import os
-from pdb import set_trace
 import shutil
 import subprocess
-from pathlib import Path
 import sys
-from typing import Any, Dict, Union, Optional
-from codeanalyzer.utils import logger
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
 
 from codeanalyzer.schema.py_schema import PyApplication, PyModule
 from codeanalyzer.semantic_analysis.codeql import CodeQLLoader
-from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import (
-    CodeQLExceptions,
-)
+from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
 from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
+from codeanalyzer.utils import logger
 
 
 class AnalyzerCore:
@@ -290,11 +287,7 @@ class AnalyzerCore:
 
     def analyze(self) -> PyApplication:
         """Return the path to the CodeQL database."""
-        return (
-            PyApplication.builder()
-            .with_symbol_table(self._build_symbol_table())
-            .build()
-        )
+        return PyApplication.builder().symbol_table(self._build_symbol_table()).build()
 
     def _compute_checksum(self, root: Path) -> str:
         """Compute SHA256 checksum of all Python source files in a project directory. If somethings changes, the

codeanalyzer/schema/__init__.py

@@ -1,13 +1,13 @@
 from .py_schema import (
     PyApplication,
-    PyImport,
+    PyCallable,
+    PyCallableParameter,
+    PyClass,
+    PyClassAttribute,
     PyComment,
+    PyImport,
     PyModule,
-    PyClass,
     PyVariableDeclaration,
-    PyCallable,
-    PyClassAttribute,
-    PyCallableParameter
 )
 
 __all__ = [
@@ -19,5 +19,5 @@ __all__ = [
     "PyVariableDeclaration",
     "PyCallable",
     "PyClassAttribute",
-    "PyCallableParameter"
+    "PyCallableParameter",
 ]

codeanalyzer/schema/py_schema.py

@@ -20,12 +20,88 @@ This module defines the data models used to represent Python code structures
 for static analysis purposes.
 """
 
+import inspect
 from pathlib import Path
 from typing import Any, Dict, List, Optional
-from typing_extensions import Literal
+import gzip
+
 from pydantic import BaseModel
+from typing_extensions import Literal
+import msgpack
 
-import inspect
+
+def msgpk(cls):
+    """
+    Decorator that adds MessagePack serialization methods to Pydantic models.
+
+    Adds methods:
+        - to_msgpack_bytes() -> bytes: Serialize to compact binary format
+        - from_msgpack_bytes(data: bytes) -> cls: Deserialize from binary format
+        - to_msgpack_dict() -> dict: Convert to msgpack-compatible dict
+        - from_msgpack_dict(data: dict) -> cls: Create instance from msgpack dict
+    """
+
+    def _prepare_for_serialization(obj: Any) -> Any:
+        """Convert objects to serialization-friendly format."""
+        if isinstance(obj, Path):
+            return str(obj)
+        elif isinstance(obj, dict):
+            return {
+                _prepare_for_serialization(k): _prepare_for_serialization(v)
+                for k, v in obj.items()
+            }
+        elif isinstance(obj, list):
+            return [_prepare_for_serialization(item) for item in obj]
+        elif isinstance(obj, tuple):
+            return tuple(_prepare_for_serialization(item) for item in obj)
+        elif isinstance(obj, set):
+            return [_prepare_for_serialization(item) for item in obj]
+        elif hasattr(obj, "model_dump"):  # Pydantic model
+            return _prepare_for_serialization(obj.model_dump())
+        else:
+            return obj
+
+    def to_msgpack_bytes(self) -> bytes:
+        """Serialize the model to compact binary format using MessagePack + gzip."""
+        data = _prepare_for_serialization(self.model_dump())
+        msgpack_data = msgpack.packb(data, use_bin_type=True)
+        return gzip.compress(msgpack_data)
+
+    @classmethod
+    def from_msgpack_bytes(cls_obj, data: bytes):
+        """Deserialize from MessagePack + gzip binary format."""
+        decompressed_data = gzip.decompress(data)
+        obj_dict = msgpack.unpackb(decompressed_data, raw=False)
+        return cls_obj.model_validate(obj_dict)
+
+    def to_msgpack_dict(self) -> dict:
+        """Convert to msgpack-compatible dictionary format."""
+        return _prepare_for_serialization(self.model_dump())
+
+    @classmethod
+    def from_msgpack_dict(cls_obj, data: dict):
+        """Create instance from msgpack-compatible dictionary."""
+        return cls_obj.model_validate(data)
+
+    def get_msgpack_size(self) -> int:
+        """Get the size of the msgpack serialization in bytes."""
+        return len(self.to_msgpack_bytes())
+
+    def get_compression_ratio(self) -> float:
+        """Get compression ratio compared to JSON."""
+        json_size = len(self.model_dump_json().encode("utf-8"))
+        msgpack_gzip_size = self.get_msgpack_size()
+        return msgpack_gzip_size / json_size if json_size > 0 else 1.0
+
+    # Add methods to the class
+    cls.to_msgpack_bytes = to_msgpack_bytes
+    cls.from_msgpack_bytes = from_msgpack_bytes
+    cls.to_msgpack_dict = to_msgpack_dict
+    cls.from_msgpack_dict = from_msgpack_dict
+    cls.get_msgpack_size = get_msgpack_size
+    cls.get_compression_ratio = get_compression_ratio
+
+    return cls
 
 
 def builder(cls):
@@ -33,7 +109,7 @@ def builder(cls):
     Decorator that generates a builder class for a Pydantic models defined below.
 
     It creates methods like:
-        - with_<fieldname>(value)
+        - <fieldname>(value)
         - build() to instantiate the model
 
     It supports nested builder patterns and is mypy-compatible.
@@ -70,12 +146,12 @@ def builder(cls):
                 setattr(self, f"_{f}", value)
                 return self
 
-            method.__name__ = f"with_{f}"
+            method.__name__ = f"{f}"
             method.__annotations__ = {"value": t, "return": builder_name}
             method.__doc__ = f"Set {f} ({t.__name__})"
             return method
 
-        namespace[f"with_{field}"] = make_method()
+        namespace[f"{field}"] = make_method()
 
     # Create a build method that constructs the model instance using the values set in the builder.
     def build(self):
@@ -92,26 +168,9 @@ def builder(cls):
 
 
 @builder
+@msgpk
 class PyImport(BaseModel):
-    """Represents a Python import statement.
-
-    Attributes:
-        module (str): The name of the module being imported.
-        name (str): The name of the imported entity (e.g., function, class).
-        alias (Optional[str]): An optional alias for the imported entity.
-        start_line (int): The line number where the import statement starts.
-        end_line (int): The line number where the import statement ends.
-        start_column (int): The starting column of the import statement.
-        end_column (int): The ending column of the import statement.
-
-    Example:
-        - import numpy as np will be represented as:
-            PyImport(module="numpy", name="np", alias="np", start_line=1, end_line=1, start_column=0, end_column=16)
-        - from math import sqrt will be represented as:
-            PyImport(module="math", name="sqrt", alias=None, start_line=2, end_line=2, start_column=0, end_column=20
-        - from os.path import join as path_join will be represented as:
-            PyImport(module="os.path", name="path_join", alias="join", start_line=3, end_line=3, start_column=0, end_column=30)
-    """
+    """Represents a Python import statement."""
 
     module: str
     name: str
@@ -123,18 +182,9 @@ class PyImport(BaseModel):
 
 
 @builder
+@msgpk
 class PyComment(BaseModel):
-    """
-    Represents a Python comment.
-
-    Attributes:
-        content (str): The actual comment string (without the leading '#').
-        start_line (int): The line number where the comment starts.
-        end_line (int): The line number where the comment ends (same as start_line for single-line comments).
-        start_column (int): The starting column of the comment.
-        end_column (int): The ending column of the comment.
-        is_docstring (bool): Whether this comment is actually a docstring (triple-quoted string).
-    """
+    """Represents a Python comment."""
 
     content: str
     start_line: int = -1
@@ -145,20 +195,9 @@ class PyComment(BaseModel):
 
 
 @builder
+@msgpk
 class PySymbol(BaseModel):
-    """
-    Represents a symbol used or declared in Python code.
-
-    Attributes:
-        name (str): The name of the symbol (e.g., 'x', 'self.x', 'os.path').
-        scope (Literal['local', 'nonlocal', 'global', 'class', 'module']): The scope where the symbol is accessed.
-        kind (Literal['variable', 'parameter', 'attribute', 'function', 'class', 'module']): The kind of symbol.
-        type (Optional[str]): Inferred or annotated type, if available.
-        qualified_name (Optional[str]): Fully qualified name (e.g., 'self.x', 'os.path.join').
-        is_builtin (bool): Whether this is a Python builtin.
-        lineno (int): Line number where the symbol is accessed or declared.
-        col_offset (int): Column offset.
-    """
+    """Represents a symbol used or declared in Python code."""
 
     name: str
     scope: Literal["local", "nonlocal", "global", "class", "module"]
@@ -171,11 +210,9 @@ class PySymbol(BaseModel):
 
 
 @builder
+@msgpk
 class PyVariableDeclaration(BaseModel):
-    """Represents a Python variable declaration.
-
-    Attributes:
-    """
+    """Represents a Python variable declaration."""
 
     name: str
     type: Optional[str]
@@ -189,18 +226,9 @@ class PyVariableDeclaration(BaseModel):
 
 
 @builder
+@msgpk
 class PyCallableParameter(BaseModel):
-    """Represents a parameter of a Python callable (function/method).
-
-    Attributes:
-        name (str): The name of the parameter.
-        type (str): The type of the parameter.
-        default_value (str): The default value of the parameter, if any.
-        start_line (int): The line number where the parameter is defined.
-        end_line (int): The line number where the parameter definition ends.
-        start_column (int): The column number where the parameter starts.
-        end_column (int): The column number where the parameter ends.
-    """
+    """Represents a parameter of a Python callable (function/method)."""
 
     name: str
     type: Optional[str] = None
@@ -212,10 +240,9 @@ class PyCallableParameter(BaseModel):
 
 
 @builder
+@msgpk
 class PyCallsite(BaseModel):
-    """
-    Represents a Python call site (function or method invocation) with contextual metadata.
-    """
+    """Represents a Python call site (function or method invocation) with contextual metadata."""
 
     method_name: str
     receiver_expr: Optional[str] = None
@@ -231,26 +258,9 @@ class PyCallsite(BaseModel):
 
 
 @builder
+@msgpk
 class PyCallable(BaseModel):
-    """Represents a Python callable (function/method).
-
-    Attributes:
-        name (str): The name of the callable.
-        signature (str): The fully qualified name of the callable (e.g., module.function_name).
-        docstring (PyComment): The docstring of the callable.
-        decorators (List[str]): List of decorators applied to the callable.
-        parameters (List[PyCallableParameter]): List of parameters for the callable.
-        return_type (Optional[str]): The type of the return value, if specified.
-        code (str): The actual code of the callable.
-        start_line (int): The line number where the callable is defined.
-        end_line (int): The line number where the callable definition ends.
-        code_start_line (int): The line number where the code block starts.
-        accessed_symbols (List[str]): Symbols accessed within the callable.
-        call_sites (List[str]): Call sites of this callable.
-        is_entrypoint (bool): Whether this callable is an entry point.
-        local_variables (List[PyVariableDeclaration]): Local variables within the callable.
-        cyclomatic_complexity (int): Cyclomatic complexity of the callable.
-    """
+    """Represents a Python callable (function/method)."""
 
     name: str
     path: str
@@ -274,16 +284,9 @@ class PyCallable(BaseModel):
 
 
 @builder
+@msgpk
 class PyClassAttribute(BaseModel):
-    """Represents a Python class attribute.
-
-    Attributes:
-        name (str): The name of the attribute.
-        type (str): The type of the attribute.
-        docstring (PyComment): The docstring of the attribute.
-        start_line (int): The line number where the attribute is defined.
-        end_line (int): The line number where the attribute definition ends.
-    """
+    """Represents a Python class attribute."""
 
     name: str
     type: Optional[str] = None
@@ -293,20 +296,9 @@ class PyClassAttribute(BaseModel):
 
 
 @builder
+@msgpk
 class PyClass(BaseModel):
-    """Represents a Python class.
-
-    Attributes:
-        name (str): The name of the class.
-        signature (str): The fully qualified name of the class (e.g., module.class_name).
-        docstring (PyComment): The docstring of the class.
-        base_classes (List[str]): List of base class names.
-        methods (Dict[str, PyCallable]): Mapping of method names to their callable representations.
-        attributes (Dict[str, PyClassAttribute]): Mapping of attribute names to their variable declarations.
-        inner_classes (Dict[str, "PyClass"]): Mapping of inner class names to their class representations.
-        start_line (int): The line number where the class definition starts.
-        end_line (int): The line number where the class definition ends.
-    """
+    """Represents a Python class."""
 
     name: str
     signature: str  # e.g., module.class_name
@@ -325,18 +317,9 @@ class PyClass(BaseModel):
 
 
 @builder
+@msgpk
 class PyModule(BaseModel):
-    """Represents a Python module.
-
-    Attributes:
-        file_path (str): The file path of the module.
-        module_name (str): The name of the module (e.g., module.submodule).
-        imports (List[PyImport]): List of import statements in the module.
-        comments (List[PyComment]): List of comments in the module.
-        classes (Dict[str, PyClass]): Mapping of class names to their class representations.
-        functions (Dict[str, PyCallable]): Mapping of function names to their callable representations.
-        variables (List[PyVariableDeclaration]): List of variable declarations in the module.
-    """
+    """Represents a Python module."""
 
     file_path: str
     module_name: str
@@ -348,13 +331,8 @@ class PyModule(BaseModel):
 
 
 @builder
+@msgpk
 class PyApplication(BaseModel):
-    """Represents a Python application.
-
-    Attributes:
-        name (str): The name of the application.
-        version (str): The version of the application.
-        description (str): A brief description of the application.
-    """
+    """Represents a Python application."""
 
     symbol_table: dict[Path, PyModule]

codeanalyzer/semantic_analysis/codeql/__init__.py

@@ -19,8 +19,8 @@ CodeQL package
 """
 
 from .codeql_analysis import CodeQL
-from .codeql_query_runner import CodeQLQueryRunner
-from .codeql_loader import CodeQLLoader
 from .codeql_exceptions import CodeQLExceptions
+from .codeql_loader import CodeQLLoader
+from .codeql_query_runner import CodeQLQueryRunner
 
 __all__ = ["CodeQL", "CodeQLQueryRunner", "CodeQLLoader", "CodeQLExceptions"]

codeanalyzer/semantic_analysis/codeql/codeql_analysis.py

@@ -21,9 +21,10 @@ for Python projects and execute queries against them.
 """
 
 from pathlib import Path
+from typing import Union
+
 from networkx import DiGraph
 from pandas import DataFrame
-from typing import Union
 
 from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQueryRunner
 
@@ -130,4 +131,3 @@ class CodeQL:
         Returns:
             DiGraph: A directed graph representing the call graph of the application.
         """
-        pass

codeanalyzer/semantic_analysis/codeql/codeql_loader.py

@@ -1,9 +1,9 @@
 import platform
-import requests
 import zipfile
 from pathlib import Path
+
+import requests
 from codeanalyzer.utils import logger
-from tqdm import tqdm
 
 
 class CodeQLLoader:
@@ -43,22 +43,11 @@ class CodeQLLoader:
         logger.info(f"Downloading CodeQL CLI from {download_url}")
         with requests.get(download_url, stream=True) as r:
             r.raise_for_status()
-            total_size = int(r.headers.get("content-length", 0))
             block_size = 8192  # 8KB
 
-            with (
-                open(archive_path, "wb") as f,
-                tqdm(
-                    total=total_size,
-                    unit="B",
-                    unit_scale=True,
-                    unit_divisor=1024,
-                    desc="Downloading CodeQL",
-                ) as bar,
-            ):
+            with open(archive_path, "wb") as f:
                 for chunk in r.iter_content(chunk_size=block_size):
                     f.write(chunk)
-                    bar.update(len(chunk))
 
         extract_dir = temp_dir / filename.replace(".zip", "")
         extract_dir.mkdir(exist_ok=True)

codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py

@@ -20,11 +20,12 @@ This module provides functionality to run CodeQL queries against CodeQL database
 and process the results.
 """
 
+import shlex
 import subprocess
 import tempfile
 from pathlib import Path
-import shlex
 from typing import List
+
 import pandas as pd
 from pandas import DataFrame