chronicle-devkit 0.1.0__py3-none-any.whl

chronicle/scanning/scanners/django_migrations.py

@@ -0,0 +1,156 @@
+from pathlib import Path
+import ast
+from datetime import datetime,timezone
+from .base import Scanner
+from chronicle.scanning.context import ScanContext
+from chronicle.storage.observations import Observation
+from .django_model import MigrationOperation
+
+class DjangoMigrationScanner(Scanner):
+    def __init__(self, project_root: Path) -> None:
+        super().__init__(project_root)
+        
+    def scan(self, contexts: list[ScanContext]) -> list[Observation]:
+        observation = []
+        last_migration = None
+        last_migration_name = None
+        last_app = None
+        for context in contexts:
+            last_migration = context.get_state("django_migrations",".last_migration")
+            if last_migration:
+                break
+        for migration_file in self._find_migrations():
+            migration_name = migration_file.stem
+            app_label = migration_file.parent.parent.name
+            if last_migration:
+                last_app , last_migration_name = last_migration.split(":",1,)
+                if app_label == last_app:
+                    if self._migration_num(migration_name) <= self._migration_num(last_migration_name):
+                        continue
+            observation.append(
+                self._parse_migration(
+                    migration_file
+                )
+            )
+            
+        return observation
+        
+    def _find_migrations(self) -> list[Path]:
+        migration_files = []
+        for path in self.project_root.rglob(
+            "migrations/*.py"
+        ):
+            if self._is_ignored(path):
+                continue
+            migration_files.append(path)
+        return migration_files
+    
+    def _is_ignored(self, path:Path) -> bool:
+        IGNORED_DIRECTORIES = {
+                    ".git",
+                    ".venv",
+                    "venv",
+                    "env",
+                    "node_modules",
+                    "__pycache__",
+                }
+        return any(part in IGNORED_DIRECTORIES for part in path.parts)
+    
+    def _parse_migration(self,migration_file:Path) -> Observation:
+        source = migration_file.read_text(encoding="utf-8")
+        migration_name = migration_file.stem
+        app_label = migration_file.parent.parent.name
+        tree = ast.parse(source)
+        dependencies = []
+        parsed_operations = []
+        
+        for node in ast.walk(tree):
+            if isinstance(node,ast.ClassDef):
+                if node.name == "Migration":
+                    migration_class = node
+                    for statement in migration_class.body:
+                        if isinstance(statement,ast.Assign):
+                            target = statement.targets[0]
+                            if isinstance(target, ast.Name):
+                                if target.id == "dependencies":
+                                    dependencies = ast.literal_eval(statement.value)
+                                elif target.id == "operations":
+                                    if not isinstance(
+                                        statement.value,
+                                        ast.List,
+                                    ):
+                                        continue
+
+                                    parsed_operations.extend(
+                                        self.get_operations(
+                                            statement.value
+                                        )
+                                    )
+                                    
+        return Observation(
+            source="django",
+            type="migration",
+            external_id=f"{app_label}:{migration_name}",
+            timestamp=datetime.now(timezone.utc),
+            data={
+                "app": app_label,
+                "name": migration_name,
+                "dependencies": dependencies,
+                "operations": parsed_operations,
+            },
+        )
+        
+    def get_operations(
+        self,
+        operations_node: ast.List,
+    ) -> list[MigrationOperation]:
+
+        operations = []
+
+        for operation_node in operations_node.elts:
+
+            if not isinstance(
+                operation_node,
+                ast.Call,
+            ):
+                continue
+
+            if not isinstance(
+                operation_node.func,
+                ast.Attribute,
+            ):
+                continue
+
+            operation_name = operation_node.func.attr
+
+            model_name = None
+            field_name = None
+
+            for keyword in operation_node.keywords:
+
+                if keyword.arg == "model_name":
+
+                    model_name = ast.literal_eval(
+                        keyword.value
+                    )
+
+                elif keyword.arg == "name":
+
+                    field_name = ast.literal_eval(
+                        keyword.value
+                    )
+
+            operations.append(
+                MigrationOperation(
+                    operation=operation_name,
+                    details={
+                        "model": model_name,
+                        "field": field_name,
+                    },
+                ).to_dict()
+            )
+
+        return operations
+    
+    def _migration_num(self,migration_name:str) -> int:
+        return int(migration_name.split("_",1)[0])

chronicle/scanning/scanners/django_model.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+@dataclass
+class MigrationOperation:
+    operation:str
+    details:dict
+    
+    def to_dict(self) -> dict:
+        return {
+            "operation":self.operation,
+            "details":self.details
+        }

chronicle/scanning/scanners/git.py

@@ -0,0 +1,94 @@
+import subprocess
+from pathlib import Path
+from datetime import datetime
+from chronicle.integrations.git import GitIntegration
+from ...storage.models import Observation
+from .git_models import FileChange
+from .base import Scanner
+from chronicle.scanning.context import ScanContext
+
+class GitScanner(Scanner):
+    """Scanner for Git repo history."""
+    
+    def __init__(self, project_root: Path) -> None:
+        super().__init__(project_root)
+        self.git = GitIntegration(project_root)
+    
+    def scan(self,contexts:list[ScanContext]) -> list[Observation]:
+        for context in contexts:
+            last_commit = context.get_state("git","last_commit")
+            if last_commit:
+                break
+        try:
+            if last_commit:
+                output = self.git.run(
+                    "log",
+                    f"{last_commit}..HEAD",
+                    "--format=%H|%P|%aI|%an|%s",
+                )
+            else:
+                output = self.git.run(
+                    "log",
+                    "--format=%H|%P|%aI|%an|%s",
+                )
+        except subprocess.CalledProcessError:
+            return []
+        observation:list[Observation] = []
+        for line in output.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            
+            commit_hash, parents, timestamp, author, msg = line.split("|",4)
+            
+            parents_hash = parents.split() if parents else []
+            
+            changes = self.get_changed_file(commit_hash=commit_hash)
+            
+            observation.append(Observation(
+                source="git",
+                type="commit",
+                external_id=commit_hash,
+                timestamp=datetime.fromisoformat(timestamp),
+                data={
+                    "hash":commit_hash,
+                    "message":msg,
+                    "author":author,
+                    "parents":parents_hash,
+                    "changes": [
+                        {
+                            "path": change.path,
+                            "status": change.status,
+                        }
+                        for change in changes
+                    ]
+                }
+            ))
+            
+        return observation
+    
+    def get_changed_file(self,commit_hash:str) -> list[FileChange]:
+        changes_observed = self.git.run(
+            "diff-tree",
+            "--no-commit-id",
+            "--name-status",
+            "-r",
+            commit_hash,
+        )
+        
+        changes = []
+        for line in changes_observed.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            
+            status, path = line.split("\t", 1)
+
+            changes.append(
+                FileChange(
+                    path=path,
+                    status=status,
+                )
+            )
+            
+        return changes

chronicle/scanning/scanners/git_models.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+@dataclass
+class FileChange:
+    path:str
+    status:str

chronicle/storage/analysis_state.py

@@ -0,0 +1,44 @@
+import sqlite3
+from datetime import datetime, timezone
+
+class AnalysisStateRepo:
+    def __init__(self,conn:sqlite3.Connection) -> None:
+        self.conn = conn
+        
+    def get(self,key:str,analyzer:str) -> str | None:
+        row = self.conn.execute(
+            """
+            SELECT value FROM analysis_state WHERE key=? AND analyzer=?;
+            """,
+            (key,analyzer),
+        ).fetchone()
+        
+        if row is None:
+            return None
+        
+        return row["value"] 
+    
+    def set(self,key:str,analyzer:str,value:str) -> None:
+        self.conn.execute(
+            """
+            INSERT INTO analysis_state(
+                analyzer,
+                key,
+                value,
+                updated_at
+            )
+            VALUES (?,?,?,?)
+            
+            ON CONFLICT(analyzer,key)
+            DO UPDATE SET 
+            value = excluded.value,
+            updated_at = excluded.updated_at
+            """,
+            (
+                analyzer,
+                key,
+                value,
+                datetime.now(timezone.utc).isoformat(),
+            ),
+        )
+        

chronicle/storage/database.py

@@ -0,0 +1,9 @@
+import sqlite3
+from pathlib import Path
+
+def connect(database_path:Path) -> sqlite3.Connection:
+    connection = sqlite3.connect(database_path)
+    connection.row_factory = sqlite3.Row
+    cursor = connection.cursor()
+    cursor.execute("PRAGMA foreign_keys = ON;")
+    return connection

chronicle/storage/findings.py

@@ -0,0 +1,53 @@
+import json 
+import sqlite3
+from chronicle.storage.models import Findings
+
+class FindingsRepo:
+    def __init__(self,conn:sqlite3.Connection) -> None:
+        self.conn = conn
+        
+    def save(self, finding:Findings) -> bool:
+        cursor = self.conn.execute(
+            """
+            INSERT INTO 
+            findings(
+                analyzer,
+                severity,
+                title,
+                message,
+                observation_id,
+                data
+            ) VALUES (?,?,?,?,?,?)
+            """,
+            (
+                finding.analyzer,
+                finding.severity,  
+                finding.title,
+                finding.message,
+                finding.observation_id,
+                json.dumps(finding.data), 
+            ),
+        )
+        
+        return cursor.rowcount == 1
+    
+    def list_all(self) -> list[Findings]:
+        findings = []
+        
+        rows = self.conn.execute(
+        """
+        SELECT * FROM findings ORDER BY observation_id;
+        """
+        ).fetchall()
+        
+        for row in rows:
+            findings.append(Findings(
+                analyzer=row["analyzer"],
+                severity=row["severity"],
+                title=row["title"],
+                message=row["message"],
+                observation_id=row["observation_id"],
+                data=json.loads(row["data"]),
+            ))
+            
+        return findings

chronicle/storage/models.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass,field
+from datetime import datetime
+
+@dataclass
+class Observation:
+    source:str
+    type:str
+    external_id:str 
+    timestamp:datetime
+    data:dict
+    id:int | None = None
+    
+@dataclass
+class Findings:
+    analyzer: str
+    severity: str
+    title: str
+    message: str
+    observation_id: int
+    data: dict = field(default_factory=dict)

chronicle/storage/observations.py

@@ -0,0 +1,93 @@
+import json
+import sqlite3
+from chronicle.storage.models import Observation
+from datetime import datetime
+
+class ObservationRepo:
+    def __init__(self, connection:sqlite3.Connection) -> None:
+        self.conn = connection
+        
+    def save(self,observation:Observation)->bool:
+        cursor = self.conn.execute(
+            """
+            INSERT OR IGNORE INTO observations(
+                source,
+                type,
+                external_id,
+                timestamp,
+                data
+            ) VALUES (?,?,?,?,?)
+            """,
+            (
+                observation.source,
+                observation.type,
+                observation.external_id,
+                observation.timestamp.isoformat(),
+                json.dumps(observation.data),
+            ),
+        )
+        
+        return cursor.rowcount == 1
+    
+    def list_all(self,id:int | None = None) -> list[Observation]:
+        observations = []
+        if id:
+            row = self.conn.execute(
+                """
+                SELECT
+                    id,
+                    source,
+                    type,
+                    external_id,
+                    timestamp,
+                    data
+                FROM observations 
+                WHERE id = ?;
+                """,
+                (id,),
+            ).fetchone()
+            
+            if row is None:
+                return []
+            
+            observations.append(
+                Observation(
+                    source=row["source"],
+                    type=row["type"],
+                    external_id=row["external_id"],
+                    timestamp=datetime.fromisoformat(
+                        row["timestamp"]
+                    ),
+                    data=json.loads(row["data"]),
+                )
+            )
+        else:
+            rows = self.conn.execute(
+                """
+                SELECT
+                    id,
+                    source,
+                    type,
+                    external_id,
+                    timestamp,
+                    data
+                FROM observations
+                ORDER BY id
+                """
+            ).fetchall()
+            
+            for row in rows:
+                observations.append(
+                    Observation(
+                        id=row["id"],
+                        source=row["source"],
+                        type=row["type"],
+                        external_id=row["external_id"],
+                        timestamp=datetime.fromisoformat(
+                            row["timestamp"]
+                        ),
+                        data=json.loads(row["data"]),
+                    )
+                )
+                
+        return observations

chronicle/storage/scan_state.py

@@ -0,0 +1,44 @@
+import sqlite3
+from datetime import datetime, timezone
+
+class ScanStateRepo:
+    def __init__(self,conn:sqlite3.Connection) -> None:
+        self.conn = conn
+        
+    def get(self,key:str,scanner:str) -> str | None:
+        row = self.conn.execute(
+            """
+            SELECT VALUE FROM scan_state WHERE key=? AND scanner=?;
+            """,
+            (key,scanner),
+        ).fetchone()
+        
+        if row is None:
+            return None
+        
+        return row["value"] 
+    
+    def set(self,key:str,scanner:str,value:str) -> None:
+        self.conn.execute(
+            """
+            INSERT INTO scan_state(
+                scanner,
+                key,
+                value,
+                updated_at
+            )
+            VALUES (?,?,?,?)
+            
+            ON CONFLICT(scanner,key)
+            DO UPDATE SET 
+            value = excluded.value,
+            updated_at = excluded.updated_at
+            """,
+            (
+                scanner,
+                key,
+                value,
+                datetime.now(timezone.utc).isoformat(),
+            )
+        )
+        

chronicle/storage/schema.py

@@ -0,0 +1,47 @@
+import sqlite3
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS observations(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    type TEXT NOT NULL,
+    external_id TEXT NOT NULL,
+    timestamp TEXT NOT NULL,
+    data TEXT NOT NULL,
+    UNIQUE(source,type,external_id)
+);
+
+CREATE TABLE IF NOT EXISTS scan_state(
+    scanner TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY(scanner , key)
+);
+
+CREATE TABLE IF NOT EXISTS analysis_state(
+    analyzer TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY(analyzer , key)
+);
+
+CREATE TABLE IF NOT EXISTS findings(
+    analyzer TEXT NOT NULL,
+    severity TEXT NOT NULL,
+    title TEXT NOT NULL,
+    message TEXT NOT NULL,
+    observation_id INTEGER,
+    data TEXT NOT NULL,
+    UNIQUE(analyzer,observation_id,data),
+    FOREIGN KEY (observation_id)
+    REFERENCES observations (id)
+        ON DELETE CASCADE
+        ON UPDATE CASCADE
+);
+"""
+
+def init_schema(connection:sqlite3.Connection) -> None:
+    connection.executescript(SCHEMA)
+    connection.commit()