bioguider 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

bioguider/database/code_structure_db.py

@@ -0,0 +1,489 @@
+import sqlite3
+from sqlite3 import Connection
+import os
+from time import strftime
+from typing import Optional, List, Dict, Any
+import logging
+import json
+
+logging = logging.getLogger(__name__)
+
+CODE_STRUCTURE_TABLE_NAME = "SourceCodeStructure"
+
+code_structure_create_table_query = f"""
+CREATE TABLE IF NOT EXISTS {CODE_STRUCTURE_TABLE_NAME} (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name VARCHAR(256) NOT NULL,
+    path VARCHAR(512) NOT NULL,
+    start_lineno INTEGER NOT NULL,
+    end_lineno INTEGER NOT NULL,
+    parent VARCHAR(256),
+    doc_string TEXT,
+    params TEXT,
+    reference_to TEXT,
+    reference_by TEXT,
+    datetime TEXT NOT NULL DEFAULT (strftime('%Y-%m-%d %H:%M:%f', 'now')),
+    UNIQUE (name, path, start_lineno, end_lineno, parent)
+);
+"""
+
+code_structure_insert_query = f"""
+INSERT INTO {CODE_STRUCTURE_TABLE_NAME}(name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', 'now'))
+ON CONFLICT(name, path, start_lineno, end_lineno, parent) DO UPDATE SET doc_string=excluded.doc_string, params=excluded.params, 
+reference_to=excluded.reference_to, reference_by=excluded.reference_by, datetime=strftime('%Y-%m-%d %H:%M:%f', 'now');
+"""
+
+code_structure_select_by_path_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE path = ?;
+"""
+
+code_structure_select_by_name_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE name = ?;
+"""
+
+code_structure_select_by_name_and_path_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE name = ? AND path = ?;
+"""
+
+code_structure_select_by_id_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE id = ?;
+"""
+
+code_structure_select_by_parent_and_parentpath_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE parent = ? AND path = ?;
+"""
+
+code_structure_select_by_parent_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE parent = ?;
+"""
+
+code_structure_update_query = f"""
+UPDATE {CODE_STRUCTURE_TABLE_NAME} 
+SET name = ?, path = ?, start_lineno = ?, end_lineno = ?, parent = ?, doc_string = ?, params = ?, reference_to = ?, reference_by = ?, datetime = strftime('%Y-%m-%d %H:%M:%f', 'now')
+WHERE id = ?;
+"""
+
+code_structure_delete_query = f"""
+DELETE FROM {CODE_STRUCTURE_TABLE_NAME} WHERE id = ?;
+"""
+
+code_structure_select_by_name_and_parent_and_path_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE name = ? AND parent = ? AND path = ?;
+"""
+
+code_structure_select_by_name_and_parent_query = f"""
+SELECT id, name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, datetime 
+FROM {CODE_STRUCTURE_TABLE_NAME} 
+WHERE name = ? AND parent = ?;
+"""
+
+class CodeStructureDb:
+    def __init__(self, author: str, repo_name: str, data_folder: str = None):
+        self.author = author
+        self.repo_name = repo_name
+        self.data_folder = data_folder
+        self.connection: Connection | None = None
+
+    def _ensure_tables(self) -> bool:
+        if self.connection is None:
+            return False
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_create_table_query)
+            self.connection.commit()
+            return True
+        except Exception as e:
+            logging.error(e)
+            return False
+        
+    def _connect_to_db(self) -> bool:
+        if self.connection is not None:
+            return True
+        db_path = self.data_folder
+        if db_path is None:
+            db_path = os.environ.get("DATA_FOLDER", "./data")
+        db_path = os.path.join(db_path, "databases")
+        if not os.path.exists(db_path):
+            try:
+                os.makedirs(db_path, exist_ok=True)
+            except Exception as e:
+                logging.error(e)
+                return False
+        db_path = os.path.join(db_path, "databases")
+        # Ensure the local path exists
+        try:
+            os.makedirs(db_path, exist_ok=True)
+        except Exception as e:
+            logging.error(e)
+            return False
+        db_path = os.path.join(db_path, f"{self.author}_{self.repo_name}.db")
+        if not os.path.exists(db_path):
+            try:
+                with open(db_path, "w"):
+                    pass
+            except Exception as e:
+                logging.error(e)
+                return False
+        self.connection = sqlite3.connect(db_path)
+        return True
+    
+    def insert_code_structure(
+        self,
+        name: str,
+        path: str,
+        start_lineno: int,
+        end_lineno: int,
+        parent: str = None,
+        doc_string: str = None,
+        params: str = None,
+        reference_to: str = None,
+        reference_by: str = None
+    ) -> bool:
+        """Insert a new code structure entry into the database."""
+        if parent is None:
+            parent = ""
+        if path is None:
+            path = ""
+        res = self._connect_to_db()
+        if not res:
+            return False
+        res = self._ensure_tables()
+        if not res:
+            return False
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(
+                code_structure_insert_query, 
+                (name, path, start_lineno, end_lineno, parent, doc_string, json.dumps(params) if params is not None else None, reference_to, reference_by)
+            )
+            self.connection.commit()
+            return True
+        except Exception as e:
+            logging.error(e)
+            return False
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_path(self, path: str) -> List[Dict[str, Any]]:
+        """Select all code structures by file path."""
+        res = self._connect_to_db()
+        if not res:
+            return []
+        res = self._ensure_tables()
+        if not res:
+            return []
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_path_query, (path,))
+            rows = cursor.fetchall()
+            return [
+                {
+                    "id": row[0],
+                    "name": row[1],
+                    "path": row[2],
+                    "start_lineno": row[3],
+                    "end_lineno": row[4],
+                    "parent": row[5],
+                    "doc_string": row[6],
+                    "params": row[7],
+                    "reference_to": row[8],
+                    "reference_by": row[9],
+                    "datetime": row[10]
+                }
+                for row in rows
+            ]
+        except Exception as e:
+            logging.error(e)
+            return []
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_name(self, name: str) -> List[Dict[str, Any]]:
+        """Select all code structures by name."""
+        res = self._connect_to_db()
+        if not res:
+            return []
+        res = self._ensure_tables()
+        if not res:
+            return []
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_name_query, (name,))
+            rows = cursor.fetchall()
+            return [
+                {
+                    "id": row[0],
+                    "name": row[1],
+                    "path": row[2],
+                    "start_lineno": row[3],
+                    "end_lineno": row[4],
+                    "parent": row[5],
+                    "doc_string": row[6],
+                    "params": row[7],
+                    "reference_to": row[8],
+                    "reference_by": row[9],
+                    "datetime": row[10]
+                }
+                for row in rows
+            ]
+        except Exception as e:
+            logging.error(e)
+            return []
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_name_and_path(self, name: str, path: str) -> Optional[Dict[str, Any]]:
+        """Select a code structure by name and path."""
+        res = self._connect_to_db()
+        if not res:
+            return None
+        res = self._ensure_tables()
+        if not res:
+            return None
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_name_and_path_query, (name, path))
+            row = cursor.fetchone()
+            if row is None:
+                return None
+            return {
+                "id": row[0],
+                "name": row[1],
+                "path": row[2],
+                "start_lineno": row[3],
+                "end_lineno": row[4],
+                "parent": row[5],
+                "doc_string": row[6],
+                "params": row[7],
+                "reference_to": row[8],
+                "reference_by": row[9],
+                "datetime": row[10]
+            }
+        except Exception as e:
+            logging.error(e)
+            return None
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_name_and_parent(self, name: str, parent: str) -> List[Dict[str, Any]]:
+        """Select all code structures by name and parent."""
+        res = self._connect_to_db()
+        if not res:
+            return []
+        res = self._ensure_tables()
+        if not res:
+            return []
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_name_and_parent_query, (name, parent))
+            rows = cursor.fetchall()
+            return [
+                {
+                    "id": row[0],
+                    "name": row[1],
+                    "path": row[2],
+                    "start_lineno": row[3],
+                    "end_lineno": row[4],
+                    "parent": row[5],
+                    "doc_string": row[6],
+                    "params": row[7],
+                    "reference_to": row[8],
+                    "reference_by": row[9],
+                    "datetime": row[10]
+                }
+                for row in rows
+            ]
+        except Exception as e:
+            logging.error(e)
+            return []
+        finally:
+            self.connection.close()
+            self.connection = None
+
+
+    def select_by_name_and_parent_and_path(self, name: str, parent: str, path: str) -> Optional[Dict[str, Any]]:
+        """Select a code structure by name and parent."""
+        res = self._connect_to_db()
+        if not res:
+            return None
+        res = self._ensure_tables()
+        if not res:
+            return None
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_name_and_parent_and_path_query, (name, parent, path))
+            row = cursor.fetchone()
+            if row is None:
+                return None
+            return {
+                "id": row[0],
+                "name": row[1],
+                "path": row[2],
+                "start_lineno": row[3],
+                "end_lineno": row[4],
+                "parent": row[5],
+                "doc_string": row[6],
+                "params": row[7],
+                "reference_to": row[8],
+                "reference_by": row[9],
+                "datetime": row[10]
+            }
+        except Exception as e:
+            logging.error(e)
+            return None
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_id(self, id: int) -> Optional[Dict[str, Any]]:
+        """Select a code structure by ID."""
+        res = self._connect_to_db()
+        if not res:
+            return None
+        res = self._ensure_tables()
+        if not res:
+            return None
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_select_by_id_query, (id,))
+            row = cursor.fetchone()
+            if row is None:
+                return None
+            return {
+                "id": row[0],
+                "name": row[1],
+                "path": row[2],
+                "start_lineno": row[3],
+                "end_lineno": row[4],
+                "parent": row[5],
+                "doc_string": row[6],
+                "params": row[7],
+                "reference_to": row[8],
+                "reference_by": row[9],
+                "datetime": row[10]
+            }
+        except Exception as e:
+            logging.error(e)
+            return None
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def update_code_structure(
+        self,
+        id: int,
+        name: str,
+        path: str,
+        start_lineno: int,
+        end_lineno: int,
+        parent: str = None,
+        doc_string: str = None,
+        params: str = None,
+        reference_to: str = None,
+        reference_by: str = None
+    ) -> bool:
+        """Update an existing code structure entry."""
+        res = self._connect_to_db()
+        if not res:
+            return False
+        res = self._ensure_tables()
+        if not res:
+            return False
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(
+                code_structure_update_query, 
+                (name, path, start_lineno, end_lineno, parent, doc_string, params, reference_to, reference_by, id)
+            )
+            self.connection.commit()
+            return cursor.rowcount > 0
+        except Exception as e:
+            logging.error(e)
+            return False
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def select_by_parent(self, parent: str, path: str | None = None) -> List[Dict[str, Any]]:
+        """Select all code structures by parent."""
+        res = self._connect_to_db()
+        if not res:
+            return []
+        res = self._ensure_tables()
+        if not res:
+            return []
+        try:
+            cursor = self.connection.cursor()
+            if path is not None:
+                cursor.execute(code_structure_select_by_parent_and_parentpath_query, (parent, path))
+            else:
+                cursor.execute(code_structure_select_by_parent_query, (parent,))
+            rows = cursor.fetchall()
+            return [
+                {
+                    "id": row[0],
+                    "name": row[1],
+                    "path": row[2],
+                    "start_lineno": row[3],
+                    "end_lineno": row[4],
+                    "parent": row[5],
+                    "doc_string": row[6],
+                    "params": row[7],
+                    "reference_to": row[8],
+                    "reference_by": row[9],
+                    "datetime": row[10]
+                }
+                for row in rows
+            ]
+        except Exception as e:
+            logging.error(e)
+            return []
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def delete_code_structure(self, id: int) -> bool:
+        """Delete a code structure entry by ID."""
+        res = self._connect_to_db()
+        if not res:
+            return False
+        res = self._ensure_tables()
+        if not res:
+            return False
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(code_structure_delete_query, (id,))
+            self.connection.commit()
+            return cursor.rowcount > 0
+        except Exception as e:
+            logging.error(e)
+            return False
+        finally:
+            self.connection.close()
+            self.connection = None
+
+    def get_db_file(self) -> str:
+        """Get the database file path."""
+        db_path = os.environ.get("DATA_FOLDER", "./data")
+        db_path = os.path.join(db_path, "databases")
+        db_path = os.path.join(db_path, f"{self.author}_{self.repo_name}.db")
+        return db_path

bioguider/generation/__init__.py

@@ -0,0 +1,39 @@
+from .models import (
+    EvaluationReport,
+    SuggestionItem,
+    StyleProfile,
+    PlannedEdit,
+    DocumentPlan,
+    OutputArtifact,
+    GenerationManifest,
+)
+from .report_loader import EvaluationReportLoader
+from .suggestion_extractor import SuggestionExtractor
+from .repo_reader import RepoReader
+from .style_analyzer import StyleAnalyzer
+from .change_planner import ChangePlanner
+from .document_renderer import DocumentRenderer
+from .output_manager import OutputManager
+from .llm_content_generator import LLMContentGenerator
+from .llm_cleaner import LLMCleaner
+
+__all__ = [
+    "EvaluationReport",
+    "SuggestionItem",
+    "StyleProfile",
+    "PlannedEdit",
+    "DocumentPlan",
+    "OutputArtifact",
+    "GenerationManifest",
+    "EvaluationReportLoader",
+    "SuggestionExtractor",
+    "RepoReader",
+    "StyleAnalyzer",
+    "ChangePlanner",
+    "DocumentRenderer",
+    "OutputManager",
+    "LLMContentGenerator",
+    "LLMCleaner",
+]
+
+

bioguider/generation/change_planner.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from typing import List, Dict
+
+from .models import SuggestionItem, StyleProfile, DocumentPlan, PlannedEdit
+
+
+class ChangePlanner:
+    def build_plan(
+        self,
+        repo_path: str,
+        style: StyleProfile,
+        suggestions: List[SuggestionItem],
+        available_files: Dict[str, str],
+    ) -> DocumentPlan:
+        planned: List[PlannedEdit] = []
+        seen_headers: set[tuple[str, str]] = set()
+
+        def section_header(title: str) -> str:
+            # use heading level 2 for inserts to be safe
+            h = style.heading_style or "#"
+            return f"{h*2} {title}\n\n"
+
+        for s in suggestions:
+            for target in s.target_files:
+                if target not in available_files:
+                    # allow planning; renderer will skip if missing
+                    pass
+
+                if s.action == "add_dependencies_section":
+                    content = section_header("Dependencies") + "- List required packages and versions.\n"
+                    header_key = (target, (s.anchor_hint or "Dependencies").strip().lower())
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": s.anchor_hint or "Dependencies"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "add_system_requirements_section":
+                    content = section_header("System Requirements") + "- OS and R version requirements.\n"
+                    header_key = (target, (s.anchor_hint or "System Requirements").strip().lower())
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": s.anchor_hint or "System Requirements"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "mention_license_section":
+                    content = section_header("License") + "This project is released under the MIT License. See LICENSE for details.\n"
+                    header_key = (target, (s.anchor_hint or "License").strip().lower())
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": s.anchor_hint or "License"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "normalize_headings_structure":
+                    # Minimal placeholder: avoid heavy rewrites
+                    # Plan a no-op or a small note; actual normalization could be added later
+                    continue
+                elif s.action == "add_usage_section":
+                    content = section_header("Usage") + "- Brief example of typical workflow.\n"
+                    header_key = (target, "usage")
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": "Usage"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "replace_intro":
+                    # Replace intro block (between H1 and first H2) with a clean Overview section
+                    content = section_header("Overview") + "- Clear 2–3 sentence summary of purpose and audience.\n"
+                    header_key = (target, "overview")
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="replace_intro_block",
+                        anchor={"type": "header", "value": "Overview"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "clarify_mandatory_vs_optional":
+                    content = section_header("Dependencies") + (
+                        "- Mandatory: ...\n- Optional: ...\n"
+                    )
+                    header_key = (target, "dependencies")
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": "Dependencies"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+                elif s.action == "add_hardware_requirements":
+                    content = section_header("Hardware Requirements") + (
+                        "- Recommended: >=16 GB RAM, multi-core CPU for large datasets.\n"
+                    )
+                    header_key = (target, (s.anchor_hint or "Hardware Requirements").strip().lower())
+                    if header_key in seen_headers:
+                        continue
+                    planned.append(PlannedEdit(
+                        file_path=target,
+                        edit_type="append_section",
+                        anchor={"type": "header", "value": s.anchor_hint or "Hardware Requirements"},
+                        content_template=content,
+                        rationale=s.source.get("evidence", ""),
+                        suggestion_id=s.id,
+                    ))
+                    seen_headers.add(header_key)
+
+        return DocumentPlan(repo_path=repo_path, style_profile=style, planned_edits=planned)
+
+