data-advisor 0.1.0__py3-none-any.whl

data_advisor/search.py

@@ -0,0 +1,679 @@
+"""
+search.py - Intelligent code search engine for local code banks and snippets.
+
+Allows students and developers to quickly retrieve code templates, sensor scripts,
+and machine learning pipelines during timed examinations.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+SUPPORTED_EXTENSIONS = {
+    ".py", ".ipynb", ".ino", ".cpp", ".c", ".h", ".hpp",
+    ".txt", ".md", ".sh", ".sql", ".json", ".yaml", ".yml"
+}
+
+IGNORE_DIRS = {
+    ".git", ".pytest_cache", ".venv", "venv", "env", "__pycache__",
+    "node_modules", "build", "dist", ".idea", ".vscode"
+}
+
+# Semantic query expansion dictionary for common technical terms
+QUERY_ALIASES = {
+    "rf": ["random forest"],
+    "dt": ["decision tree"],
+    "lr": ["linear regression", "logistic regression"],
+    "nb": ["naive bayes"],
+    "knn": ["k nearest neighbors", "kneighbors"],
+    "kmeans": ["k means", "k-means", "clustering"],
+    "eda": ["exploratory", "visualization", "data analysis"],
+    "iss": ["international space station", "iss-now"],
+    "usgs": ["earthquake"],
+    "coap": ["aiocoap", "coap server", "coap client"],
+    "dht": ["dht22", "dht11", "temperature", "humidity"],
+    "ldr": ["light intensity", "analogread", "photoresistor"],
+    "pir": ["motion detection", "motion sensor"],
+    "pca": ["principal component", "dimensionality reduction"],
+    "nlp": ["tfidf", "countvectorizer", "tokenizer"],
+    "apriori": ["association rules", "frequent itemsets", "groceries", "lift"],
+}
+
+
+def safe_print(text: str) -> None:
+    """Print text safely across operating systems without charmap/cp1252 encode errors."""
+    try:
+        print(text)
+    except UnicodeEncodeError:
+        enc = getattr(sys.stdout, "encoding", None) or "utf-8"
+        try:
+            print(text.encode(enc, errors="replace").decode(enc))
+        except Exception:
+            print(text.encode("ascii", errors="replace").decode("ascii"))
+
+
+def extract_code_from_file(file_path: Path) -> str:
+    """Read file content, automatically extracting code cells if it's a Jupyter Notebook."""
+    try:
+        if file_path.suffix.lower() == ".ipynb":
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                nb = json.load(f)
+            code_cells = []
+            for cell in nb.get("cells", []):
+                if cell.get("cell_type") == "code":
+                    source = cell.get("source", [])
+                    cell_text = "".join(source) if isinstance(source, list) else source
+                    if cell_text.strip():
+                        code_cells.append(cell_text)
+            return "\n\n# --- [Notebook Cell] ---\n".join(code_cells)
+        else:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                return f.read()
+    except Exception as exc:
+        return f"# [Error reading {file_path.name}: {exc}]"
+
+
+def extract_file_description(file_path: Path) -> str:
+    """Extract a concise 1-line description from file header, docstring or initial comment."""
+    content = extract_code_from_file(file_path)
+    lines = content.strip().splitlines()
+    for line in lines[:15]:
+        line_clean = line.strip().lstrip("/*#\"'").rstrip("*/\"'").strip()
+        if len(line_clean) > 8 and not line_clean.startswith("=") and not line_clean.lower().endswith(file_path.suffix.lower()):
+            return line_clean[:80]
+    return file_path.name
+
+
+class CodeSearchEngine:
+    """
+    Dynamic code search engine that scans directories of code files,
+    ranks them by relevance to ANY given keyword(s), and retrieves complete files.
+    
+    Dynamically scans the target directory on every search, automatically picking
+    up any newly pasted, modified, or removed files without requiring a restart.
+    """
+
+    def __init__(self, search_paths: Optional[Union[str, Path, List[Union[str, Path]]]] = None) -> None:
+        if search_paths is None:
+            self.search_paths = self._discover_default_paths()
+        elif isinstance(search_paths, (str, Path)):
+            self.search_paths = [Path(search_paths)]
+        else:
+            self.search_paths = [Path(p) for p in search_paths]
+
+    def _discover_default_paths(self) -> List[Path]:
+        """Auto-detect code bank directory in cwd or project root, prioritizing 'code_bank'."""
+        cwd = Path.cwd()
+        package_root = Path(__file__).resolve().parent.parent.parent
+
+        # 1. Check for 'code_bank' directory
+        for base in [cwd, package_root]:
+            cb = base / "code_bank"
+            if cb.exists() and cb.is_dir():
+                return [cb]
+
+        # 2. Check other common folder names
+        for candidate in ["codes", "snippets", "scripts", "examples"]:
+            for base in [cwd, package_root]:
+                p = base / candidate
+                if p.exists() and p.is_dir():
+                    return [p]
+
+        # 3. Default to current working directory
+        return [cwd]
+
+    def collect_code_files(self) -> List[Path]:
+        """Recursively gather all eligible code files across configured search paths."""
+        files: List[Path] = []
+        seen = set()
+        for root_path in self.search_paths:
+            if not root_path.exists():
+                continue
+            if root_path.is_file():
+                if root_path.suffix.lower() in SUPPORTED_EXTENSIONS and root_path.resolve() not in seen:
+                    files.append(root_path)
+                    seen.add(root_path.resolve())
+                continue
+
+            for dirpath, dirnames, filenames in os.walk(root_path):
+                # Prune ignored directories
+                dirnames[:] = [d for d in dirnames if d not in IGNORE_DIRS and not d.startswith(".")]
+                for fn in sorted(filenames):
+                    p = Path(dirpath) / fn
+                    if p.suffix.lower() in SUPPORTED_EXTENSIONS and p.resolve() not in seen:
+                        files.append(p)
+                        seen.add(p.resolve())
+        return files
+
+    def list_all(self, print_output: bool = True) -> str:
+        """List all indexed code snippets with a 1-line summary."""
+        all_files = self.collect_code_files()
+        divider = "=" * 80
+        paths_str = ", ".join(str(p) for p in self.search_paths)
+
+        if not all_files:
+            msg = (
+                f"\n{divider}\n"
+                f"  DATA ADVISOR CODE BANK: 0 FILES FOUND\n"
+                f"{divider}\n"
+                f"[*] Monitored Folder: {paths_str}\n"
+                f"[*] The folder is currently empty.\n"
+                f"[*] Paste your Python (.py), Jupyter Notebook (.ipynb), Arduino (.ino, .cpp),\n"
+                f"    or text files into 'code_bank/'.\n"
+                f"[*] Any keywords you search for will immediately be found from your pasted files!\n"
+                f"{divider}\n"
+            )
+            if print_output:
+                safe_print(msg)
+            return msg
+
+        lines = [
+            f"\n{divider}",
+            f"  DATA ADVISOR CODE BANK: {len(all_files)} CODE FILE(S) AVAILABLE",
+            divider,
+            f"  Folder: {paths_str}",
+            f"  {'#':<3} | {'File Name':<34} | {'Description'}",
+            f"  {'-'*3}-|-{'-'*34}-|-{'-'*38}",
+        ]
+
+        for idx, fp in enumerate(all_files, 1):
+            desc = extract_file_description(fp)
+            lines.append(f"  {idx:<3} | {fp.name:<34} | {desc}")
+
+        lines.append(divider)
+        lines.append("[*] Search any keyword : search('<keyword>') or data-advisor --search '<keyword>'")
+        lines.append("[*] Get complete file  : get_file('<q_num or filename>') or data-advisor --get '<name>'")
+        lines.append("")
+
+        out_str = "\n".join(lines)
+        if print_output:
+            safe_print(out_str)
+        return out_str
+
+    def score_file(self, file_path: Path, query: Union[str, List[str], Tuple[str, ...]]) -> Tuple[int, str]:
+        """
+        Compute a relevance score for a file against any keyword(s).
+        Returns (score, file_content).
+        """
+        content = extract_code_from_file(file_path)
+        if not content.strip():
+            return 0, ""
+
+        score = 0
+        if isinstance(query, (list, tuple)):
+            q_raw = " ".join(str(k) for k in query).strip().lower()
+        else:
+            q_raw = str(query).strip().lower()
+
+        if not q_raw:
+            return 0, content
+
+        filename_lower = file_path.name.lower()
+        filename_clean = filename_lower.replace("_", " ").replace("-", " ")
+        content_lower = content.lower()
+
+        # Handle Question numbers (e.g. 'q1', 'q 1', 'question 1', 'q01', '1', '24')
+        q_num_match = re.search(r"(?:^|\b)(?:q|question)?\s*(\d+)(?:\b|$)", q_raw)
+        if q_num_match:
+            num_str = q_num_match.group(1)
+            try:
+                num_int = int(num_str)
+                num_2d = f"{num_int:02d}"
+            except ValueError:
+                num_int = None
+                num_2d = num_str
+
+            if (
+                f"question_{num_str}." in filename_lower
+                or f"question_{num_2d}." in filename_lower
+                or f"q{num_str}." in filename_lower
+                or f"q{num_2d}." in filename_lower
+                or filename_lower.startswith(f"{num_2d}_")
+                or filename_lower.startswith(f"{num_str}_")
+                or file_path.parent.name.lower().startswith(f"{num_2d}_")
+            ):
+                score += 500
+            elif (
+                f"question {num_str}" in content_lower
+                or f"question {num_2d}" in content_lower
+                or f"question_{num_str}" in content_lower
+                or f"question_{num_2d}" in content_lower
+            ):
+                score += 200
+
+        # Exact phrase match in filename
+        if q_raw in filename_clean or q_raw in filename_lower:
+            score += 250
+
+        # Extract all search tokens
+        raw_tokens = [tok for tok in re.split(r"[\s,;\-_+&|/]+", q_raw) if len(tok) >= 1]
+        if not raw_tokens:
+            raw_tokens = [q_raw]
+
+        # Aliases expansion for common machine learning / exam abbreviations
+        expanded_tokens = list(raw_tokens)
+        for tok in raw_tokens:
+            if tok in QUERY_ALIASES:
+                expanded_tokens.extend(QUERY_ALIASES[tok])
+
+        # Exact phrase match in content
+        if q_raw in content_lower:
+            count = content_lower.count(q_raw)
+            score += min(150, 60 + (count * 20))
+
+        # Check each keyword token
+        all_tokens_present = True
+        for tok in raw_tokens:
+            found_token = False
+
+            if tok in filename_clean or tok in file_path.parent.name.lower():
+                score += 70
+                found_token = True
+                if re.search(rf"\b{re.escape(tok)}\b", filename_clean):
+                    score += 30
+
+            c = content_lower.count(tok)
+            if c > 0:
+                found_token = True
+                score += min(50, 15 + (c * 7))
+                if re.search(rf"\b{re.escape(tok)}\b", content_lower):
+                    score += 20
+
+            if not found_token:
+                all_tokens_present = False
+
+        if all_tokens_present and len(raw_tokens) > 1:
+            score += 100
+
+        if score == 0:
+            for alias in expanded_tokens:
+                if alias in filename_clean or alias in content_lower:
+                    score += 40
+
+        return score, content
+
+    def resolve_file(self, query: Union[str, int]) -> Optional[Path]:
+        """
+        Resolve a question number, filename, relative path, or search term
+        to the exact single target Path.
+        """
+        all_files = self.collect_code_files()
+        if not all_files:
+            return None
+
+        q_str = str(query).strip()
+
+        # 1. Direct path exists
+        direct_path = Path(q_str)
+        if direct_path.exists() and direct_path.is_file():
+            return direct_path
+
+        # 2. Check within configured search paths
+        for base in self.search_paths:
+            candidate = base / q_str
+            if candidate.exists() and candidate.is_file():
+                return candidate
+
+        # 3. Match question number e.g. 1, '1', '01', 'q1', 'q01', 'question 1', 'question_01'
+        m = re.search(r"(?:^|\b)(?:q|question)?\s*(\d+)(?:\b|$)", q_str.lower())
+        if m:
+            num = int(m.group(1))
+            num_2d = f"{num:02d}"
+
+            # Priority A: question_XX.py
+            for fp in all_files:
+                if fp.name.lower() in (f"question_{num_2d}.py", f"question_{num}.py"):
+                    return fp
+
+            # Priority B: Folder matching number e.g. 01_dht22_serial
+            candidates = []
+            for fp in all_files:
+                parent_lower = fp.parent.name.lower()
+                if parent_lower.startswith(f"{num_2d}_") or parent_lower.startswith(f"{num}_"):
+                    candidates.append(fp)
+
+            if candidates:
+                # Prefer sketch.ino or primary python script or app
+                for preferred in ["sketch.ino", "app.py", f"question_{num_2d}.py"]:
+                    for c in candidates:
+                        if c.name.lower() == preferred:
+                            return c
+                for c in candidates:
+                    if c.suffix.lower() in (".py", ".ino") and not c.name.startswith("__"):
+                        return c
+                return candidates[0]
+
+        # 4. Exact filename match (case-insensitive)
+        for fp in all_files:
+            if fp.name.lower() == q_str.lower():
+                return fp
+
+        # 5. Filename without extension match
+        for fp in all_files:
+            if fp.stem.lower() == q_str.lower():
+                return fp
+
+        # 6. Fallback: highest scoring match from search engine
+        scored = []
+        for fp in all_files:
+            score, _ = self.score_file(fp, q_str)
+            if score > 0:
+                scored.append((score, fp))
+        if scored:
+            scored.sort(key=lambda x: x[0], reverse=True)
+            return scored[0][1]
+
+        return None
+
+    def get_file(
+        self,
+        query: Union[str, int],
+        print_output: bool = True,
+        line_numbers: bool = False,
+    ) -> str:
+        """
+        Retrieve and optionally print the COMPLETE, raw content of a file
+        by question number, filename, or keyword.
+        """
+        target = self.resolve_file(query)
+        if target is None:
+            msg = f"[!] Could not find any code file matching: '{query}'"
+            if print_output:
+                safe_print(msg)
+            return msg
+
+        raw_content = extract_code_from_file(target)
+
+        divider = "=" * 80
+        lines = raw_content.splitlines()
+        line_count = len(lines)
+        byte_size = target.stat().st_size if target.exists() else len(raw_content)
+
+        header = (
+            f"\n{divider}\n"
+            f"  COMPLETE FILE: {target.name}  ({line_count} lines, {byte_size:,} bytes)\n"
+            f"  FILE PATH    : {target.resolve()}\n"
+            f"{divider}\n"
+        )
+
+        if line_numbers:
+            numbered = [f"{i:4d} | {l}" for i, l in enumerate(lines, start=1)]
+            body = "\n".join(numbered)
+        else:
+            body = raw_content
+
+        full_display = f"{header}{body}\n{divider}\n"
+        if print_output:
+            safe_print(full_display)
+
+        return raw_content
+
+    def run_file(
+        self,
+        query: Union[str, int],
+        args: Optional[List[str]] = None,
+        timeout: Optional[int] = None,
+    ) -> int:
+        """Execute a Python file directly by question number, filename, or query."""
+        import subprocess
+
+        target = self.resolve_file(query)
+        if target is None:
+            safe_print(f"[!] Could not find any code file matching: '{query}'")
+            return 1
+
+        if target.suffix.lower() != ".py":
+            safe_print(f"[!] Cannot directly execute non-Python file: {target.name} (use get_file() to view code)")
+            return 1
+
+        cmd = [sys.executable, str(target.resolve())] + (args or [])
+        safe_print(f"[*] Executing {target.name} (Path: {target.resolve()})...\n")
+        try:
+            proc = subprocess.run(cmd, cwd=str(target.parent), timeout=timeout)
+            return proc.returncode
+        except Exception as err:
+            safe_print(f"[!] Execution failed: {err}")
+            return 1
+
+    def search(
+        self,
+        key: Optional[Union[str, List[str], Tuple[str, ...]]] = None,
+        max_results: int = 3,
+        print_output: bool = True,
+    ) -> str:
+        """
+        Search for matching files and return formatted code response.
+        If key is None, empty, 'list', or 'all', displays directory listing.
+        """
+        if key is None:
+            return self.list_all(print_output=print_output)
+
+        if isinstance(key, (list, tuple)):
+            if not key:
+                return self.list_all(print_output=print_output)
+            key_str = " ".join(str(k) for k in key)
+        else:
+            key_str = str(key).strip()
+            if not key_str or key_str.lower() in ("list", "all", "dir", "help", "--list", "-l"):
+                return self.list_all(print_output=print_output)
+
+        all_files = self.collect_code_files()
+        if not all_files:
+            divider = "=" * 80
+            paths_str = ", ".join(str(p) for p in self.search_paths)
+            msg = (
+                f"\n{divider}\n"
+                f"  DATA ADVISOR CODE BANK: 0 FILES FOUND\n"
+                f"{divider}\n"
+                f"[*] Monitored Folder: {paths_str}\n"
+                f"[*] The folder is currently empty.\n"
+                f"[*] Paste your Python (.py), Jupyter Notebook (.ipynb), Arduino (.ino, .cpp),\n"
+                f"    or text files into 'code_bank/'.\n"
+                f"{divider}\n"
+            )
+            if print_output:
+                safe_print(msg)
+            return msg
+
+        results: List[Tuple[int, Path, str]] = []
+        for fp in all_files:
+            score, content = self.score_file(fp, key_str)
+            if score > 0:
+                results.append((score, fp, content))
+
+        results.sort(key=lambda x: x[0], reverse=True)
+
+        if not results:
+            available = [fp.name for fp in all_files[:10]]
+            msg = (
+                f"\n[!] No matches found for keyword: '{key_str}'\n"
+                f"[*] Available code files in code bank ({len(all_files)} total):\n"
+                f"    - " + "\n    - ".join(available)
+            )
+            if len(all_files) > 10:
+                msg += f"\n    ... and {len(all_files) - 10} more. Run search('list') to see all."
+            if print_output:
+                safe_print(msg)
+            return msg
+
+        output_blocks = []
+        divider = "=" * 80
+        thin_div = "-" * 80
+
+        top_results = results[:max_results]
+        header = (
+            f"\n{divider}\n"
+            f"  DATA ADVISOR CODE SEARCH RESULTS FOR: '{key_str}' ({len(top_results)} match(es))\n"
+            f"{divider}\n"
+        )
+        output_blocks.append(header)
+
+        for rank, (score, file_path, content) in enumerate(top_results, 1):
+            block = [
+                f"\n[MATCH {rank}/{len(top_results)}] File: {file_path.name}  |  Relevance Score: {score}",
+                f"Path: {file_path.resolve()}",
+                thin_div,
+                content.strip(),
+                thin_div,
+            ]
+            output_blocks.append("\n".join(block))
+
+        full_output = "\n".join(output_blocks)
+        if print_output:
+            safe_print(full_output)
+
+        return full_output
+
+
+# =============================================================================
+# TOP-LEVEL CONVENIENCE FUNCTIONS
+# =============================================================================
+
+def search(
+    key: Optional[Union[str, List[str], Tuple[str, ...]]] = None,
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+    max_results: int = 3,
+    print_output: bool = True,
+) -> str:
+    """
+    Search local code files and return matching code.
+
+    Parameters
+    ----------
+    key : Optional[Union[str, List[str], Tuple[str, ...]]], default=None
+        Search query, e.g. 'wokwi', 'linear regression', 'q1', 'esp32', ['kmeans', 'elbow'].
+        If None or 'list', lists all available code files in the code bank.
+    folder : Optional[Union[str, Path, List[Union[str, Path]]]], default=None
+        Directory or list of directories containing code files.
+        If None, automatically scans './code_bank'.
+    max_results : int, default=3
+        Maximum number of matching files to return.
+    print_output : bool, default=True
+        Whether to print the formatted response directly to the console.
+
+    Returns
+    -------
+    str
+        The formatted code response text.
+    """
+    engine = CodeSearchEngine(search_paths=folder)
+    return engine.search(key=key, max_results=max_results, print_output=print_output)
+
+
+def get_file(
+    query: Union[str, int],
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+    print_output: bool = True,
+    line_numbers: bool = False,
+) -> str:
+    """
+    Retrieve and display the COMPLETE raw file content by question number, filename, or keyword.
+
+    Parameters
+    ----------
+    query : Union[str, int]
+        Question number (e.g. 1, 'q1', 'question_01', 24),
+        filename (e.g. 'question_01.py', 'sketch.ino', 'app.py'),
+        or keyword / path (e.g. '04_mqtt_esp32_python/subscriber.py', 'iss').
+    folder : Optional[Union[str, Path, List[Union[str, Path]]]], default=None
+        Directory containing code files (defaults to code_bank).
+    print_output : bool, default=True
+        If True, prints the complete file content to the console.
+    line_numbers : bool, default=False
+        If True, prints line numbers alongside the code.
+
+    Returns
+    -------
+    str
+        The exact, complete raw content of the matching file.
+
+    Examples
+    --------
+    >>> from data_advisor import get_file
+    >>> get_file("q1")             # Displays complete code for Question 1
+    >>> get_file(24)               # Displays complete code for Question 24
+    >>> get_file("sketch.ino")     # Displays ESP32 Arduino sketch
+    >>> code = get_file("q05", print_output=False)  # Returns code as string
+    """
+    engine = CodeSearchEngine(search_paths=folder)
+    return engine.get_file(query=query, print_output=print_output, line_numbers=line_numbers)
+
+
+def read_file(
+    query: Union[str, int],
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+) -> str:
+    """
+    Return the COMPLETE raw file content as a string without printing to console.
+
+    Parameters
+    ----------
+    query : Union[str, int]
+        Question number or filename.
+    folder : Optional[Union[str, Path, List[Union[str, Path]]]], default=None
+        Folder to search within.
+
+    Returns
+    -------
+    str
+        Raw file content string.
+    """
+    return get_file(query=query, folder=folder, print_output=False)
+
+
+def view_file(
+    query: Union[str, int],
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+    line_numbers: bool = False,
+) -> str:
+    """
+    Convenience alias for get_file(query, print_output=True).
+    Displays the entire file completely in the console with header metadata.
+    """
+    return get_file(query=query, folder=folder, print_output=True, line_numbers=line_numbers)
+
+
+def run_file(
+    query: Union[str, int],
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+    args: Optional[List[str]] = None,
+    timeout: Optional[int] = None,
+) -> int:
+    """
+    Execute a Python file directly by question number, filename, or query.
+
+    Parameters
+    ----------
+    query : Union[str, int]
+        Question number (e.g. 1, 'q1', 'question_24') or filename.
+    folder : Optional[Union[str, Path, List[Union[str, Path]]]], default=None
+        Folder containing the code file.
+    args : Optional[List[str]], default=None
+        Command-line arguments to pass to the script.
+    timeout : Optional[int], default=None
+        Execution timeout in seconds.
+
+    Returns
+    -------
+    int
+        The process return code (0 = success).
+    """
+    engine = CodeSearchEngine(search_paths=folder)
+    return engine.run_file(query=query, args=args, timeout=timeout)
+
+
+def list_files(
+    folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
+    print_output: bool = True,
+) -> List[Path]:
+    """
+    List all indexed code snippets with a 1-line summary and return the list of Paths.
+    """
+    engine = CodeSearchEngine(search_paths=folder)
+    engine.list_all(print_output=print_output)
+    return engine.collect_code_files()