querymind-cli 0.1.0__py3-none-any.whl

app/data/type_caster.py

@@ -0,0 +1,128 @@
+"""
+Shared column type-casting logic used by both CSVConnector and ExcelConnector.
+
+smart_cast_df(df) processes every column:
+  - Already datetime  → leave alone
+  - Already numeric   → packed-date check, then whole-number downcast
+  - Object/string     → try numeric → packed-date check → whole-number downcast
+"""
+
+import pandas as pd
+
+
+def _try_packed_date(int_series: pd.Series, original_series: pd.Series):
+    """
+    Try to parse an integer series as a packed date
+    (DDMMYYYY, MMDDYYYY, YYYYMMDD) with zero-padding for 7-digit values.
+
+    Returns (datetime_series, fmt) if successful, None otherwise.
+    """
+    digits = int_series.astype(str).str.len()
+    mostly_7_8 = ((digits >= 7) & (digits <= 8)).sum() / len(digits)
+
+    if mostly_7_8 <= 0.8:
+        return None
+
+    padded = int_series.astype(str).str.zfill(8)
+
+    for fmt in ("%d%m%Y", "%m%d%Y", "%Y%m%d"):
+        try:
+            candidate = pd.to_datetime(padded, format=fmt, errors="raise")
+            if not (
+                (candidate.dt.year >= 1900).all() and (candidate.dt.year <= 2100).all()
+            ):
+                continue
+
+            full_padded = (
+                pd.to_numeric(original_series, errors="coerce")
+                .astype("Int64")
+                .astype(str)
+                .str.zfill(8)
+                .replace("<NA>", pd.NaT)
+            )
+            result = pd.to_datetime(full_padded, format=fmt, errors="coerce")
+            return result, fmt
+
+        except Exception:
+            continue
+
+    return None
+
+
+def _try_downcast_to_int(series: pd.Series) -> pd.Series:
+    """
+    If all non-null values in a float series are whole numbers
+    (e.g. 553.0, 1733.0), convert to nullable Int64 so they
+    display as 553, 1733 instead of 553.0, 1733.0.
+
+    Uses Int64 (nullable) rather than int64 so NaN rows are preserved.
+    """
+    if series.dtype not in ("float64", "float32"):
+        return series
+
+    non_null = series.dropna()
+    if len(non_null) == 0:
+        return series
+
+    if (non_null == non_null.astype("int64")).all():
+        return series.astype("Int64")
+
+    return series
+
+
+def smart_cast_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Intelligently cast each column to the most appropriate type.
+
+    Processing order per column:
+    1. Already datetime → skip
+    2. Numeric dtype (int/float from Excel) → packed-date check,
+       then whole-number float → Int64 downcast
+    3. Object/string → try numeric cast (>70% parseable),
+       then packed-date check, then whole-number downcast
+    """
+    for col in df.columns:
+        dtype_str = str(df[col].dtype)
+
+        # Already datetime — nothing to do
+        if "datetime" in dtype_str:
+            continue
+
+        # ── Already numeric (common with Excel-loaded columns) ────────────
+        if df[col].dtype in ("int64", "int32", "float64", "float32", "Int64", "Int32"):
+            int_series = (
+                pd.to_numeric(df[col], errors="coerce").dropna().astype("int64")
+            )
+            if len(int_series) > 0:
+                result = _try_packed_date(int_series, df[col])
+                if result is not None:
+                    dt_col, fmt = result
+                    df[col] = dt_col
+                    print(f"📅 '{col}' detected as packed date ({fmt})")
+                    continue
+
+            # Downcast whole-number floats to Int64
+            df[col] = _try_downcast_to_int(df[col])
+            continue
+
+        # ── Object / string → try numeric cast first ──────────────────────
+        if df[col].dtype == object or dtype_str in ("string", "str"):
+            converted = pd.to_numeric(df[col], errors="coerce")
+            ratio = converted.notna().sum() / max(len(df), 1)
+
+            if ratio <= 0.7:
+                continue  # Not numeric enough — leave as object/string
+
+            int_series = converted.dropna().astype("int64")
+            if len(int_series) > 0:
+                result = _try_packed_date(int_series, df[col])
+                if result is not None:
+                    dt_col, fmt = result
+                    df[col] = dt_col
+                    print(f"📅 '{col}' detected as packed date ({fmt})")
+                    continue
+
+            # Downcast whole-number floats before storing
+            df[col] = _try_downcast_to_int(converted)
+
+    return df

app/executor/sheet_selector.py

@@ -0,0 +1,120 @@
+import pandas as pd
+from rich.console import Console
+from rich.table import Table
+from rich.prompt import Prompt
+
+console = Console()
+
+EXIT_WORDS = {"exit", "quit", "/exit", "/quit", "bye", "q", ":q"}
+
+
+def _ask(message: str) -> str:
+    """Prompt wrapper that raises UserExitError on exit words or Ctrl+C."""
+    # Import here to avoid circular import (main imports sheet_selector)
+    from app.cli.main import UserExitError
+
+    try:
+        value = Prompt.ask(message)
+    except (KeyboardInterrupt, EOFError):
+        raise UserExitError()
+    if value.strip().lower() in EXIT_WORDS:
+        raise UserExitError()
+    return value
+
+
+def get_sheet_info(file_path: str) -> dict:
+    """Returns {sheet_name: {"rows": int, "cols": int, "columns": [str]}}"""
+    xl = pd.ExcelFile(file_path)
+    info = {}
+    for name in xl.sheet_names:
+        try:
+            full_df = xl.parse(name)
+            info[name] = {
+                "rows": len(full_df),
+                "cols": len(full_df.columns),
+                "columns": full_df.columns.tolist(),
+            }
+        except Exception:
+            info[name] = {"rows": "?", "cols": "?", "columns": []}
+    return info
+
+
+def prompt_sheet_selection(file_path: str) -> list:
+    """
+    Interactive sheet picker.
+    Returns list of selected sheet names.
+    Raises UserExitError if the user types an exit command.
+    """
+    console.print("\n[bold cyan]📋 Excel Sheet Selection[/bold cyan]")
+
+    try:
+        sheet_info = get_sheet_info(file_path)
+    except Exception as e:
+        console.print(f"[red]❌ Could not read sheets: {e}[/red]")
+        return []
+
+    sheet_names = list(sheet_info.keys())
+
+    table = Table(title="Available Sheets", border_style="blue", show_lines=True)
+    table.add_column("#", style="bold yellow", width=4)
+    table.add_column("Sheet", style="bold white")
+    table.add_column("Rows", justify="right")
+    table.add_column("Columns", justify="right")
+    table.add_column("Sample columns", style="dim")
+
+    for i, name in enumerate(sheet_names, 1):
+        info = sheet_info[name]
+        sample = ", ".join(str(c) for c in info["columns"][:5])
+        if len(info["columns"]) > 5:
+            sample += f" … (+{len(info['columns']) - 5} more)"
+        table.add_row(str(i), name, str(info["rows"]), str(info["cols"]), sample)
+
+    console.print(table)
+    console.print(
+        "\n[dim]Options:[/dim]\n"
+        "  [yellow]•[/yellow] Sheet number(s) separated by commas: [bold]1[/bold] or [bold]1,2[/bold]\n"
+        "  [yellow]•[/yellow] Type [bold]all[/bold] to load all sheets\n"
+        "  [yellow]•[/yellow] Type [bold]exit[/bold] to quit\n"
+    )
+
+    while True:
+        raw = _ask("[cyan]👉 Select sheet(s)[/cyan]").strip().lower()
+
+        if raw == "all":
+            selected = sheet_names
+            break
+
+        parts = [p.strip() for p in raw.split(",")]
+        try:
+            indices = [int(p) for p in parts if p]
+            selected = []
+            valid = True
+            for idx in indices:
+                if 1 <= idx <= len(sheet_names):
+                    name = sheet_names[idx - 1]
+                    if name not in selected:
+                        selected.append(name)
+                else:
+                    console.print(
+                        f"[red]❌ '{idx}' is out of range (1–{len(sheet_names)})[/red]"
+                    )
+                    valid = False
+                    break
+            if valid and selected:
+                break
+        except ValueError:
+            console.print(
+                "[red]❌ Please enter numbers, 'all', or 'exit' to quit.[/red]"
+            )
+
+    if len(selected) == 1:
+        console.print(f"\n[green]✅ Loading sheet:[/green] [bold]{selected[0]}[/bold]")
+    else:
+        console.print(
+            f"\n[green]✅ Loading {len(selected)} sheets:[/green] "
+            f"[bold]{', '.join(selected)}[/bold]\n"
+            "[yellow]ℹ️  Sheets will be merged with a '_sheet' column added "
+            "so you can filter per sheet in queries.[/yellow]"
+        )
+
+    return selected

app/llm/ollama_client.py

@@ -0,0 +1,47 @@
+import requests
+
+
+class OllamaClient:
+    """
+    Handles all communication with Ollama LLM.
+    Optimized for speed + reliability.
+    """
+
+    def __init__(self, model="phi"):
+        self.url = "http://localhost:11434/api/generate"
+        self.model = model
+
+    def generate(self, prompt: str) -> str:
+        try:
+            response = requests.post(
+                self.url,
+                json={
+                    "model": self.model,
+                    "prompt": prompt,
+                    "stream": False,
+                    "options": {
+                        "num_predict": 100,
+                        "temperature": 0,
+                        "top_p": 0.9,
+                    },
+                },
+                timeout=30,
+            )
+
+            if response.status_code != 200:
+                return f"ERROR: Bad response {response.status_code}"
+
+            data = response.json()
+
+            # print("DEBUG FULL RESPONSE:", data)
+
+            return data.get("response", "").strip()
+
+        except requests.exceptions.ConnectionError:
+            return "ERROR: Could not connect to Ollama. Is it running?"
+
+        except requests.exceptions.Timeout:
+            return "ERROR: Ollama request timed out"
+
+        except Exception as e:
+            return f"ERROR: {str(e)}"

app/prompts/interpreter_prompt.txt

@@ -0,0 +1,28 @@
+You are a data analyst assistant.
+
+Your job is to convert a user query into structured intent.
+
+Dataset schema:
+{schema}
+
+User query:
+{query}
+
+Return ONLY JSON in this format:
+
+{
+  "metric": "<column_name>",
+  "dimension": "<column_name>",
+  "operation": "sum | mean | count",
+  "query_type": "comparison | aggregation | trend | top_n",
+  "limit": number or null
+}
+
+Rules:
+- Use only columns from schema
+- "revenue", "sales", "spending" → numeric columns
+- "highest", "most" → comparison
+- "top N" → top_n
+- "average" → mean
+- If unclear → choose best match
+- Do NOT explain anything

app/security/input_guard.py

@@ -0,0 +1,133 @@
+import re
+
+BLOCKED_KEYWORDS = ["password", "ssn", "credit card", "api key", "private key"]
+
+ANALYTICAL_KEYWORDS = {
+    # intent words
+    "highest",
+    "lowest",
+    "top",
+    "bottom",
+    "most",
+    "least",
+    "best",
+    "worst",
+    "average",
+    "avg",
+    "mean",
+    "total",
+    "sum",
+    "count",
+    "max",
+    "min",
+    "trend",
+    "over time",
+    "monthly",
+    "daily",
+    "weekly",
+    "yearly",
+    "compare",
+    "comparison",
+    "distribution",
+    "breakdown",
+    "ascending",
+    "descending",
+    "asc",
+    "desc",
+    "increasing",
+    "decreasing",
+    "lowest to highest",
+    "highest to lowest",
+    "sorted",
+    "order",
+    # question words
+    "show",
+    "give",
+    "find",
+    "list",
+    "get",
+    "what",
+    "which",
+    "how many",
+    "how much",
+    "where",
+    "who",
+    # common data domain words
+    "sales",
+    "revenue",
+    "profit",
+    "spend",
+    "spending",
+    "spent",
+    "cost",
+    "item",
+    "items",
+    "product",
+    "products",
+    "category",
+    "location",
+    "payment",
+    "method",
+    "customer",
+    "customers",
+    "order",
+    "orders",
+    "region",
+    "city",
+    "country",
+    "store",
+    "date",
+    "month",
+    "year",
+    "by",
+    "per",
+    "across",
+    "between",
+    "vs",
+    "versus",
+}
+
+
+class InputGuard:
+    def __init__(self, extra_domain_words=None):
+        """
+        extra_domain_words: pass your semantic_map column names here so that
+        queries referencing column names directly are always accepted.
+        Example: InputGuard(extra_domain_words=["corrected_t_spent", "payment_method"])
+        """
+        self._extra = set(extra_domain_words or [])
+
+    def run(self, context):
+        query = context.get("user_query", "").strip()
+
+        # --- Empty ---
+        if not query:
+            context["error"] = "Please enter a question."
+            return context
+
+        query_lower = query.lower()
+
+        # --- Sensitive content ---
+        for word in BLOCKED_KEYWORDS:
+            if word in query_lower:
+                context["error"] = (
+                    "⛔ Sensitive query detected. Please ask about your data."
+                )
+                return context
+
+        # --- Gibberish / no intent ---
+        domain_words = ANALYTICAL_KEYWORDS | self._extra
+        has_intent = any(kw in query_lower for kw in domain_words)
+
+        if not has_intent:
+            context["error"] = (
+                "❓ I couldn't understand that as a data question.\n\n"
+                "Try something like:\n"
+                "  • 'top 5 items by sales'\n"
+                "  • 'highest revenue by location'\n"
+                "  • 'average spend by payment method'\n"
+                "  • 'total sales trend over time'"
+            )
+            return context
+
+        return context

app/security/schema_filter.py

@@ -0,0 +1,20 @@
+SENSITIVE_COLUMNS = ["password", "ssn", "credit_card"]
+
+
+class SchemaFilter:
+    def run(self, context):
+        schema = context["schema"]
+
+        if not schema:
+            return context
+
+        filtered_columns = [
+            col
+            for col in schema["columns"]
+            if col["name"].lower() not in SENSITIVE_COLUMNS
+        ]
+
+        schema["columns"] = filtered_columns
+        context["schema"] = schema
+
+        return context

app/tests/llm_test.py

@@ -0,0 +1,18 @@
+from app.agents.llm_intepreter import LLMInterpreter
+
+context = {
+    "user_query": "Which location has highest revenue?",
+    "schema": {
+        "columns": [
+            {"name": "location"},
+            {"name": "payment_method"},
+            {"name": "total_spent"},
+        ]
+    },
+}
+
+agent = LLMInterpreter()
+
+result = agent.run(context)
+
+print(result["intent"])

app/tools/analyzer.py

@@ -0,0 +1,157 @@
+import pandas as pd
+
+
+class Analyzer:
+    """
+    Executes the structured intent produced by the interpreter.
+
+    Sheet-aware: if context["intent"]["sheet"] is set (e.g. the user said
+    "in sheet Orders"), only that sheet's rows are used for analysis.
+    Otherwise the full combined dataframe is used.
+
+    Supported query types
+    ---------------------
+    comparison  – groupby dimension, sum metric, sort descending
+    top_n       – like comparison, limited to N rows
+    aggregation – groupby dimension, mean or sum
+    trend       – groupby time dimension, sum, sort by index
+    """
+
+    def run(self, context: dict) -> dict:
+        intent = context.get("intent", {})
+
+        metric = intent.get("metric")
+        dimension = intent.get("dimension")
+        query_type = intent.get("query_type")
+        target_sheet = intent.get("sheet")  # set by InterpreterAgent for sheet queries
+
+        # ── Sheet-aware dataframe selection ──────────────────────────────
+        if target_sheet and target_sheet in context.get("sheet_dataframes", {}):
+            df = context["sheet_dataframes"][target_sheet].copy()
+        else:
+            df = context["dataframe"].copy()
+
+        # ── Guard: columns must exist ─────────────────────────────────────
+        all_columns = df.columns.tolist()
+        visible_cols = [c for c in all_columns if c != "_sheet"]
+
+        if not metric or metric not in all_columns:
+            numeric_in_sheet = df.select_dtypes(include="number").columns.tolist()
+            id_hints = {"id", "_id", "key", "code", "num", "no", "number"}
+            real_numeric = [
+                c for c in numeric_in_sheet if not any(h in c.lower() for h in id_hints)
+            ]
+
+            if target_sheet and not real_numeric:
+                context["error"] = (
+                    f"The '{target_sheet}' sheet has no numeric columns to measure.\n"
+                    f"  Columns in this sheet: {visible_cols}\n\n"
+                    f"This sheet is likely a lookup/reference table.\n"
+                    f"Try querying a sheet that has numeric data, like Orders."
+                )
+            elif target_sheet and real_numeric:
+                context["error"] = (
+                    f"'{metric}' is not available in the '{target_sheet}' sheet.\n"
+                    f"  Available numeric columns here: {real_numeric}\n"
+                    f"  Try: 'top 5 by {real_numeric[0]} in {target_sheet}'"
+                )
+            else:
+                context["error"] = (
+                    f"Metric column '{metric}' not found.\n"
+                    f"  Available columns: {visible_cols}"
+                )
+            return context
+
+        # Hard guard: internal/system columns must never be used as dimension
+        INTERNAL_COLS = {"_sheet"}
+        if dimension in INTERNAL_COLS:
+            context["error"] = (
+                f"'{dimension}' is an internal system column and cannot be "
+                f"used as a dimension.\n\n"
+                f"  Please rephrase and specify a real dimension column.\n"
+                f"  Available columns: {visible_cols}"
+            )
+            return context
+
+        if not dimension or dimension not in all_columns:
+            context["error"] = (
+                f"Dimension column '{dimension}' not found.\n"
+                f"  Available columns: {visible_cols}"
+            )
+            return context
+        if not query_type:
+            context["error"] = "No query type detected. Please rephrase your question."
+            return context
+
+        # ── Date granularity: group datetime columns by month/year/week ──
+        # If the dimension is a datetime column, extract the requested period
+        # so "which month" groups by month label, not individual dates.
+        if pd.api.types.is_datetime64_any_dtype(df[dimension]):
+            granularity = intent.get("time_granularity", "day")
+            if granularity == "year":
+                df[dimension] = df[dimension].dt.to_period("Y").astype(str)
+            elif granularity == "month":
+                df[dimension] = df[dimension].dt.to_period("M").astype(str)
+            elif granularity == "week":
+                df[dimension] = df[dimension].dt.to_period("W").astype(str)
+            else:
+                df[dimension] = df[dimension].dt.date.astype(str)
+        else:
+            # ── Clean categorical dimension ───────────────────────────────
+            df[dimension] = (
+                df[dimension]
+                .astype(str)
+                .str.strip()
+                .replace(["ERROR", "UNKNOWN", "Unknown", "nan", ""], "Unknown")
+            )
+
+        # ── Coerce metric to numeric ──────────────────────────────────────
+        df[metric] = pd.to_numeric(df[metric], errors="coerce")
+        df = df.dropna(subset=[metric])
+
+        if df.empty:
+            context["error"] = f"No numeric data found in '{metric}' after cleaning."
+            return context
+
+        # ── Run analysis ──────────────────────────────────────────────────
+        try:
+            ascending = intent.get("ascending", False)
+
+            if query_type == "comparison":
+                result = (
+                    df.groupby(dimension)[metric].sum().sort_values(ascending=ascending)
+                )
+
+            elif query_type == "top_n":
+                n = intent.get("limit") or 5
+                result = (
+                    df.groupby(dimension)[metric]
+                    .sum()
+                    .sort_values(ascending=ascending)
+                    .head(n)
+                )
+
+            elif query_type == "aggregation":
+                op = intent.get("operation", "sum")
+                result = (
+                    df.groupby(dimension)[metric].mean()
+                    if op == "mean"
+                    else df.groupby(dimension)[metric].sum()
+                )
+                # Always sort by value so display and insight are consistent
+                result = result.sort_values(ascending=ascending)
+
+            elif query_type == "trend":
+                result = df.groupby(dimension)[metric].sum().sort_index()
+
+            else:
+                context["error"] = f"Unsupported query type: '{query_type}'"
+                return context
+
+            context["analysis"] = result
+            context["target_sheet"] = target_sheet  # for InsightGenerator label
+            return context
+
+        except Exception as e:
+            context["error"] = f"Analysis failed: {e}"
+            return context