langid-chat 0.1.0__tar.gz

langid_chat-0.1.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: langid-chat
+Version: 0.1.0
+Summary: A portable language detector (15 languages incl. code-mixed Indian) with a chat web UI
+Author: orewamash
+License: MIT
+Project-URL: Homepage, https://github.com/orewamash/langid
+Keywords: language-detection,nlp,machine-learning,chatbot
+Classifier: Programming Language :: Python :: 3
+Classifier: Natural Language :: English
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: scikit-learn>=1.2
+Requires-Dist: numpy
+
+# Language Detection AI
+
+A classical machine-learning language detector: feed it a sentence, it tells
+you which language it is written in. Built with Python + scikit-learn using
+character n-grams + Naive Bayes / Linear SVC.
+
+Includes detection of **Roman-script code-mixed Indian languages** (Tanglish,
+Hinglish, Teluglish, Kanglish, Manglish).
+
+## Install & run the chatbot UI
+
+The UI ships as a pip-installable package (`langid-chat`) that bundles the
+trained model, so anyone can run it with a single command — no repo clone
+required.
+
+```bash
+pip install langid-chat
+
+# Launch the chat UI (http://127.0.0.1:8000)
+langid-web
+
+# Or use the CLI
+langid "Kya kar rahe ho bhai"
+langid --interactive
+```
+
+`langid-web` supports `--host` and `--port` (e.g. `langid-web --port 9000`).
+
+To install from a source checkout instead:
+
+```bash
+pip install -e .
+langid-web
+# or without installing at all:
+python web/app.py
+```
+
+## Quick start
+
+```bash
+# 0. (Optional) Re-fetch the code-mixed corpora into data/raw (cached by default)
+python src/fetch_codemix.py
+
+# 1. Prepare data -- the deployed model is trained on the FULL dataset with
+#    balanced class weights (keeps all code-mixed signal):
+python src/preprocess.py --no-balance
+python src/train.py --class-weight balanced
+
+# 2. Test new sentences
+python detect.py "Kya kar rahe ho bhai"
+python detect.py --interactive
+```
+
+> On Windows set `$env:PYTHONIOENCODING='utf-8'` first so multilingual text
+> prints correctly.
+
+`detect.py` also reports a **confidence score** per guess. Below a calibrated
+threshold it says `LOW CONFIDENCE` instead of guessing — short or ambiguous
+input (e.g. `ok`, `vanakkam`) gets flagged rather than answered wrongly.
+Tune with `--min-confidence 0.5`, or force an always-guess with `0`.
+
+## Languages supported (15)
+
+Tamil, Russian, Arabic (distinct scripts) · Spanish, Portuguese, French,
+Italian (linguistically close) · English, German, Dutch · plus code-mixed:
+Tanglish, Hinglish, Teluglish, Kanglish, Manglish.
+
+## Results
+
+Best model (LinearSVC, full data + balanced class weights) reaches **~98% test
+accuracy** across the 15 classes; script-distinct languages (Arabic, Russian,
+Tamil) hit ~100%. High-confidence calls (>= 45%) are ~99% correct on the test
+split; the only reliable misdetections left are 1–2 word Roman-script
+fragments, which are exactly the ones the confidence gate flags.
+
+**Limitations:** it only knows the 15 trained languages and still guesses on
+anything fed to it without confidences if you lower the threshold. Very short
+text is inherently unreliable and is flagged, not silently misanswered.
+
+See `results/results.md` for the full write-up and `project.md`/`tasks.md` for
+scope and tasks.

langid_chat-0.1.0/README.md

@@ -0,0 +1,81 @@
+# Language Detection AI
+
+A classical machine-learning language detector: feed it a sentence, it tells
+you which language it is written in. Built with Python + scikit-learn using
+character n-grams + Naive Bayes / Linear SVC.
+
+Includes detection of **Roman-script code-mixed Indian languages** (Tanglish,
+Hinglish, Teluglish, Kanglish, Manglish).
+
+## Install & run the chatbot UI
+
+The UI ships as a pip-installable package (`langid-chat`) that bundles the
+trained model, so anyone can run it with a single command — no repo clone
+required.
+
+```bash
+pip install langid-chat
+
+# Launch the chat UI (http://127.0.0.1:8000)
+langid-web
+
+# Or use the CLI
+langid "Kya kar rahe ho bhai"
+langid --interactive
+```
+
+`langid-web` supports `--host` and `--port` (e.g. `langid-web --port 9000`).
+
+To install from a source checkout instead:
+
+```bash
+pip install -e .
+langid-web
+# or without installing at all:
+python web/app.py
+```
+
+## Quick start
+
+```bash
+# 0. (Optional) Re-fetch the code-mixed corpora into data/raw (cached by default)
+python src/fetch_codemix.py
+
+# 1. Prepare data -- the deployed model is trained on the FULL dataset with
+#    balanced class weights (keeps all code-mixed signal):
+python src/preprocess.py --no-balance
+python src/train.py --class-weight balanced
+
+# 2. Test new sentences
+python detect.py "Kya kar rahe ho bhai"
+python detect.py --interactive
+```
+
+> On Windows set `$env:PYTHONIOENCODING='utf-8'` first so multilingual text
+> prints correctly.
+
+`detect.py` also reports a **confidence score** per guess. Below a calibrated
+threshold it says `LOW CONFIDENCE` instead of guessing — short or ambiguous
+input (e.g. `ok`, `vanakkam`) gets flagged rather than answered wrongly.
+Tune with `--min-confidence 0.5`, or force an always-guess with `0`.
+
+## Languages supported (15)
+
+Tamil, Russian, Arabic (distinct scripts) · Spanish, Portuguese, French,
+Italian (linguistically close) · English, German, Dutch · plus code-mixed:
+Tanglish, Hinglish, Teluglish, Kanglish, Manglish.
+
+## Results
+
+Best model (LinearSVC, full data + balanced class weights) reaches **~98% test
+accuracy** across the 15 classes; script-distinct languages (Arabic, Russian,
+Tamil) hit ~100%. High-confidence calls (>= 45%) are ~99% correct on the test
+split; the only reliable misdetections left are 1–2 word Roman-script
+fragments, which are exactly the ones the confidence gate flags.
+
+**Limitations:** it only knows the 15 trained languages and still guesses on
+anything fed to it without confidences if you lower the threshold. Very short
+text is inherently unreliable and is flagged, not silently misanswered.
+
+See `results/results.md` for the full write-up and `project.md`/`tasks.md` for
+scope and tasks.

langid_chat-0.1.0/langid/__init__.py

@@ -0,0 +1,12 @@
+"""langid — a portable, pip-installable language detector + chat web UI.
+
+This package bundles the trained character n-gram model and a dependency-free
+chat UI (stdlib http.server) so users can get a language detector with a single
+``pip install``:
+
+    pip install langid
+    langid-web            # http://127.0.0.1:8000 (chat UI)
+    langid "Kya kar rahe ho bhai"   # CLI detection
+"""
+
+__version__ = "0.1.0"

langid_chat-0.1.0/langid/cli.py

@@ -0,0 +1,59 @@
+"""Command-line entry point (``langid``)."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .language import (
+    DEFAULT_MIN_CONFIDENCE,
+    load_pipeline,
+    predict_with_confidence,
+    print_predictions,
+)
+
+
+def main(argv=None) -> None:
+    parser = argparse.ArgumentParser(description="Detect language of input text.")
+    parser.add_argument("texts", nargs="*", help="one or more texts to classify")
+    parser.add_argument("--interactive", action="store_true",
+                        help="prompt repeatedly for input")
+    parser.add_argument(
+        "--min-confidence", type=float, default=DEFAULT_MIN_CONFIDENCE,
+        help=f"flag guesses below this confidence (default {DEFAULT_MIN_CONFIDENCE})",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        vectorizer, model, languages = load_pipeline()
+    except FileNotFoundError:
+        print("No trained artifacts found. Install a build with artifacts, "
+              "or run src/train.py in a source checkout.")
+        return 1
+
+    if args.interactive:
+        print("Type text and press Enter (empty line to quit).")
+        while True:
+            text = input("> ").strip()
+            if not text:
+                break
+            preds, confs = predict_with_confidence(
+                vectorizer, model, languages, [text]
+            )
+            print_predictions([text], preds, confs, args.min_confidence)
+        return 0
+
+    texts = [t for t in args.texts if t.strip()]
+    if not texts:
+        parser.print_help()
+        return 0
+
+    preds, confs = predict_with_confidence(
+        vectorizer, model, languages, texts
+    )
+    print_predictions(texts, preds, confs, args.min_confidence)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

langid_chat-0.1.0/langid/language.py

@@ -0,0 +1,71 @@
+"""Prediction engine: load the saved model/vectorizer and classify text.
+
+Reuses the SAVED vectorizer (transform only -- never fit_transform on new
+data) and the saved model so predictions exactly match training. Reports a
+confidence score per guess; short/non-distinctive fragments are flagged rather
+than answered with a wrong guess.
+
+This module is importable both from a source checkout and from the installed
+``langid`` package (artifacts ship inside the wheel).
+"""
+
+from __future__ import annotations
+
+import math
+import pickle
+from pathlib import Path
+
+ARTIFACT_DIR = Path(__file__).resolve().parent / "artifacts"
+# Calibrated on the test split of the deployed LinearSVC: calls at/above 0.45
+# average ~99% correct; everything below is short/non-distinctive.
+DEFAULT_MIN_CONFIDENCE = 0.45
+
+
+def load_artifact(name: str):
+    with open(ARTIFACT_DIR / name, "rb") as f:
+        return pickle.load(f)
+
+
+def load_pipeline():
+    return (
+        load_artifact("vectorizer.pkl"),
+        load_artifact("model.pkl"),
+        load_artifact("languages.pkl"),
+    )
+
+
+def softmax(scores):
+    m = max(scores)
+    exps = [math.exp(s - m) for s in scores]
+    total = sum(exps)
+    return [e / total for e in exps]
+
+
+def predict_with_confidence(vectorizer, model, languages, texts):
+    """Return (prediction, confidence) per text.
+
+    MultinomialNB has real probabilities; LinearSVC only has decision scores,
+    so we normalise them with a softmax as a confidence proxy.
+    """
+    X_vec = vectorizer.transform(texts)
+    if hasattr(model, "predict_proba"):
+        proba = model.predict_proba(X_vec)
+        pred_idx = proba.argmax(axis=1)
+        confs = proba.max(axis=1)
+        return [languages[i] for i in pred_idx], confs
+    scores = model.decision_function(X_vec)
+    confs = []
+    for row in range(scores.shape[0]):
+        confs.append(max(softmax(scores[row].tolist())))
+    return list(model.predict(X_vec)), confs
+
+
+def print_predictions(texts, preds, confs, min_confidence):
+    for text, pred, conf in zip(texts, preds, confs):
+        if conf >= min_confidence:
+            print(f"{text!r} -> {pred} ({conf:.0%})")
+        else:
+            print(
+                f"{text!r} -> LOW CONFIDENCE ({conf:.0%}, best guess "
+                f"'{pred}') -- text too short/non-distinctive for a reliable call"
+            )

langid_chat-0.1.0/langid/server.py

@@ -0,0 +1,104 @@
+"""Chat web UI server (``langid-web``).
+
+Serves the chatbot-style interface and exposes one POST endpoint that runs the
+bundled model (same pipeline as the CLI). Pure stdlib -- no framework.
+
+    langid-web                # http://127.0.0.1:8000
+    langid-web --port 9000    # custom port
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from .language import load_pipeline, predict_with_confidence
+
+STATIC_DIR = Path(__file__).resolve().parent / "static"
+HOST = "127.0.0.1"
+PORT = 8000
+
+CONTENT_TYPES = {
+    ".html": "text/html; charset=utf-8",
+    ".css": "text/css; charset=utf-8",
+    ".js": "application/javascript; charset=utf-8",
+}
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "langid/1.0"
+
+    def _send(self, status, body, content_type):
+        data = body.encode("utf-8") if isinstance(body, str) else body
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self):
+        path = self.path.split("?")[0]
+        if path == "/":
+            path = "/index.html"
+        rel = path.lstrip("/")
+        file = (STATIC_DIR / rel).resolve()
+        if not str(file).startswith(str(STATIC_DIR.resolve())) or not file.is_file():
+            self._send(404, "not found", "text/plain; charset=utf-8")
+            return
+        fmt = CONTENT_TYPES.get(file.suffix, "application/octet-stream")
+        self._send(200, file.read_bytes(), fmt)
+
+    def do_POST(self):
+        if self.path.split("?")[0] != "/api/detect":
+            self._send(404, "not found", "text/plain; charset=utf-8")
+            return
+        length = int(self.headers.get("Content-Length", 0))
+        try:
+            payload = json.loads(self.rfile.read(length) or b"{}")
+        except json.JSONDecodeError:
+            payload = {}
+        text = (payload.get("text") or "").strip()
+        if not text:
+            self._send(400, json.dumps({"error": "empty text"}),
+                       "application/json; charset=utf-8")
+            return
+        try:
+            vectorizer, model, languages = load_pipeline()
+        except FileNotFoundError as exc:
+            self._send(500, json.dumps({"error": f"model unavailable: {exc}"}),
+                       "application/json; charset=utf-8")
+            return
+        (pred,), (conf,) = predict_with_confidence(
+            vectorizer, model, languages, [text]
+        )
+        self._send(200, json.dumps({
+            "text": text,
+            "language": pred,
+            "confidence": round(float(conf), 4),
+        }), "application/json; charset=utf-8")
+
+    def log_message(self, fmt, *args):
+        sys.stderr.write(f"[langid-web] {fmt % args}\n")
+
+
+def main(argv=None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--host", default=HOST)
+    parser.add_argument("--port", type=int, default=PORT)
+    args = parser.parse_args(argv)
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+    url = f"http://{args.host}:{args.port}"
+    print(f"langid chat UI running at {url}")
+    print("Press Ctrl+C to stop.")
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nstopped.")
+
+
+if __name__ == "__main__":
+    main()

langid_chat-0.1.0/langid/static/app.js

@@ -0,0 +1,213 @@
+(() => {
+  const chat = document.getElementById("chat");
+  const input = document.getElementById("input");
+  const sendBtn = document.getElementById("send");
+  const newChat = document.getElementById("newChat");
+  const burger = document.getElementById("burger");
+  const sidebar = document.getElementById("sidebar");
+  const brandSub = document.getElementById("brandSub");
+
+  const MIN_CONF = 0.45;
+  const SAMPLE_GROUPS = [
+    {
+      title: "Script-distinct",
+      tags: ["Tamil", "Russian", "Arabic"],
+      items: [
+        "வணக்கம் உலகம்",
+        "Привет, как дела?",
+        "السلام عليكم",
+      ],
+    },
+    {
+      title: "Romance & Germanic",
+      tags: ["Spanish", "French", "German"],
+      items: [
+        "Hola, ¿cómo estás?",
+        "Bonjour tout le monde",
+        "Das Wetter ist heute schön",
+      ],
+    },
+    {
+      title: "Code-mixed Indian",
+      tags: ["Tanglish", "Hinglish", "Manglish"],
+      items: [
+        "Namma ooru santhe hogona",
+        "Kya kar rahe ho bhai?",
+        "Njan nalla thirakil aanu",
+      ],
+    },
+  ];
+
+  const FLAVOR = {
+    English: "straight-up English",
+    French: "Français",
+    German: "Deutsch",
+    Spanish: "Español",
+    Portuguese: "Português",
+    Italian: "Italiano",
+    Dutch: "Nederlands",
+    Russian: "Русский",
+    Arabic: "العربية",
+    Tamil: "தமிழ்",
+    Hinglish: "Hinglish (Hindi + English)",
+    Tanglish: "Tanglish (Tamil + English)",
+    Teluglish: "Teluglish (Telugu + English)",
+    Kanglish: "Kanglish (Kannada + English)",
+    Manglish: "Manglish (Malayalam + English)",
+  };
+
+  let inner = null;
+
+  function ensureChat() {
+    if (!inner) {
+      inner = document.createElement("div");
+      inner.className = "chat-inner";
+      chat.appendChild(inner);
+    }
+    return inner;
+  }
+
+  function scrollDown() {
+    chat.scrollTop = chat.scrollHeight;
+  }
+
+  function addMsg(role, avatarEmoji, html) {
+    const wrap = document.createElement("div");
+    wrap.className = `msg ${role} done`;
+    wrap.innerHTML = `
+      <div class="avatar">${avatarEmoji}</div>
+      <div class="bubble">${html}</div>`;
+    ensureChat().appendChild(wrap);
+    scrollDown();
+    return wrap;
+  }
+
+  function greet() {
+    const container = ensureChat();
+    container.innerHTML = "";
+    const wrap = document.createElement("div");
+    wrap.className = "msg greet done";
+    wrap.innerHTML = `
+      <div class="bubble">
+        <div class="greet-logo">🌐</div>
+        <h2>Hey, I'm LangID</h2>
+        <p>Type any sentence and I'll tell you which language it's written in —
+           including code-mixed ones like Tanglish or Hinglish. I'll show my
+           confidence, and if a fragment is too short to call, I'll say so
+           instead of guessing.</p>
+        <div class="prompt-groups">
+          ${SAMPLE_GROUPS.map(g => `
+            <div class="prompt-group">
+              <div class="prompt-label">${g.title}</div>
+              <div class="chips">
+                ${g.items.map(s => `<button class="chip" data-tags="${g.tags.join(",")}">${s}</button>`).join("")}
+              </div>
+            </div>`).join("")}
+        </div>
+        <div class="greet-foot">15 languages · LinearSVC · character n-grams</div>
+      </div>`;
+    container.appendChild(wrap);
+    wrap.querySelectorAll(".chip").forEach(c => {
+      c.addEventListener("click", () => sendMessage(c.textContent));
+    });
+    scrollDown();
+  }
+
+  function typingMsg() {
+    const wrap = document.createElement("div");
+    wrap.className = "msg typing";
+    wrap.innerHTML = `
+      <div class="avatar">🧠</div>
+      <div class="bubble"><i></i><i></i><i></i></div>`;
+    ensureChat().appendChild(wrap);
+    scrollDown();
+    return wrap;
+  }
+
+  function replyFor(language, conf) {
+    const pct = Math.round(conf * 100);
+    const flavor = FLAVOR[language] || language;
+    if (conf >= MIN_CONF) {
+      return `That's <span class="lang">${flavor}</span> — I'm ${pct}% sure${pct >= 90 ? " 🎯" : ""}.`;
+    }
+    return `Hmm, I'm only ${pct}% confident, so I'd rather not guess wrong. ` +
+           `That text is too short or ambiguous — want to give me a longer sentence?`;
+  }
+
+  function sendMessage(text) {
+    if (!text || !text.trim()) return;
+    addMsg("user", "🙂", escapeHtml(text.trim()));
+    const typing = typingMsg();
+
+    const delay = 450 + Math.random() * 350;
+    setTimeout(async () => {
+      try {
+        const res = await fetch("/api/detect", {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ text: text.trim() }),
+        });
+        if (!res.ok) throw new Error(await res.text());
+        const data = await res.json();
+        const low = data.confidence < MIN_CONF;
+        typing.outerHTML = `
+          <div class="msg done">
+            <div class="avatar">🧠</div>
+            <div class="bubble">
+              ${replyFor(data.language, data.confidence)}
+              <span class="conf-pill ${low ? "low" : ""}">${Math.round(data.confidence * 100)}%</span>
+              <span class="bubble-note">detected with the same pipeline as detect.py</span>
+            </div>
+          </div>`;
+      } catch (err) {
+        typing.outerHTML = `
+          <div class="msg done">
+            <div class="avatar">🧠</div>
+            <div class="bubble">Something went wrong on my end (${escapeHtml(String(err))}). Try again?</div>
+          </div>`;
+      }
+      scrollDown();
+    }, delay);
+  }
+
+  function escapeHtml(s) {
+    return s.replace(/[&<>"']/g, c => ({
+      "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;",
+    }[c]));
+  }
+
+  function submit() {
+    const text = input.value;
+    if (!text.trim()) return;
+    input.value = "";
+    autoResize();
+    sendBtn.disabled = true;
+    sendMessage(text);
+  }
+
+  function autoResize() {
+    input.style.height = "auto";
+    input.style.height = Math.min(input.scrollHeight, 140) + "px";
+  }
+
+  input.addEventListener("input", () => {
+    autoResize();
+    sendBtn.disabled = !input.value.trim();
+  });
+  input.addEventListener("keydown", e => {
+    if (e.key === "Enter" && !e.shiftKey) {
+      e.preventDefault();
+      submit();
+    }
+  });
+  sendBtn.addEventListener("click", submit);
+  newChat.addEventListener("click", () => {
+    greet();
+    input.focus();
+    brandSub.textContent = "new conversation";
+  });
+  burger.addEventListener("click", () => sidebar.classList.toggle("closed"));
+
+  greet();
+  input.focus();
+})();

langid_chat-0.1.0/langid/static/index.html

@@ -0,0 +1,64 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>LangID — Language Chat</title>
+  <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>🌐</text></svg>">
+  <link rel="stylesheet" href="/style.css">
+</head>
+<body>
+  <aside class="sidebar" id="sidebar">
+    <div class="side-head">
+      <button class="new-chat" id="newChat">
+        <svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M12 5v14M5 12h14"/></svg>
+        New chat
+      </button>
+    </div>
+    <div class="side-label">Conversations</div>
+    <div class="convo-list">
+      <div class="convo active" data-title="Hello world, how are you?">
+        <svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>
+        <span>Language — boot</span>
+      </div>
+      <div class="convo" data-title="Vanakkam friends!">
+        <svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>
+        <span>Code-mixed samples</span>
+      </div>
+    </div>
+    <div class="side-foot">
+      <div class="chip-label">15 languages · LinearSVC · char n-grams</div>
+    </div>
+  </aside>
+
+  <main class="main">
+    <header class="topbar">
+      <button class="burger" id="burger" aria-label="Toggle sidebar">
+        <svg viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M3 6h18M3 12h18M3 18h18"/></svg>
+      </button>
+      <div class="brand">
+        <div class="logo">🌐</div>
+        <div>
+          <div class="brand-name">LangID</div>
+          <div class="brand-sub" id="brandSub">Powered by char n-grams</div>
+        </div>
+      </div>
+      <div class="status"><span class="dot"></span> model ready</div>
+    </header>
+
+    <section class="chat" id="chat" aria-live="polite"></section>
+
+    <footer class="composer">
+      <div class="composer-inner">
+        <textarea id="input" rows="1" placeholder="Type a sentence and I'll tell you its language…"></textarea>
+        <button id="send" class="send-btn" aria-label="Send" disabled>
+          <svg viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 2 11 13M22 2l-7 20-4-9-9-4z"/></svg>
+        </button>
+      </div>
+      <div class="composer-hint">LangID is trained on 15 languages — including Tanglish, Hinglish, Teluglish…</div>
+    </footer>
+  </main>
+
+  <script src="/app.js"></script>
+</body>
+</html>

langid_chat-0.1.0/langid/static/style.css

@@ -0,0 +1,343 @@
+:root {
+  --bg: #0b0c0f;
+  --bg-2: #0f1013;
+  --panel: #15161a;
+  --panel-2: #1b1d23;
+  --panel-3: #23252c;
+  --border: #24262e;
+  --border-2: #2e313a;
+  --text: #ededf2;
+  --muted: #9a9daa;
+  --faint: #676a76;
+  --accent: #8b5cf6;
+  --accent-2: #6366f1;
+  --accent-soft: rgba(139, 92, 246, .14);
+  --user-bubble: linear-gradient(135deg, #6d5cf6, #a855f7);
+  --amber: #f5a623;
+  --ok: #34d399;
+  --radius: 18px;
+  --topbar-h: 60px;
+}
+
+* { box-sizing: border-box; margin: 0; padding: 0; }
+
+html, body { height: 100%; }
+
+body {
+  background:
+    radial-gradient(1200px 600px at 90% -10%, rgba(139, 92, 246, .10), transparent 60%),
+    radial-gradient(1000px 500px at -10% 110%, rgba(99, 102, 241, .08), transparent 55%),
+    var(--bg);
+  color: var(--text);
+  font: 15px/1.55 system-ui, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+  display: flex;
+  overflow: hidden;
+  -webkit-font-smoothing: antialiased;
+}
+
+::selection { background: rgba(139, 92, 246, .35); }
+
+/* custom scrollbar (WebKit) */
+::-webkit-scrollbar { width: 10px; height: 10px; }
+::-webkit-scrollbar-thumb {
+  background: var(--border-2); border-radius: 99px;
+  border: 2px solid transparent; background-clip: padding-box;
+}
+::-webkit-scrollbar-thumb:hover { background: var(--faint); border-color: transparent; background-clip: padding-box; }
+::-webkit-scrollbar-track { background: transparent; }
+
+/* ---------- sidebar ---------- */
+.sidebar {
+  width: 272px;
+  flex: 0 0 272px;
+  background: var(--bg-2);
+  border-right: 1px solid var(--border);
+  display: flex;
+  flex-direction: column;
+  padding: 12px;
+  transition: margin-left .28s cubic-bezier(.4, 0, .2, 1);
+}
+.sidebar.closed { margin-left: -272px; }
+
+.new-chat {
+  display: flex; align-items: center; justify-content: center; gap: 8px;
+  background: #fff;
+  color: #0b0c0f;
+  border: none; cursor: pointer;
+  padding: 11px 14px; border-radius: 12px;
+  font: 600 14px system-ui, sans-serif;
+  transition: transform .12s ease, background .15s ease, box-shadow .15s ease;
+}
+.new-chat:hover {
+  transform: translateY(-1px);
+  background: #f2f2f5;
+  box-shadow: 0 8px 24px rgba(0, 0, 0, .3);
+}
+.new-chat:active { transform: translateY(0); }
+
+.side-label {
+  margin: 22px 4px 8px;
+  font-size: 11px; letter-spacing: .1em;
+  text-transform: uppercase; color: var(--faint);
+}
+
+.convo-list { display: flex; flex-direction: column; gap: 2px; overflow-y: auto; padding-right: 2px; }
+
+.convo {
+  display: flex; align-items: center; gap: 10px;
+  padding: 9px 10px; border-radius: 9px;
+  color: var(--muted); cursor: pointer;
+  white-space: nowrap; overflow: hidden;
+  border: 1px solid transparent;
+  transition: background .14s, color .14s, border-color .14s;
+}
+.convo span { overflow: hidden; text-overflow: ellipsis; }
+.convo svg { flex: 0 0 auto; color: var(--faint); }
+.convo:hover { background: var(--panel); color: var(--text); }
+.convo.active { background: var(--panel); color: var(--text); border-color: var(--border); }
+
+.side-foot { margin-top: auto; padding-top: 14px; border-top: 1px solid var(--border); }
+.chip-label {
+  font-size: 11px; color: var(--faint); text-align: center;
+  padding: 6px;
+}
+
+/* ---------- main ---------- */
+.main {
+  flex: 1;
+  display: flex; flex-direction: column;
+  min-width: 0;
+}
+
+.topbar {
+  display: flex; align-items: center; gap: 14px;
+  height: var(--topbar-h);
+  padding: 0 20px;
+  border-bottom: 1px solid var(--border);
+  background: rgba(11, 12, 15, .72);
+  backdrop-filter: blur(12px) saturate(1.2);
+  -webkit-backdrop-filter: blur(12px) saturate(1.2);
+  flex-shrink: 0;
+  position: relative; z-index: 5;
+}
+.burger {
+  background: transparent; border: none; cursor: pointer;
+  color: var(--muted); padding: 7px; border-radius: 8px;
+  transition: color .12s, background .12s;
+}
+.burger:hover { color: var(--text); background: var(--panel-2); }
+
+.brand { display: flex; align-items: center; gap: 11px; }
+.logo {
+  width: 34px; height: 34px; border-radius: 10px;
+  background: linear-gradient(135deg, var(--accent), var(--accent-2));
+  display: grid; place-items: center; font-size: 17px;
+  box-shadow: 0 4px 18px rgba(139, 92, 246, .45);
+}
+.brand-name { font-weight: 650; font-size: 15px; letter-spacing: -.01em; }
+.brand-sub { font-size: 12px; color: var(--muted); }
+
+.status {
+  margin-left: auto; display: flex; align-items: center; gap: 7px;
+  font-size: 12px; color: var(--muted);
+}
+.dot {
+  width: 8px; height: 8px; border-radius: 50%; background: var(--ok);
+  box-shadow: 0 0 0 0 rgba(52, 211, 153, .5);
+  animation: pulse 2.4s infinite;
+}
+@keyframes pulse {
+  0% { box-shadow: 0 0 0 0 rgba(52, 211, 153, .45); }
+  70% { box-shadow: 0 0 0 7px rgba(52, 211, 153, 0); }
+  100% { box-shadow: 0 0 0 0 rgba(52, 211, 153, 0); }
+}
+
+/* ---------- chat ---------- */
+.chat {
+  flex: 1;
+  overflow-y: auto;
+  padding: 28px 0 8px;
+  scroll-behavior: smooth;
+}
+.chat-inner {
+  max-width: 760px;
+  margin: 0 auto;
+  display: flex; flex-direction: column; gap: 22px;
+  padding: 0 20px;
+}
+
+.msg { display: flex; gap: 12px; align-items: flex-start; }
+.msg.user { flex-direction: row-reverse; }
+
+.avatar {
+  flex: 0 0 auto;
+  width: 32px; height: 32px; border-radius: 10px;
+  display: grid; place-items: center; font-size: 16px;
+  background: var(--panel-2); border: 1px solid var(--border);
+  box-shadow: 0 2px 8px rgba(0,0,0,.25);
+}
+.msg.user .avatar { background: linear-gradient(135deg, var(--accent), var(--accent-2)); }
+
+.bubble {
+  max-width: 76%;
+  padding: 12px 16px;
+  border-radius: var(--radius);
+  font-size: 15px;
+  background: var(--panel-2);
+  border: 1px solid var(--border);
+  box-shadow: 0 1px 2px rgba(0,0,0,.25);
+}
+.msg.user .bubble {
+  color: #fff;
+  background: var(--user-bubble);
+  border: none;
+  border-bottom-right-radius: 6px;
+  box-shadow: 0 4px 18px rgba(139, 92, 246, .25);
+}
+
+.msg .bubble .lang {
+  font-weight: 750;
+  background: linear-gradient(90deg, #a5b4fc, #c4b5fd);
+  -webkit-background-clip: text;
+  background-clip: text;
+  color: transparent;
+}
+
+.conf-pill {
+  display: inline-flex; align-items: center; gap: 5px;
+  margin-left: 8px;
+  padding: 2px 10px; border-radius: 99px;
+  font-size: 12px; font-weight: 650;
+  background: rgba(52, 211, 153, .13);
+  color: var(--ok);
+  border: 1px solid rgba(52, 211, 153, .2);
+  vertical-align: 2px;
+}
+.conf-pill.low { background: rgba(245, 166, 35, .12); color: var(--amber); border-color: rgba(245, 166, 35, .2); }
+
+.bubble-note { display: block; margin-top: 10px; font-size: 12.5px; color: var(--muted); }
+.msg.user .bubble-note { color: rgba(255,255,255,.82); }
+
+/* greeting / empty state */
+.greet { width: 100%; }
+.greet .bubble {
+  width: 100%; max-width: none;
+  text-align: center;
+  padding: 40px 28px 32px;
+  border-radius: 22px;
+  background:
+    radial-gradient(120% 150% at 50% -10%, rgba(139, 92, 246, .16), transparent 55%),
+    var(--panel);
+  border: 1px solid var(--border-2);
+}
+.greet-logo {
+  width: 56px; height: 56px; margin: 0 auto 18px;
+  border-radius: 16px; display: grid; place-items: center; font-size: 28px;
+  background: linear-gradient(135deg, var(--accent), var(--accent-2));
+  box-shadow: 0 8px 30px rgba(139, 92, 246, .5);
+}
+.greet .bubble h2 {
+  font-size: 22px; margin-bottom: 8px; letter-spacing: -.02em;
+  font-weight: 700;
+}
+.greet .bubble p {
+  color: var(--muted); max-width: 520px; margin: 0 auto 26px;
+}
+
+.prompt-groups { display: flex; flex-direction: column; gap: 22px; text-align: left; }
+.prompt-label {
+  font-size: 11px; letter-spacing: .1em; text-transform: uppercase;
+  color: var(--faint); margin-bottom: 9px;
+}
+.chips { display: flex; flex-wrap: wrap; gap: 8px; }
+.chip {
+  background: var(--bg-2);
+  border: 1px solid var(--border-2);
+  color: var(--text);
+  font: 13.5px system-ui, sans-serif;
+  padding: 8px 14px; border-radius: 99px; cursor: pointer;
+  transition: border-color .15s, background .15s, color .15s, transform .12s, box-shadow .15s;
+}
+.chip:hover {
+  border-color: var(--accent);
+  background: var(--accent-soft);
+  color: #d6ccfb;
+  transform: translateY(-1px);
+  box-shadow: 0 4px 14px rgba(139, 92, 246, .2);
+}
+.chip:active { transform: translateY(0); }
+
+.greet-foot {
+  margin-top: 26px; padding-top: 20px;
+  border-top: 1px solid var(--border);
+  font-size: 12px; color: var(--faint);
+}
+
+/* typing indicator */
+.typing .bubble { display: flex; gap: 5px; padding: 18px 20px; align-items: center; }
+.typing .bubble i {
+  width: 7px; height: 7px; border-radius: 50%;
+  background: var(--muted);
+  animation: bounce 1.2s infinite ease-in-out;
+}
+.typing .bubble i:nth-child(2) { animation-delay: .15s; }
+.typing .bubble i:nth-child(3) { animation-delay: .3s; }
+@keyframes bounce { 0%, 60%, 100% { transform: translateY(0); opacity: .45; } 30% { transform: translateY(-5px); opacity: 1; } }
+
+.msg.done .bubble { animation: pop .22s cubic-bezier(.2, .8, .3, 1.1); }
+@keyframes pop { from { transform: translateY(7px); opacity: 0; } to { transform: none; opacity: 1; } }
+
+/* ---------- composer ---------- */
+.composer {
+  flex-shrink: 0;
+  padding: 12px 20px 16px;
+  border-top: 1px solid var(--border);
+  background: linear-gradient(to top, var(--bg) 70%, transparent);
+}
+.composer-inner {
+  max-width: 760px; margin: 0 auto;
+  display: flex; align-items: flex-end; gap: 10px;
+  background: var(--panel-2);
+  border: 1px solid var(--border-2);
+  border-radius: 18px;
+  padding: 8px 8px 8px 16px;
+  box-shadow: 0 8px 30px rgba(0,0,0,.4);
+  transition: border-color .15s, box-shadow .15s;
+}
+.composer-inner:focus-within {
+  border-color: var(--accent);
+  box-shadow: 0 8px 34px rgba(139, 92, 246, .22);
+}
+
+.composer textarea {
+  flex: 1;
+  background: transparent; border: none; outline: none; resize: none;
+  color: var(--text); font: 15px/1.45 system-ui, sans-serif;
+  padding: 9px 0; max-height: 140px;
+}
+.composer textarea::placeholder { color: var(--faint); }
+
+.send-btn {
+  flex: 0 0 auto;
+  width: 40px; height: 40px; border-radius: 12px;
+  border: none; cursor: pointer; color: #0b0c0f; background: #fff;
+  display: grid; place-items: center;
+  transition: filter .12s, transform .12s, opacity .15s, background .15s;
+}
+.send-btn:hover:not(:disabled) { filter: brightness(.95); transform: translateY(-1px); }
+.send-btn:active:not(:disabled) { transform: translateY(0); }
+.send-btn:disabled { opacity: .32; cursor: default; }
+
+.composer-hint {
+  max-width: 760px; margin: 10px auto 0;
+  text-align: center; font-size: 12px; color: var(--faint);
+}
+
+/* ---------- responsive ---------- */
+@media (max-width: 760px) {
+  .sidebar { position: fixed; z-index: 20; height: 100%; }
+  .sidebar.closed { margin-left: -272px; }
+  .bubble { max-width: 86%; }
+  .status { display: none; }
+  .greet .bubble { padding: 28px 18px 24px; }
+}

langid_chat-0.1.0/langid_chat.egg-info/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: langid-chat
+Version: 0.1.0
+Summary: A portable language detector (15 languages incl. code-mixed Indian) with a chat web UI
+Author: orewamash
+License: MIT
+Project-URL: Homepage, https://github.com/orewamash/langid
+Keywords: language-detection,nlp,machine-learning,chatbot
+Classifier: Programming Language :: Python :: 3
+Classifier: Natural Language :: English
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: scikit-learn>=1.2
+Requires-Dist: numpy
+
+# Language Detection AI
+
+A classical machine-learning language detector: feed it a sentence, it tells
+you which language it is written in. Built with Python + scikit-learn using
+character n-grams + Naive Bayes / Linear SVC.
+
+Includes detection of **Roman-script code-mixed Indian languages** (Tanglish,
+Hinglish, Teluglish, Kanglish, Manglish).
+
+## Install & run the chatbot UI
+
+The UI ships as a pip-installable package (`langid-chat`) that bundles the
+trained model, so anyone can run it with a single command — no repo clone
+required.
+
+```bash
+pip install langid-chat
+
+# Launch the chat UI (http://127.0.0.1:8000)
+langid-web
+
+# Or use the CLI
+langid "Kya kar rahe ho bhai"
+langid --interactive
+```
+
+`langid-web` supports `--host` and `--port` (e.g. `langid-web --port 9000`).
+
+To install from a source checkout instead:
+
+```bash
+pip install -e .
+langid-web
+# or without installing at all:
+python web/app.py
+```
+
+## Quick start
+
+```bash
+# 0. (Optional) Re-fetch the code-mixed corpora into data/raw (cached by default)
+python src/fetch_codemix.py
+
+# 1. Prepare data -- the deployed model is trained on the FULL dataset with
+#    balanced class weights (keeps all code-mixed signal):
+python src/preprocess.py --no-balance
+python src/train.py --class-weight balanced
+
+# 2. Test new sentences
+python detect.py "Kya kar rahe ho bhai"
+python detect.py --interactive
+```
+
+> On Windows set `$env:PYTHONIOENCODING='utf-8'` first so multilingual text
+> prints correctly.
+
+`detect.py` also reports a **confidence score** per guess. Below a calibrated
+threshold it says `LOW CONFIDENCE` instead of guessing — short or ambiguous
+input (e.g. `ok`, `vanakkam`) gets flagged rather than answered wrongly.
+Tune with `--min-confidence 0.5`, or force an always-guess with `0`.
+
+## Languages supported (15)
+
+Tamil, Russian, Arabic (distinct scripts) · Spanish, Portuguese, French,
+Italian (linguistically close) · English, German, Dutch · plus code-mixed:
+Tanglish, Hinglish, Teluglish, Kanglish, Manglish.
+
+## Results
+
+Best model (LinearSVC, full data + balanced class weights) reaches **~98% test
+accuracy** across the 15 classes; script-distinct languages (Arabic, Russian,
+Tamil) hit ~100%. High-confidence calls (>= 45%) are ~99% correct on the test
+split; the only reliable misdetections left are 1–2 word Roman-script
+fragments, which are exactly the ones the confidence gate flags.
+
+**Limitations:** it only knows the 15 trained languages and still guesses on
+anything fed to it without confidences if you lower the threshold. Very short
+text is inherently unreliable and is flagged, not silently misanswered.
+
+See `results/results.md` for the full write-up and `project.md`/`tasks.md` for
+scope and tasks.

langid_chat-0.1.0/langid_chat.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+pyproject.toml
+langid/__init__.py
+langid/cli.py
+langid/language.py
+langid/server.py
+langid/artifacts/languages.pkl
+langid/artifacts/model.pkl
+langid/artifacts/vectorizer.pkl
+langid/static/app.js
+langid/static/index.html
+langid/static/style.css
+langid_chat.egg-info/PKG-INFO
+langid_chat.egg-info/SOURCES.txt
+langid_chat.egg-info/dependency_links.txt
+langid_chat.egg-info/entry_points.txt
+langid_chat.egg-info/requires.txt
+langid_chat.egg-info/top_level.txt

langid_chat-0.1.0/langid_chat.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

langid_chat-0.1.0/langid_chat.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+langid = langid.cli:main
+langid-web = langid.server:main

langid_chat-0.1.0/langid_chat.egg-info/requires.txt

@@ -0,0 +1,2 @@
+scikit-learn>=1.2
+numpy

langid_chat-0.1.0/langid_chat.egg-info/top_level.txt

@@ -0,0 +1 @@
+langid

langid_chat-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "langid-chat"
+version = "0.1.0"
+description = "A portable language detector (15 languages incl. code-mixed Indian) with a chat web UI"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "orewamash" }]
+keywords = ["language-detection", "nlp", "machine-learning", "chatbot"]
+urls = { Homepage = "https://github.com/orewamash/langid" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Natural Language :: English",
+    "Topic :: Text Processing :: Linguistic",
+]
+dependencies = [
+    "scikit-learn>=1.2",
+    "numpy",
+]
+
+[project.scripts]
+langid = "langid.cli:main"
+langid-web = "langid.server:main"
+
+[tool.setuptools]
+packages = ["langid"]
+
+[tool.setuptools.package-data]
+langid = [
+    "static/*.html",
+    "static/*.css",
+    "static/*.js",
+    "artifacts/*.pkl",
+]

langid_chat-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+