icdev 0.0.3__py3-none-any.whl

tools/writing/grammar_checker.py

@@ -0,0 +1,270 @@
+"""WriteGuard Grammar Checker — Deterministic regex-based grammar analysis (D-WG-2).
+
+Runs regex patterns from common_errors.json against input text.
+No LLM required — fully air-gap safe, reproducible results.
+"""
+
+import json
+import re
+from pathlib import Path
+
+
+# Default rules file path
+_RULES_DIR = Path(__file__).resolve().parent.parent.parent / "context" / "writing" / "grammar_rules"
+_DEFAULT_RULES_FILE = _RULES_DIR / "common_errors.json"
+_DEFAULT_VOCAB_FILE = _RULES_DIR / "govcon_vocabulary.json"
+
+
+def _load_rules(rules_file=None):
+    """Load grammar rules from JSON file."""
+    path = Path(rules_file) if rules_file else _DEFAULT_RULES_FILE
+    if not path.exists():
+        return []
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return data.get("rules", [])
+
+
+def _load_vocabulary(vocab_file=None):
+    """Load govcon vocabulary from JSON file."""
+    path = Path(vocab_file) if vocab_file else _DEFAULT_VOCAB_FILE
+    if not path.exists():
+        return {}
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def check_grammar(text, rules_file=None, categories=None, max_findings=50):
+    """Run grammar check on text using regex patterns.
+
+    Args:
+        text: Input text to analyze.
+        rules_file: Path to custom rules JSON. Defaults to common_errors.json.
+        categories: List of categories to check. None = all.
+        max_findings: Maximum findings to return per category.
+
+    Returns:
+        dict with 'findings' list and 'summary' stats.
+    """
+    if not text or not text.strip():
+        return {"findings": [], "summary": {"total": 0, "by_category": {}, "by_severity": {}}}
+
+    rules = _load_rules(rules_file)
+    findings = []
+    lines = text.split("\n")
+
+    for rule in rules:
+        if categories and rule.get("category") not in categories:
+            continue
+
+        pattern = rule.get("pattern", "")
+        if not pattern:
+            continue
+
+        try:
+            compiled = re.compile(pattern, re.IGNORECASE | re.MULTILINE)
+        except re.error:
+            continue
+
+        category_count = 0
+        for line_num, line in enumerate(lines, 1):
+            if category_count >= max_findings:
+                break
+
+            for match in compiled.finditer(line):
+                if category_count >= max_findings:
+                    break
+
+                finding = {
+                    "rule_id": rule.get("id", ""),
+                    "category": rule.get("category", "grammar"),
+                    "severity": rule.get("severity", "info"),
+                    "message": rule.get("message", ""),
+                    "suggestion": rule.get("suggestion", ""),
+                    "context": _get_context(line, match.start(), match.end()),
+                    "line_number": line_num,
+                    "char_offset": match.start(),
+                    "char_length": match.end() - match.start(),
+                    "matched_text": match.group(0),
+                    "auto_fixable": rule.get("auto_fixable", False),
+                    "confidence": 1.0,
+                }
+                findings.append(finding)
+                category_count += 1
+
+    # Build summary
+    by_category = {}
+    by_severity = {}
+    for f in findings:
+        cat = f["category"]
+        sev = f["severity"]
+        by_category[cat] = by_category.get(cat, 0) + 1
+        by_severity[sev] = by_severity.get(sev, 0) + 1
+
+    return {
+        "findings": findings,
+        "summary": {
+            "total": len(findings),
+            "by_category": by_category,
+            "by_severity": by_severity,
+        },
+    }
+
+
+def check_passive_voice(text):
+    """Calculate passive voice percentage in text.
+
+    Returns:
+        dict with 'passive_count', 'total_sentences', 'passive_pct', 'instances'.
+    """
+    if not text or not text.strip():
+        return {"passive_count": 0, "total_sentences": 0, "passive_pct": 0.0, "instances": []}
+
+    sentences = _split_sentences(text)
+    total = len(sentences)
+    if total == 0:
+        return {"passive_count": 0, "total_sentences": 0, "passive_pct": 0.0, "instances": []}
+
+    passive_pattern = re.compile(
+        r"\b(?:is|are|was|were|be|been|being)\s+(?:\w+ly\s+)?"
+        r"(?:accomplished|achieved|administered|affected|analyzed|applied|approved|"
+        r"assessed|assigned|authorized|built|calculated|completed|conducted|configured|"
+        r"considered|controlled|created|defined|delivered|deployed|described|designed|"
+        r"detected|determined|developed|documented|ensured|established|evaluated|"
+        r"executed|expected|generated|given|identified|implemented|improved|included|"
+        r"indicated|installed|integrated|intended|known|maintained|managed|made|met|"
+        r"mitigated|monitored|needed|noted|observed|obtained|operated|performed|planned|"
+        r"prepared|presented|processed|produced|prohibited|proposed|protected|provided|"
+        r"published|received|recommended|recorded|reduced|referenced|removed|replaced|"
+        r"reported|required|resolved|restricted|reviewed|satisfied|scanned|secured|"
+        r"selected|sent|specified|stored|submitted|supported|tested|tracked|"
+        r"transmitted|updated|used|validated|verified|written)\b",
+        re.IGNORECASE,
+    )
+
+    instances = []
+    for i, sent in enumerate(sentences):
+        if passive_pattern.search(sent):
+            instances.append({"sentence_index": i, "text": sent.strip()[:120]})
+
+    passive_count = len(instances)
+    passive_pct = (passive_count / total * 100) if total > 0 else 0.0
+
+    return {
+        "passive_count": passive_count,
+        "total_sentences": total,
+        "passive_pct": round(passive_pct, 1),
+        "instances": instances,
+    }
+
+
+def check_sentence_length(text, max_words=35):
+    """Analyze sentence length distribution.
+
+    Returns:
+        dict with 'avg_length', 'max_length', 'long_sentences', 'distribution'.
+    """
+    if not text or not text.strip():
+        return {"avg_length": 0.0, "max_length": 0, "long_sentences": [], "distribution": {}}
+
+    sentences = _split_sentences(text)
+    if not sentences:
+        return {"avg_length": 0.0, "max_length": 0, "long_sentences": [], "distribution": {}}
+
+    lengths = [len(s.split()) for s in sentences if s.strip()]
+    if not lengths:
+        return {"avg_length": 0.0, "max_length": 0, "long_sentences": [], "distribution": {}}
+
+    avg = sum(lengths) / len(lengths)
+    max_len = max(lengths)
+
+    long = []
+    for i, (sent, wc) in enumerate(zip(sentences, lengths)):
+        if wc > max_words:
+            long.append({"sentence_index": i, "word_count": wc, "text": sent.strip()[:120]})
+
+    # Distribution buckets
+    dist = {"short_1_10": 0, "medium_11_20": 0, "long_21_35": 0, "very_long_36_plus": 0}
+    for wc in lengths:
+        if wc <= 10:
+            dist["short_1_10"] += 1
+        elif wc <= 20:
+            dist["medium_11_20"] += 1
+        elif wc <= 35:
+            dist["long_21_35"] += 1
+        else:
+            dist["very_long_36_plus"] += 1
+
+    return {
+        "avg_length": round(avg, 1),
+        "max_length": max_len,
+        "long_sentences": long,
+        "distribution": dist,
+        "total_sentences": len(lengths),
+    }
+
+
+def check_govcon_vocabulary(text, vocab_file=None):
+    """Check for proper use of govcon vocabulary and acronyms.
+
+    Returns:
+        dict with 'undefined_acronyms', 'confused_pairs', 'findings'.
+    """
+    if not text or not text.strip():
+        return {"undefined_acronyms": [], "confused_pairs": [], "findings": []}
+
+    vocab = _load_vocabulary(vocab_file)
+    findings = []
+    acronyms = vocab.get("acronyms", {})
+    confused = vocab.get("confused_pairs", [])
+
+    # Check for undefined acronyms (uppercase 2-6 letter words not in vocabulary)
+    found_acronyms = set(re.findall(r"\b[A-Z]{2,6}\b", text))
+    undefined = []
+    for acr in found_acronyms:
+        if acr not in acronyms and acr not in {"US", "IT", "AI", "ML", "II", "III", "IV"}:
+            # Check if it's defined at first use in the text
+            definition_pattern = re.compile(
+                rf"[A-Za-z\s]+\({re.escape(acr)}\)|{re.escape(acr)}\s*[-—]\s*[A-Z]",
+            )
+            if not definition_pattern.search(text):
+                undefined.append(acr)
+
+    # Check for confused pairs
+    confused_found = []
+    for pair in confused:
+        wrong = pair.get("wrong", "")
+        if wrong and re.search(rf"\b{re.escape(wrong)}\b", text, re.IGNORECASE):
+            confused_found.append(pair)
+
+    return {
+        "undefined_acronyms": sorted(undefined),
+        "confused_pairs": confused_found,
+        "findings": findings,
+    }
+
+
+def _get_context(line, start, end, context_chars=40):
+    """Get surrounding context for a match."""
+    ctx_start = max(0, start - context_chars)
+    ctx_end = min(len(line), end + context_chars)
+    prefix = "..." if ctx_start > 0 else ""
+    suffix = "..." if ctx_end < len(line) else ""
+    return f"{prefix}{line[ctx_start:ctx_end]}{suffix}"
+
+
+def _split_sentences(text):
+    """Split text into sentences using regex heuristic."""
+    # Handle common abbreviations
+    cleaned = text
+    abbrevs = ["Mr.", "Mrs.", "Ms.", "Dr.", "Jr.", "Sr.", "St.", "vs.", "etc.", "i.e.", "e.g.", "U.S.", "Gov."]
+    for abbr in abbrevs:
+        cleaned = cleaned.replace(abbr, abbr.replace(".", "<!DOT!>"))
+
+    # Split on sentence boundaries
+    sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z])", cleaned)
+
+    # Restore abbreviation dots
+    sentences = [s.replace("<!DOT!>", ".") for s in sentences]
+
+    return [s for s in sentences if s.strip()]

tools/writing/plagiarism_detector.py

@@ -0,0 +1,106 @@
+"""WriteGuard Plagiarism Detector — RAG similarity wrapper (D-WG-5).
+
+Uses RAGRetriever.search() with 0.85 cosine similarity threshold.
+Excludes self-match via content hash.
+"""
+
+import hashlib
+
+
+def check_plagiarism(text, threshold=0.85, source_types=None, db_path=None):
+    """Check text against internal corpus for plagiarism.
+
+    Uses RAG vector similarity search against known content sources.
+
+    Args:
+        text: Input text to check.
+        threshold: Similarity threshold (0-1). Default 0.85.
+        source_types: List of RAG source types to search.
+        db_path: Optional database path.
+
+    Returns:
+        dict with 'max_similarity', 'matches', 'findings'.
+    """
+    if not text or not text.strip() or len(text) < 50:
+        return {"max_similarity": 0.0, "matches": [], "findings": []}
+
+    content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()[:32]
+
+    if source_types is None:
+        source_types = [
+            "proposal_knowledge_base",
+            "proposal_section_drafts",
+            "wg_snippets",
+        ]
+
+    matches = []
+    max_similarity = 0.0
+
+    try:
+        from tools.rag.retriever import RAGRetriever
+
+        retriever = RAGRetriever()
+
+        # Break text into chunks for comparison
+        chunks = _chunk_text(text, chunk_size=500, overlap=100)
+
+        for chunk in chunks:
+            results = retriever.search(
+                query=chunk,
+                top_k=5,
+                source_types=source_types,
+            )
+
+            for r in results:
+                similarity = r.get("score", 0.0)
+                # Exclude self-match
+                chunk_hash = hashlib.sha256(
+                    r.get("content", "").encode("utf-8")
+                ).hexdigest()[:32]
+                if chunk_hash == content_hash:
+                    continue
+
+                if similarity >= threshold:
+                    matches.append({
+                        "similarity": round(similarity, 3),
+                        "source_type": r.get("source_type", "unknown"),
+                        "source_id": r.get("id", ""),
+                        "matched_text": r.get("content", "")[:200],
+                        "query_chunk": chunk[:200],
+                    })
+                    max_similarity = max(max_similarity, similarity)
+
+    except (ImportError, Exception):
+        # RAG not available — return no matches
+        return {"max_similarity": 0.0, "matches": [], "findings": [],
+                "note": "RAG retriever unavailable"}
+
+    findings = []
+    if matches:
+        findings.append({
+            "category": "plagiarism",
+            "severity": "high" if max_similarity >= 0.95 else "medium",
+            "message": f"Found {len(matches)} similar passage(s) (max similarity: {max_similarity:.0%}).",
+            "rule_id": "PLAG-001",
+            "confidence": max_similarity,
+        })
+
+    return {
+        "max_similarity": round(max_similarity, 3),
+        "matches": matches,
+        "findings": findings,
+        "chunks_checked": len(chunks) if 'chunks' in dir() else 0,
+    }
+
+
+def _chunk_text(text, chunk_size=500, overlap=100):
+    """Split text into overlapping chunks for comparison."""
+    words = text.split()
+    chunks = []
+
+    for i in range(0, len(words), chunk_size - overlap):
+        chunk = " ".join(words[i : i + chunk_size])
+        if len(chunk.strip()) > 50:
+            chunks.append(chunk)
+
+    return chunks if chunks else [text]

tools/writing/readability_scorer.py

@@ -0,0 +1,201 @@
+"""WriteGuard Readability Scorer — Deterministic readability formulas (D-WG-2).
+
+Implements 5 readability formulas without NLTK dependency.
+Syllable counting via regex heuristic — fully air-gap safe.
+"""
+
+import re
+import math
+
+
+def score_readability(text):
+    """Calculate readability scores using 5 formulas.
+
+    Args:
+        text: Input text to analyze.
+
+    Returns:
+        dict with scores for each formula, grade level, and interpretation.
+    """
+    if not text or not text.strip():
+        return _empty_result()
+
+    words = _get_words(text)
+    sentences = _split_sentences(text)
+    syllables = [_count_syllables(w) for w in words]
+
+    word_count = len(words)
+    sentence_count = len(sentences)
+    syllable_count = sum(syllables)
+    polysyllable_count = sum(1 for s in syllables if s >= 3)
+    char_count = sum(len(w) for w in words)
+
+    if word_count == 0 or sentence_count == 0:
+        return _empty_result()
+
+    avg_words_per_sentence = word_count / sentence_count
+    avg_syllables_per_word = syllable_count / word_count
+
+    # 1. Flesch-Kincaid Grade Level
+    fk_grade = (0.39 * avg_words_per_sentence) + (11.8 * avg_syllables_per_word) - 15.59
+
+    # 2. Flesch Reading Ease (higher = easier)
+    fk_ease = 206.835 - (1.015 * avg_words_per_sentence) - (84.6 * avg_syllables_per_word)
+    fk_ease = max(0.0, min(100.0, fk_ease))
+
+    # 3. Gunning Fog Index
+    fog = 0.4 * (avg_words_per_sentence + (100.0 * polysyllable_count / word_count))
+
+    # 4. SMOG Index (requires >= 30 sentences for accuracy)
+    if sentence_count >= 3:
+        smog = 1.0430 * math.sqrt(polysyllable_count * (30.0 / sentence_count)) + 3.1291
+    else:
+        smog = fk_grade  # Fallback to FK when too few sentences
+
+    # 5. Coleman-Liau Index
+    L = (char_count / word_count) * 100  # avg chars per 100 words
+    S = (sentence_count / word_count) * 100  # avg sentences per 100 words
+    cli = (0.0588 * L) - (0.296 * S) - 15.8
+
+    # 6. Automated Readability Index (ARI)
+    ari = (4.71 * (char_count / word_count)) + (0.5 * avg_words_per_sentence) - 21.43
+
+    # Average grade level (excluding Flesch Reading Ease which is a different scale)
+    grade_level = (fk_grade + fog + smog + cli + ari) / 5.0
+
+    return {
+        "flesch_kincaid_grade": round(fk_grade, 1),
+        "flesch_reading_ease": round(fk_ease, 1),
+        "gunning_fog": round(fog, 1),
+        "smog": round(smog, 1),
+        "coleman_liau": round(cli, 1),
+        "ari": round(ari, 1),
+        "grade_level": round(grade_level, 1),
+        "interpretation": _interpret_grade(grade_level),
+        "ease_interpretation": _interpret_ease(fk_ease),
+        "stats": {
+            "word_count": word_count,
+            "sentence_count": sentence_count,
+            "syllable_count": syllable_count,
+            "polysyllable_count": polysyllable_count,
+            "char_count": char_count,
+            "avg_words_per_sentence": round(avg_words_per_sentence, 1),
+            "avg_syllables_per_word": round(avg_syllables_per_word, 2),
+        },
+    }
+
+
+def _count_syllables(word):
+    """Count syllables in a word using regex heuristic.
+
+    Uses the English vowel-cluster method with common exceptions.
+    """
+    word = word.lower().strip()
+    if not word:
+        return 0
+    if len(word) <= 2:
+        return 1
+
+    # Remove trailing silent e
+    if word.endswith("e") and not word.endswith("le") and len(word) > 2:
+        word = word[:-1]
+
+    # Count vowel groups
+    vowel_groups = re.findall(r"[aeiouy]+", word)
+    count = len(vowel_groups)
+
+    # Handle common patterns
+    # -ed endings (walked = 1 syl, not 2)
+    if word.endswith("ed") and len(word) > 3 and word[-3] not in "td":
+        count = max(1, count - 1)
+
+    # -es endings
+    if word.endswith("es") and not word.endswith(("ses", "zes", "xes", "ches", "shes")):
+        count = max(1, count - 1)
+
+    # Compound vowels that are one syllable
+    for diphthong in ["ai", "au", "ay", "ea", "ee", "ei", "ey", "ie", "oa", "oo", "ou", "oy", "ue", "ui"]:
+        if diphthong in word:
+            occurrences = word.count(diphthong)
+            # Already counted as one group by regex, so no adjustment needed
+            pass
+
+    return max(1, count)
+
+
+def _get_words(text):
+    """Extract words from text."""
+    return re.findall(r"[a-zA-Z']+", text)
+
+
+def _split_sentences(text):
+    """Split text into sentences."""
+    # Handle common abbreviations
+    cleaned = text
+    abbrevs = ["Mr.", "Mrs.", "Ms.", "Dr.", "Jr.", "Sr.", "St.", "vs.", "etc.", "i.e.", "e.g.", "U.S.", "Gov."]
+    for abbr in abbrevs:
+        cleaned = cleaned.replace(abbr, abbr.replace(".", "<!DOT!>"))
+
+    sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z])", cleaned)
+    sentences = [s.replace("<!DOT!>", ".") for s in sentences]
+    return [s for s in sentences if s.strip() and len(s.split()) >= 2]
+
+
+def _interpret_grade(grade):
+    """Interpret grade level for government writing context."""
+    if grade <= 6:
+        return "Very Easy — suitable for general public"
+    elif grade <= 8:
+        return "Easy — 6th-8th grade level"
+    elif grade <= 10:
+        return "Standard — 8th-10th grade level"
+    elif grade <= 12:
+        return "Moderately Difficult — high school level"
+    elif grade <= 14:
+        return "Difficult — college level (acceptable for technical gov docs)"
+    elif grade <= 16:
+        return "Very Difficult — graduate level (may need simplification)"
+    else:
+        return "Extremely Difficult — consider significant simplification"
+
+
+def _interpret_ease(ease):
+    """Interpret Flesch Reading Ease score."""
+    if ease >= 90:
+        return "Very Easy"
+    elif ease >= 80:
+        return "Easy"
+    elif ease >= 70:
+        return "Fairly Easy"
+    elif ease >= 60:
+        return "Standard"
+    elif ease >= 50:
+        return "Fairly Difficult"
+    elif ease >= 30:
+        return "Difficult"
+    else:
+        return "Very Difficult"
+
+
+def _empty_result():
+    """Return empty result for empty input."""
+    return {
+        "flesch_kincaid_grade": 0.0,
+        "flesch_reading_ease": 0.0,
+        "gunning_fog": 0.0,
+        "smog": 0.0,
+        "coleman_liau": 0.0,
+        "ari": 0.0,
+        "grade_level": 0.0,
+        "interpretation": "No text to analyze",
+        "ease_interpretation": "N/A",
+        "stats": {
+            "word_count": 0,
+            "sentence_count": 0,
+            "syllable_count": 0,
+            "polysyllable_count": 0,
+            "char_count": 0,
+            "avg_words_per_sentence": 0.0,
+            "avg_syllables_per_word": 0.0,
+        },
+    }

tools/writing/rewriter.py

@@ -0,0 +1,96 @@
+"""WriteGuard Rewriter — LLM-assisted text improvement (D-WG-12, worker tier).
+
+Takes text + findings → improved version via qwen3 draft → Claude review.
+"""
+
+
+def rewrite(text, findings=None, style="government", max_length=5000):
+    """Rewrite text incorporating analysis findings.
+
+    Uses two-tier LLM routing: qwen3 drafts improvement, Claude reviews.
+    Falls back to deterministic suggestions if LLM unavailable.
+
+    Args:
+        text: Original text to improve.
+        findings: List of analysis findings to address.
+        style: Target writing style.
+        max_length: Maximum text length to process.
+
+    Returns:
+        dict with 'rewritten_text', 'changes_made', 'status'.
+    """
+    if not text or not text.strip():
+        return {"rewritten_text": "", "changes_made": [], "status": "no_input"}
+
+    if len(text) > max_length:
+        text = text[:max_length]
+
+    # Build improvement instructions from findings
+    instructions = _build_instructions(findings or [])
+
+    # Try LLM-assisted rewrite
+    try:
+        from tools.llm.router import LLMRouter
+        router = LLMRouter()
+
+        prompt = (
+            f"Rewrite the following text to improve quality. Apply these fixes:\n"
+            f"{instructions}\n\n"
+            f"Style: {style}\n"
+            f"Rules: Maintain original meaning. Keep technical accuracy. "
+            f"Use active voice. Follow Plain Writing Act guidelines.\n\n"
+            f"Original text:\n{text}\n\n"
+            f"Return the improved text only, no explanations."
+        )
+
+        result = router.invoke(
+            function_name="wg_rewrite",
+            prompt=prompt,
+        )
+
+        if result and isinstance(result, str) and len(result) > 20:
+            return {
+                "rewritten_text": result,
+                "changes_made": [i for i in instructions.split("\n") if i.strip()],
+                "status": "llm_rewritten",
+                "method": "two_tier",
+            }
+
+    except (ImportError, Exception):
+        pass
+
+    # Fallback: return original with suggestions
+    return {
+        "rewritten_text": text,
+        "changes_made": [],
+        "suggestions": [i for i in instructions.split("\n") if i.strip()],
+        "status": "suggestions_only",
+        "method": "deterministic_fallback",
+        "note": "LLM unavailable. Suggestions provided instead of rewrite.",
+    }
+
+
+def _build_instructions(findings):
+    """Build rewrite instructions from findings."""
+    if not findings:
+        return "Improve overall clarity and readability."
+
+    lines = []
+    seen = set()
+
+    for f in findings[:15]:  # Limit to top 15 findings
+        category = f.get("category", "")
+        message = f.get("message", "")
+        suggestion = f.get("suggestion", "")
+
+        key = f"{category}:{message[:50]}"
+        if key in seen:
+            continue
+        seen.add(key)
+
+        if suggestion:
+            lines.append(f"- {message} Suggestion: {suggestion}")
+        else:
+            lines.append(f"- {message}")
+
+    return "\n".join(lines) if lines else "Improve overall clarity and readability."