tubesentry 0.1.0__py3-none-any.whl

tubesentry/core/auditor.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+import pandas as pd
+from rich.console import Console
+
+from tubesentry.contracts.schemas import (
+    AuditSummaryReport,
+    ClassifiedChannelRecord,
+    WasteCategoryBreakdown,
+)
+from tubesentry.contracts.taxonomy import CATEGORY_DESCRIPTIONS, WasteCategory
+from tubesentry.core.heuristics import evaluate_heuristics_detailed
+from tubesentry.core.seed_db import lookup_seed_channel
+
+
+def normalize_topic_category(topic_str: str) -> str:
+    """Extracts clean topic name from Wikipedia URL or Freebase string."""
+    if "wiki/" in topic_str:
+        topic_name = topic_str.split("wiki/", 1)[1]
+    elif "/" in topic_str:
+        topic_name = topic_str.rsplit("/", 1)[-1]
+    else:
+        topic_name = topic_str
+    return topic_name.replace("_", " ").strip()
+
+
+def run_free_audit(
+    df: pd.DataFrame,
+    target_languages: list[str] | None = None,
+    target_cpa: float | None = None,
+    print_banner: bool = True,
+    currency: str = "EUR",
+) -> AuditSummaryReport:
+    """Executes the free diagnostic placement audit using the bundled seed DB and heuristics.
+
+    Reveals the exact total wasted spend across all 6 risk categories, impression-weighted metrics,
+    and the top 25 worst offending channels unmasked, while masking the tail behind Pro upgrade notices.
+    """
+    total_placements = len(df)
+    total_spend = float(df["cost"].sum()) if not df.empty and "cost" in df.columns else 0.0
+    total_impressions = int(df["impressions"].sum()) if not df.empty and "impressions" in df.columns else 0
+
+    classified_records: list[ClassifiedChannelRecord] = []
+
+    # Aggregate by unique channel_id or raw_placement to avoid duplicate rows for the same channel
+    grouped = (
+        df.groupby(
+            ["raw_placement", "channel_id", "channel_title"],
+            dropna=False,
+            as_index=False,
+        )
+        .agg(
+            {
+                "cost": "sum",
+                "impressions": "sum",
+                "views": "sum",
+                "clicks": "sum",
+                "conversions": "sum",
+            }
+        )
+        .sort_values(by="cost", ascending=False)
+    )
+
+    category_stats: dict[str, dict[str, Any]] = defaultdict(
+        lambda: {
+            "wasted_spend": 0.0,
+            "channel_count": 0,
+            "impressions": 0,
+            "views": 0,
+            "clicks": 0,
+            "conversions": 0.0,
+        }
+    )
+
+    # Accumulator for impression-weighted quality scoring (1 to 10 scale)
+    total_quality_points = 0.0
+
+    for _, row in grouped.iterrows():
+        raw_pl = str(row["raw_placement"]) if pd.notna(row["raw_placement"]) else ""
+        ch_id = str(row["channel_id"]) if pd.notna(row["channel_id"]) else None
+        title = str(row["channel_title"]) if pd.notna(row["channel_title"]) else raw_pl
+        cost = float(row["cost"])
+        impr = int(row["impressions"])
+        views = int(row["views"])
+        clicks = int(row["clicks"])
+        conv = float(row["conversions"])
+
+        # 1. Check Seed DB
+        category: str = "CLEAN"
+        confidence: float = 0.0
+        reason: str = "Clean"
+        is_young_kids = False
+        granular_sub = "CLEAN_AUDIENCE"
+
+        if ch_id:
+            seed_match = lookup_seed_channel(ch_id)
+            if seed_match:
+                category = seed_match["category"]
+                confidence = seed_match["confidence"]
+                reason = seed_match["reason"]
+                if category == "MADE_FOR_KIDS":
+                    is_young_kids = True
+                    granular_sub = "YOUNG_KIDS_TODDLER"
+
+        # 2. Check Open-Core Heuristics if not matched in seed
+        if category == "CLEAN":
+            h_res = evaluate_heuristics_detailed(
+                channel_title=title,
+                raw_placement=raw_pl,
+                impressions=impr,
+                views=views,
+                clicks=clicks,
+                cost=cost,
+                conversions=conv,
+                target_languages=target_languages,
+                target_cpa=target_cpa,
+            )
+            if h_res["category"] != "CLEAN":
+                category = h_res["category"]
+                confidence = h_res["confidence"]
+                reason = h_res["reason"]
+                is_young_kids = h_res.get("is_young_kids", False)
+                granular_sub = h_res.get("granular_sub_category", category)
+
+        # Compute placement quality points (1.0 to 10.0 scale)
+        if category == "CLEAN":
+            row_quality = 10.0
+        elif is_young_kids:
+            row_quality = 1.0
+        elif category in ("MADE_FOR_KIDS", "AI_SLOP_CLICK_FARM"):
+            row_quality = 1.5
+        elif category in ("PASSIVE_AUDIO_MUSIC", "FOREIGN_SCRIPT_LEAK"):
+            row_quality = 2.0
+        elif category == "LOW_AGE_GAMING":
+            row_quality = 2.5
+        else:  # PERFORMANCE_SINKHOLE
+            row_quality = 3.5
+
+        total_quality_points += row_quality * max(impr, 1)
+
+        if category != "CLEAN":
+            rec = ClassifiedChannelRecord(
+                channel_id=ch_id or "UC_UNKNOWN_HANDLE",
+                channel_title=title,
+                waste_category=category,  # type: ignore[arg-type]
+                waste_confidence=round(confidence, 2),
+                made_for_kids_official=(category == "MADE_FOR_KIDS"),
+                is_young_kids=is_young_kids,
+                granular_sub_category=granular_sub,
+                thumbnail_url=None,
+                detected_language=None,
+                detected_country=None,
+                topic_categories=[],
+                cost=round(cost, 2),
+                impressions=impr,
+                views=views,
+                clicks=clicks,
+                conversions=round(conv, 2),
+                reason_summary=reason,
+                is_masked=False,
+            )
+            classified_records.append(rec)
+
+            stats = category_stats[category]
+            stats["wasted_spend"] += cost
+            stats["channel_count"] += 1
+            stats["impressions"] += impr
+            stats["views"] += views
+            stats["clicks"] += clicks
+            stats["conversions"] += conv
+
+    # Sort offending channels descending by wasted cost
+    classified_records.sort(key=lambda x: x.cost, reverse=True)
+
+    total_wasted_spend = sum(stats["wasted_spend"] for stats in category_stats.values())
+    waste_percentage = (total_wasted_spend / total_spend * 100.0) if total_spend > 0 else 0.0
+    total_waste_channels = len(classified_records)
+
+    total_wasted_impressions = sum(stats["impressions"] for stats in category_stats.values())
+    wasted_impressions_percentage = (
+        (total_wasted_impressions / total_impressions * 100.0) if total_impressions > 0 else 0.0
+    )
+
+    weight_denom = max(total_impressions, len(grouped))
+    weighted_quality_score = round(total_quality_points / weight_denom, 1) if weight_denom > 0 else 10.0
+    weighted_quality_score = max(1.0, min(10.0, weighted_quality_score))
+
+    # Free Tier Masking Invariant:
+    # First 25 channels are completely unmasked.
+    # Channels 26+ are masked with UC•••••••••••• and locked placeholder title.
+    top_offenders: list[ClassifiedChannelRecord] = []
+    masked_count = 0
+
+    for idx, rec in enumerate(classified_records):
+        if idx < 25:
+            top_offenders.append(rec)
+        else:
+            masked_count += 1
+            top_offenders.append(
+                rec.model_copy(
+                    update={
+                        "channel_id": "UC••••••••••••",
+                        "channel_title": "[LOCKED IN FREE TIER — Upgrade to TubeSentry Pro]",
+                        "is_masked": True,
+                    }
+                )
+            )
+
+    breakdowns: dict[str, WasteCategoryBreakdown] = {}
+    for cat_enum in WasteCategory:
+        cat_key = cat_enum.value
+        if cat_key == "CLEAN":
+            continue
+        stats = category_stats[cat_key]
+        spend_pct = (stats["wasted_spend"] / total_spend * 100.0) if total_spend > 0 else 0.0
+        breakdowns[cat_key] = WasteCategoryBreakdown(
+            category=cat_key,  # type: ignore[arg-type]
+            wasted_spend=round(stats["wasted_spend"], 2),
+            wasted_spend_percentage=round(spend_pct, 2),
+            channel_count=int(stats["channel_count"]),
+            impressions=int(stats["impressions"]),
+            views=int(stats["views"]),
+            clicks=int(stats["clicks"]),
+            conversions=round(stats["conversions"], 2),
+            description=CATEGORY_DESCRIPTIONS.get(cat_enum, ""),
+        )
+
+    coppa_kids_spend = category_stats["MADE_FOR_KIDS"]["wasted_spend"]
+    annual_runrate = total_wasted_spend * 12.0
+
+    banner_text = _build_diagnostic_banner(
+        total_placements=total_placements,
+        total_spend=total_spend,
+        total_wasted_spend=total_wasted_spend,
+        waste_percentage=waste_percentage,
+        total_waste_channels=total_waste_channels,
+        total_impressions=total_impressions,
+        total_wasted_impressions=total_wasted_impressions,
+        wasted_impressions_percentage=wasted_impressions_percentage,
+        weighted_quality_score=weighted_quality_score,
+        breakdowns=breakdowns,
+        masked_count=masked_count,
+        currency=currency,
+    )
+
+    if print_banner:
+        console = Console(stderr=False)
+        console.print(banner_text)
+
+    return AuditSummaryReport(
+        total_placements_audited=total_placements,
+        total_spend=round(total_spend, 2),
+        total_wasted_spend=round(total_wasted_spend, 2),
+        wasted_spend_percentage=round(waste_percentage, 2),
+        total_waste_channels=total_waste_channels,
+        total_impressions_audited=total_impressions,
+        total_wasted_impressions=total_wasted_impressions,
+        wasted_impressions_percentage=round(wasted_impressions_percentage, 2),
+        weighted_quality_score=round(weighted_quality_score, 1),
+        top_topic_categories=[],
+        currency_code=currency,
+        category_breakdowns=breakdowns,
+        top_offenders=top_offenders,
+        masked_offenders_count=masked_count,
+        is_pro_audit=False,
+        coppa_kids_risk_spend=round(coppa_kids_spend, 2),
+        annualized_recoverable_runrate=round(annual_runrate, 2),
+        upgrade_prompt=banner_text,
+    )
+
+
+def _build_diagnostic_banner(
+    total_placements: int,
+    total_spend: float,
+    total_wasted_spend: float,
+    waste_percentage: float,
+    total_waste_channels: int,
+    total_impressions: int,
+    total_wasted_impressions: int,
+    wasted_impressions_percentage: float,
+    weighted_quality_score: float,
+    breakdowns: dict[str, WasteCategoryBreakdown],
+    masked_count: int,
+    currency: str = "EUR",
+) -> str:
+    """Constructs the Rich upgrade diagnostic banner string matching spec Section 4.1."""
+    sym = "€" if currency == "EUR" else ("$" if currency == "USD" else currency)
+
+    lines = [
+        "=" * 84,
+        " TUBESENTRY OPEN-CORE v0.1.0 — YOUTUBE PLACEMENT WASTE DIAGNOSTIC",
+        "=" * 84,
+        f" Audited Placements:        {total_placements:,} placements | Total Spend: {sym} {total_spend:,.2f}",
+        f" Identified Wasted Spend:   {sym} {total_wasted_spend:,.2f} ({waste_percentage:.1f}% of budget) across {total_waste_channels:,} channels",
+        f" Impression Waste Impact:   {total_wasted_impressions:,} imps ({wasted_impressions_percentage:.1f}% wasted) | Placement Quality: {weighted_quality_score:.1f}/10",
+        "-" * 84,
+        " [!] WASTED BUDGET BY CATEGORY:",
+    ]
+
+    # Sort categories by wasted spend descending
+    sorted_cats = sorted(breakdowns.values(), key=lambda b: b.wasted_spend, reverse=True)
+    idx = 1
+    category_labels = {
+        "MADE_FOR_KIDS": "COPPA / Toddler",
+        "PASSIVE_AUDIO_MUSIC": "Lofi / Sleep / BG",
+        "FOREIGN_SCRIPT_LEAK": "Cyrillic / Arabic / Asian",
+        "LOW_AGE_GAMING": "Roblox / Minecraft",
+        "AI_SLOP_CLICK_FARM": "Faceless AI / Slop",
+        "PERFORMANCE_SINKHOLE": "High CPA / 0 Conv",
+    }
+
+    for b in sorted_cats:
+        if b.wasted_spend > 0:
+            label = category_labels.get(b.category, "")
+            lines.append(
+                f" {idx}. [{b.category:<20}]  {sym} {b.wasted_spend:>10,.2f} ({b.wasted_spend_percentage:>4.1f}%) — {b.channel_count:>5,} channels ({label})"
+            )
+            idx += 1
+
+    lines.extend(
+        [
+            "-" * 84,
+            " [!] FREE TIER EXPORT LIMIT:",
+            f" Showing Top 25 worst offending channels below. {masked_count:,} additional waste channels are",
+            " masked (`UC••••••••••••`).",
+            "",
+            f" [*] Unlock All {total_waste_channels:,} Flagged Channels, 65k Google Ads Exclusion CSV & HTML Deck:",
+            "    >>> tubesentry auth --key TS_PRO_...",
+            "    Get an instant Single Audit Pass (€199) or Agency License: https://buy.polar.sh/polar_cl_lRuifr3uJS0LR0k6frhg3JEKBRZNQq75hv7cS2am9cX",
+            "=" * 84,
+        ]
+    )
+    return "\n".join(lines)

tubesentry/core/heuristics.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Any
+
+# Compiled regexes for high-performance vectorized evaluation
+YOUNG_KIDS_REGEX = re.compile(
+    r"(?i)\b(cocomelon|bebefinn|chuchu\s*tv|peppa\s*pig|super\s*simple\s*songs|"
+    r"little\s*baby\s*bum|pinkfong|baby\s*shark|vlad\s*and\s*niki|diana\s*and\s*roma|"
+    r"nursery\s*rhymes?|kinderlieder|canciones\s*infantiles|comptines|canzoni\s*per\s*bambini|"
+    r"el\s*reino\s*infantil|gallina\s*pintadita|toy\s*unboxing|learn\s*colors|alphabet\s*song|"
+    r"toddler|kindergarten|sing-along|lullaby|schlaflied|baby\s*songs?)\b"
+)
+
+KIDS_REGEX = re.compile(
+    r"(?i)\b(nursery\s*rhymes?|kinderlieder|canciones\s*infantiles|comptines|"
+    r"canzoni\s*per\s*bambini|cocomelon|bebefinn|chuchu\s*tv|peppa\s*pig|"
+    r"paw\s*patrol|blippi|super\s*simple\s*songs|little\s*baby\s*bum|pinkfong|"
+    r"baby\s*shark|vlad\s*and\s*niki|diana\s*and\s*roma|el\s*reino\s*infantil|"
+    r"gallina\s*pintadita|toy\s*unboxing|learn\s*colors|alphabet\s*song|"
+    r"toddler|kindergarten|sing-along|lullaby|schlaflied|baby\s*songs?)\b"
+)
+
+PASSIVE_MUSIC_REGEX = re.compile(
+    r"(?i)\b(lofi|lo-fi|relaxing\s*music|sleep\s*music|rain\s*sounds|white\s*noise|"
+    r"study\s*beats|ambient\s*music|meditation\s*music|healing\s*frequency|432\s*hz|"
+    r"528\s*hz|10\s*hours|8\s*hours|karaoke|instrumental\s*mix|deep\s*focus|"
+    r"coffee\s*shop\s*jazz|schlafmusik|música\s*para\s*dormir)\b"
+)
+
+LOW_AGE_GAMING_REGEX = re.compile(
+    r"(?i)\b(roblox|brookhaven|blox\s*fruits|minecraft\s*pe|brawl\s*stars|gacha\s*life|"
+    r"gacha\s*club|skibidi\s*toilet|fnf|friday\s*night\s*funkin|among\s*us\s*animation|"
+    r"subway\s*surfers|stumble\s*guys|pk\s*xd|toca\s*life|toca\s*boca|geometry\s*dash)\b"
+)
+
+AI_SLOP_REGEX = re.compile(
+    r"(?i)\b(reddit\s*stories|aita\s*stories|ai\s*facts|satisfying\s*slime|"
+    r"hydraulic\s*press\s*shorts|family\s*guy\s*clips|subway\s*surfers\s*reddit)\b"
+)
+
+# Unicode non-Latin script ranges:
+# Cyrillic: \u0400-\u04FF
+# Arabic: \u0600-\u06FF
+# Devanagari (Hindi): \u0900-\u097F
+# Bengali: \u0980-\u09FF
+# Thai: \u0E00-\u0E7F
+# Hangul: \uAC00-\uD7AF, \u1100-\u11FF
+# CJK: \u4E00-\u9FFF, \u3400-\u4DBF
+NON_LATIN_SCRIPT_PATTERN = re.compile(
+    r"[\u0400-\u04FF\u0600-\u06FF\u0900-\u097F\u0980-\u09FF\u0E00-\u0E7F"
+    r"\uAC00-\uD7AF\u1100-\u11FF\u4E00-\u9FFF\u3400-\u4DBF]"
+)
+
+
+def compute_non_latin_ratio(text: str) -> float:
+    """Computes the ratio of non-Latin script characters to total alphabetic letters.
+
+    Returns float in [0.0, 1.0].
+    """
+    if not text:
+        return 0.0
+
+    total_letters = 0
+    non_latin_letters = 0
+
+    for char in text:
+        if unicodedata.category(char).startswith("L"):
+            total_letters += 1
+            if NON_LATIN_SCRIPT_PATTERN.match(char):
+                non_latin_letters += 1
+
+    if total_letters == 0:
+        return 0.0
+    return non_latin_letters / total_letters
+
+
+def is_young_kids_text(text: str) -> bool:
+    """Checks if text specifically targets toddlers / under 10 (nursery rhymes, CoCoMelon, etc.)."""
+    return bool(YOUNG_KIDS_REGEX.search(text)) if text else False
+
+
+def is_made_for_kids_text(text: str) -> bool:
+    """Checks if text contains toddler or Made-for-Kids keywords."""
+    return bool(KIDS_REGEX.search(text)) if text else False
+
+
+def is_passive_music_text(text: str) -> bool:
+    """Checks if text contains passive audio / background music keywords."""
+    if not text:
+        return False
+    if text.endswith((" - Topic", "VEVO", " VEVO")):
+        return True
+    return bool(PASSIVE_MUSIC_REGEX.search(text))
+
+
+def is_low_age_gaming_text(text: str) -> bool:
+    """Checks if text contains underage gaming keywords."""
+    return bool(LOW_AGE_GAMING_REGEX.search(text)) if text else False
+
+
+def is_ai_slop_text(text: str) -> bool:
+    """Checks if text contains faceless AI slop keywords."""
+    return bool(AI_SLOP_REGEX.search(text)) if text else False
+
+
+def evaluate_heuristics_detailed(
+    channel_title: str,
+    raw_placement: str = "",
+    impressions: int = 0,
+    views: int = 0,
+    clicks: int = 0,
+    cost: float = 0.0,
+    conversions: float = 0.0,
+    target_languages: list[str] | None = None,
+    target_cpa: float | None = None,
+) -> dict[str, Any]:
+    """Evaluates placement heuristics with granular sub-category and toddler detection.
+
+    Returns dict with keys:
+        category, confidence, reason, is_young_kids, granular_sub_category
+    """
+    [lang.lower()[:2] for lang in (target_languages or ["de", "en"])]
+    combined_text = f"{channel_title} {raw_placement}"
+
+    # Rule 1: MADE_FOR_KIDS & YOUNG_KIDS
+    if is_made_for_kids_text(combined_text):
+        young = is_young_kids_text(combined_text)
+        sub_cat = "YOUNG_KIDS_TODDLER" if young else "GENERAL_KIDS_CARTOON"
+        reason = (
+            "Matched toddler / under-10 nursery rhyme & brainrot lexicon."
+            if young
+            else "Matched Made-for-Kids toddler/nursery rhyme lexicon."
+        )
+        return {
+            "category": "MADE_FOR_KIDS",
+            "confidence": 0.95 if young else 0.92,
+            "reason": reason,
+            "is_young_kids": young,
+            "granular_sub_category": sub_cat,
+        }
+
+    # Rule 2: PASSIVE_AUDIO_MUSIC
+    if is_passive_music_text(channel_title):
+        return {
+            "category": "PASSIVE_AUDIO_MUSIC",
+            "confidence": 0.90,
+            "reason": "Matched passive audio/background music streams.",
+            "is_young_kids": False,
+            "granular_sub_category": "PASSIVE_AUDIO_STREAM",
+        }
+
+    # Rule 3: LOW_AGE_GAMING
+    if is_low_age_gaming_text(combined_text):
+        return {
+            "category": "LOW_AGE_GAMING",
+            "confidence": 0.85,
+            "reason": "Matched underage gaming franchise lexicon (Roblox/Minecraft/etc.).",
+            "is_young_kids": False,
+            "granular_sub_category": "LOW_AGE_GAMING_SINK",
+        }
+
+    # Rule 4: FOREIGN_SCRIPT_LEAK
+    ratio = compute_non_latin_ratio(channel_title)
+    if ratio >= 0.25:
+        return {
+            "category": "FOREIGN_SCRIPT_LEAK",
+            "confidence": 0.95,
+            "reason": f"Non-Latin script ratio {ratio:.1%} in title (Cyrillic/Arabic/Asian scripts).",
+            "is_young_kids": False,
+            "granular_sub_category": "GEO_LANGUAGE_LEAK",
+        }
+
+    # Rule 5: AI_SLOP_CLICK_FARM
+    if is_ai_slop_text(combined_text) and conversions == 0:
+        return {
+            "category": "AI_SLOP_CLICK_FARM",
+            "confidence": 0.82,
+            "reason": "Matched faceless AI content lexicon with 0 conversions.",
+            "is_young_kids": False,
+            "granular_sub_category": "AI_SLOP_FARM",
+        }
+
+    # Rule 6: PERFORMANCE_SINKHOLE
+    vtr = (views / max(impressions, 1)) if impressions > 0 else 0.0
+    if (
+        target_cpa is not None
+        and target_cpa > 0
+        and cost >= 3.0 * target_cpa
+        and conversions == 0
+        and vtr < 0.15
+    ):
+        return {
+            "category": "PERFORMANCE_SINKHOLE",
+            "confidence": 0.80,
+            "reason": f"Placement spend (€{cost:.2f}) >= 3x target CPA (€{target_cpa:.2f}) with 0 conv and VTR {vtr:.1%}.",
+            "is_young_kids": False,
+            "granular_sub_category": "HIGH_CPA_SINKHOLE",
+        }
+
+    return {
+        "category": "CLEAN",
+        "confidence": 0.0,
+        "reason": "No waste patterns detected.",
+        "is_young_kids": False,
+        "granular_sub_category": "CLEAN_AUDIENCE",
+    }
+
+
+def evaluate_heuristics(
+    channel_title: str,
+    raw_placement: str = "",
+    impressions: int = 0,
+    views: int = 0,
+    clicks: int = 0,
+    cost: float = 0.0,
+    conversions: float = 0.0,
+    target_languages: list[str] | None = None,
+    target_cpa: float | None = None,
+) -> tuple[str, float, str]:
+    """Applies open-core deterministic heuristic rules to evaluate placement waste.
+
+    Returns:
+        tuple[waste_category, confidence, reason_summary]
+    """
+    res = evaluate_heuristics_detailed(
+        channel_title=channel_title,
+        raw_placement=raw_placement,
+        impressions=impressions,
+        views=views,
+        clicks=clicks,
+        cost=cost,
+        conversions=conversions,
+        target_languages=target_languages,
+        target_cpa=target_cpa,
+    )
+    return (res["category"], res["confidence"], res["reason"])