frootai 3.3.0__tar.gz

frootai-3.3.0/PKG-INFO

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: frootai
+Version: 3.3.0
+Summary: FrootAI SDK — The open glue for AI architecture. Offline access to 16 knowledge modules, 20 solution plays, cost estimation, evaluation, and A/B testing.
+Author-email: Pavleen Bali <pavleenbali@frootai.dev>
+License: MIT
+Project-URL: Homepage, https://frootai.dev
+Project-URL: Repository, https://github.com/gitpavleenbali/frootai
+Project-URL: Documentation, https://frootai.dev/api-docs
+Keywords: frootai,ai,architecture,azure,mcp,agents,rag,sdk
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# FrootAI — Python SDK
+
+> Offline-first access to 16 AI architecture knowledge modules, 20 solution plays, cost estimation, evaluation, and A/B testing. Zero external dependencies.
+
+[![PyPI](https://img.shields.io/pypi/v/frootai)](https://pypi.org/project/frootai/)
+[![Python](https://img.shields.io/pypi/pyversions/frootai)](https://pypi.org/project/frootai/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+
+## Install
+
+```bash
+pip install frootai
+```
+
+## Quick Start
+
+```python
+from frootai import FrootAI, SolutionPlay, Evaluator
+
+# Search 682KB knowledge base (16 modules across 5 FROOT layers)
+client = FrootAI()
+results = client.search("RAG architecture")
+for r in results:
+    print(f"[{r['module_id']}] {r['title']} — {r['relevance']} hits")
+
+# Get a specific module
+module = client.get_module("R2")  # RAG Architecture & Retrieval
+print(f"{module['title']}: {module['content_length']:,} chars")
+
+# List all FROOT layers
+for layer in client.list_layers():
+    print(f"{layer['emoji']} {layer['name']} ({len(layer['modules'])} modules)")
+
+# Estimate Azure costs
+cost = client.estimate_cost("01-enterprise-rag", scale="prod")
+print(f"${cost['monthly_total']}/mo")
+
+# Browse 20 solution plays
+plays = SolutionPlay.all()
+ready = SolutionPlay.ready()  # 3 production-ready
+by_layer = SolutionPlay.by_layer("R")  # Reasoning layer plays
+```
+
+## Features
+
+### Knowledge Search (offline, no API calls)
+
+```python
+client = FrootAI()
+
+# Full-text search across 16 modules (643KB of real content)
+results = client.search("embeddings", max_results=5)
+
+# Get module by ID
+mod = client.get_module("O2")  # AI Agents & Microsoft Agent Framework
+
+# List all modules
+for m in client.list_modules():
+    print(f"{m['emoji']} {m['id']} {m['title']} ({m['content_length'] // 1024}KB)")
+
+# Extract a specific section
+section = client.get_module_section("F1", "Table of Contents")
+```
+
+### Glossary (159+ terms extracted from content)
+
+```python
+# Look up a term
+term = client.lookup_term("temperature")
+
+# Search glossary
+terms = client.search_glossary("embedding", max_results=10)
+```
+
+### Cost Estimation
+
+```python
+# Estimate monthly Azure costs for a solution play
+cost = client.estimate_cost("01-enterprise-rag", scale="dev")
+# {'play': '01-enterprise-rag', 'scale': 'dev', 'monthly_total': 430, 'breakdown': {...}}
+
+cost = client.estimate_cost("01-enterprise-rag", scale="prod")
+# {'monthly_total': 3600, 'breakdown': {'openai-gpt4o': 2500, 'ai-search-standard': 750, ...}}
+```
+
+### Solution Plays (20 pre-tuned architecture blueprints)
+
+```python
+from frootai.plays import SolutionPlay
+
+play = SolutionPlay.get("03")
+print(f"{play.name}: {play.description}")
+print(f"Infrastructure: {play.infra}")
+print(f"Tuning params: {play.tuning}")
+print(f"Related modules: {play.modules}")
+
+# Filter by FROOT layer
+orchestration_plays = SolutionPlay.by_layer("O_ORCH")
+```
+
+### Evaluation (quality gates)
+
+```python
+from frootai import Evaluator
+
+evaluator = Evaluator()
+scores = {"groundedness": 4.5, "relevance": 3.2, "coherence": 4.1, "fluency": 4.8}
+
+results = evaluator.check_thresholds(scores)
+print(evaluator.summary(scores))
+# 3/4 checks passed (relevance 3.2 < threshold 4.0)
+```
+
+### A/B Testing (prompt experiments)
+
+```python
+from frootai.ab_testing import PromptExperiment, PromptVariant
+
+# You provide the model function — no fake scores
+def my_model(system_prompt, query):
+    return call_your_llm(system_prompt=system_prompt, user_message=query)
+
+def my_scorer(query, response):
+    return {"groundedness": 4.5, "relevance": 4.0}
+
+experiment = PromptExperiment(
+    name="system-prompt-v2",
+    variants=[
+        PromptVariant("control", "You are a helpful assistant."),
+        PromptVariant("expert", "You are an Azure AI expert. Cite sources."),
+    ],
+)
+
+results = experiment.run(["What is RAG?"], model_fn=my_model, scorer_fn=my_scorer)
+print(f"Winner: {experiment.pick_winner(results)}")
+```
+
+## CLI
+
+```bash
+frootai plays                    # List all 20 solution plays
+frootai plays --ready            # Show production-ready plays only
+frootai plays --layer R          # Filter by FROOT layer
+frootai search "embeddings"      # Search knowledge base
+frootai modules                  # List all 16 modules with sizes
+frootai glossary temperature     # Look up a term
+frootai cost 01-enterprise-rag   # Estimate Azure costs
+frootai cost 01-enterprise-rag --scale prod
+frootai --version                # Show version
+```
+
+## What's Inside
+
+- **16 knowledge modules** (643KB) across 5 FROOT layers: Foundations, Reasoning, Orchestration, Operations, Transformation
+- **20 solution plays** with infrastructure, tuning parameters, and module mapping
+- **159+ glossary terms** extracted from module content
+- **Cost estimation** for 10 plays with dev/prod breakdowns
+- **Evaluation framework** with configurable thresholds
+- **A/B testing framework** with real model callbacks (no fake scores)
+- **Zero external dependencies** — pure Python stdlib
+
+## FROOT Layers
+
+| Layer | Emoji | Name | Modules |
+|-------|-------|------|---------|
+| F | 🌱 | Foundations | F1-F4 (GenAI, LLMs, Glossary, Agentic OS) |
+| R | 🪵 | Reasoning | R1-R3 (Prompts, RAG, Deterministic AI) |
+| O_ORCH | 🌿 | Orchestration | O1-O3 (Semantic Kernel, Agents, MCP) |
+| O_OPS | 🏗️ | Operations | O4-O6 (Platform, Infrastructure, Copilot) |
+| T | 🍎 | Transformation | T1-T3 (Fine-Tuning, Responsible AI, Production) |
+
+## Links
+
+- **Website:** [frootai.dev](https://frootai.dev)
+- **npm MCP Server:** [frootai-mcp](https://www.npmjs.com/package/frootai-mcp)
+- **VS Code Extension:** [pavleenbali.frootai](https://marketplace.visualstudio.com/items?itemName=pavleenbali.frootai)
+- **GitHub:** [github.com/gitpavleenbali/frootai](https://github.com/gitpavleenbali/frootai)
+- **Python MCP Server:** [frootai-mcp (PyPI)](https://pypi.org/project/frootai-mcp/)
+
+## License
+
+MIT — Pavleen Bali

frootai-3.3.0/README.md

@@ -0,0 +1,182 @@
+# FrootAI — Python SDK
+
+> Offline-first access to 16 AI architecture knowledge modules, 20 solution plays, cost estimation, evaluation, and A/B testing. Zero external dependencies.
+
+[![PyPI](https://img.shields.io/pypi/v/frootai)](https://pypi.org/project/frootai/)
+[![Python](https://img.shields.io/pypi/pyversions/frootai)](https://pypi.org/project/frootai/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+
+## Install
+
+```bash
+pip install frootai
+```
+
+## Quick Start
+
+```python
+from frootai import FrootAI, SolutionPlay, Evaluator
+
+# Search 682KB knowledge base (16 modules across 5 FROOT layers)
+client = FrootAI()
+results = client.search("RAG architecture")
+for r in results:
+    print(f"[{r['module_id']}] {r['title']} — {r['relevance']} hits")
+
+# Get a specific module
+module = client.get_module("R2")  # RAG Architecture & Retrieval
+print(f"{module['title']}: {module['content_length']:,} chars")
+
+# List all FROOT layers
+for layer in client.list_layers():
+    print(f"{layer['emoji']} {layer['name']} ({len(layer['modules'])} modules)")
+
+# Estimate Azure costs
+cost = client.estimate_cost("01-enterprise-rag", scale="prod")
+print(f"${cost['monthly_total']}/mo")
+
+# Browse 20 solution plays
+plays = SolutionPlay.all()
+ready = SolutionPlay.ready()  # 3 production-ready
+by_layer = SolutionPlay.by_layer("R")  # Reasoning layer plays
+```
+
+## Features
+
+### Knowledge Search (offline, no API calls)
+
+```python
+client = FrootAI()
+
+# Full-text search across 16 modules (643KB of real content)
+results = client.search("embeddings", max_results=5)
+
+# Get module by ID
+mod = client.get_module("O2")  # AI Agents & Microsoft Agent Framework
+
+# List all modules
+for m in client.list_modules():
+    print(f"{m['emoji']} {m['id']} {m['title']} ({m['content_length'] // 1024}KB)")
+
+# Extract a specific section
+section = client.get_module_section("F1", "Table of Contents")
+```
+
+### Glossary (159+ terms extracted from content)
+
+```python
+# Look up a term
+term = client.lookup_term("temperature")
+
+# Search glossary
+terms = client.search_glossary("embedding", max_results=10)
+```
+
+### Cost Estimation
+
+```python
+# Estimate monthly Azure costs for a solution play
+cost = client.estimate_cost("01-enterprise-rag", scale="dev")
+# {'play': '01-enterprise-rag', 'scale': 'dev', 'monthly_total': 430, 'breakdown': {...}}
+
+cost = client.estimate_cost("01-enterprise-rag", scale="prod")
+# {'monthly_total': 3600, 'breakdown': {'openai-gpt4o': 2500, 'ai-search-standard': 750, ...}}
+```
+
+### Solution Plays (20 pre-tuned architecture blueprints)
+
+```python
+from frootai.plays import SolutionPlay
+
+play = SolutionPlay.get("03")
+print(f"{play.name}: {play.description}")
+print(f"Infrastructure: {play.infra}")
+print(f"Tuning params: {play.tuning}")
+print(f"Related modules: {play.modules}")
+
+# Filter by FROOT layer
+orchestration_plays = SolutionPlay.by_layer("O_ORCH")
+```
+
+### Evaluation (quality gates)
+
+```python
+from frootai import Evaluator
+
+evaluator = Evaluator()
+scores = {"groundedness": 4.5, "relevance": 3.2, "coherence": 4.1, "fluency": 4.8}
+
+results = evaluator.check_thresholds(scores)
+print(evaluator.summary(scores))
+# 3/4 checks passed (relevance 3.2 < threshold 4.0)
+```
+
+### A/B Testing (prompt experiments)
+
+```python
+from frootai.ab_testing import PromptExperiment, PromptVariant
+
+# You provide the model function — no fake scores
+def my_model(system_prompt, query):
+    return call_your_llm(system_prompt=system_prompt, user_message=query)
+
+def my_scorer(query, response):
+    return {"groundedness": 4.5, "relevance": 4.0}
+
+experiment = PromptExperiment(
+    name="system-prompt-v2",
+    variants=[
+        PromptVariant("control", "You are a helpful assistant."),
+        PromptVariant("expert", "You are an Azure AI expert. Cite sources."),
+    ],
+)
+
+results = experiment.run(["What is RAG?"], model_fn=my_model, scorer_fn=my_scorer)
+print(f"Winner: {experiment.pick_winner(results)}")
+```
+
+## CLI
+
+```bash
+frootai plays                    # List all 20 solution plays
+frootai plays --ready            # Show production-ready plays only
+frootai plays --layer R          # Filter by FROOT layer
+frootai search "embeddings"      # Search knowledge base
+frootai modules                  # List all 16 modules with sizes
+frootai glossary temperature     # Look up a term
+frootai cost 01-enterprise-rag   # Estimate Azure costs
+frootai cost 01-enterprise-rag --scale prod
+frootai --version                # Show version
+```
+
+## What's Inside
+
+- **16 knowledge modules** (643KB) across 5 FROOT layers: Foundations, Reasoning, Orchestration, Operations, Transformation
+- **20 solution plays** with infrastructure, tuning parameters, and module mapping
+- **159+ glossary terms** extracted from module content
+- **Cost estimation** for 10 plays with dev/prod breakdowns
+- **Evaluation framework** with configurable thresholds
+- **A/B testing framework** with real model callbacks (no fake scores)
+- **Zero external dependencies** — pure Python stdlib
+
+## FROOT Layers
+
+| Layer | Emoji | Name | Modules |
+|-------|-------|------|---------|
+| F | 🌱 | Foundations | F1-F4 (GenAI, LLMs, Glossary, Agentic OS) |
+| R | 🪵 | Reasoning | R1-R3 (Prompts, RAG, Deterministic AI) |
+| O_ORCH | 🌿 | Orchestration | O1-O3 (Semantic Kernel, Agents, MCP) |
+| O_OPS | 🏗️ | Operations | O4-O6 (Platform, Infrastructure, Copilot) |
+| T | 🍎 | Transformation | T1-T3 (Fine-Tuning, Responsible AI, Production) |
+
+## Links
+
+- **Website:** [frootai.dev](https://frootai.dev)
+- **npm MCP Server:** [frootai-mcp](https://www.npmjs.com/package/frootai-mcp)
+- **VS Code Extension:** [pavleenbali.frootai](https://marketplace.visualstudio.com/items?itemName=pavleenbali.frootai)
+- **GitHub:** [github.com/gitpavleenbali/frootai](https://github.com/gitpavleenbali/frootai)
+- **Python MCP Server:** [frootai-mcp (PyPI)](https://pypi.org/project/frootai-mcp/)
+
+## License
+
+MIT — Pavleen Bali

frootai-3.3.0/frootai/__init__.py

@@ -0,0 +1,27 @@
+"""FrootAI SDK — Programmatic access to the FrootAI ecosystem.
+
+From the Roots to the Fruits. It's simply Frootful.
+
+Usage:
+    from frootai import FrootAI, SolutionPlay, Evaluator
+
+    client = FrootAI()
+    results = client.search("RAG architecture")
+    module = client.get_module("R2")
+    cost = client.estimate_cost("01-enterprise-rag", scale="dev")
+
+    plays = SolutionPlay.all()
+    play = SolutionPlay.get("03")
+
+    evaluator = Evaluator()
+    evaluator.check_thresholds({"groundedness": 4.2, "relevance": 3.8})
+"""
+
+__version__ = "3.3.0"
+__author__ = "Pavleen Bali"
+
+from frootai.client import FrootAI
+from frootai.plays import SolutionPlay
+from frootai.evaluation import Evaluator
+
+__all__ = ["FrootAI", "SolutionPlay", "Evaluator", "__version__"]

frootai-3.3.0/frootai/ab_testing.py

@@ -0,0 +1,162 @@
+"""FrootAI Prompt A/B Testing Framework.
+
+Run prompt experiments across variants, measure quality, pick winners.
+Requires a model_fn callback for actual LLM inference.
+
+Usage:
+    from frootai.ab_testing import PromptExperiment, PromptVariant
+
+    def my_model(system_prompt: str, query: str) -> str:
+        # Call Azure OpenAI, local model, etc.
+        return openai_client.chat(system_prompt=system_prompt, query=query)
+
+    def my_scorer(query: str, response: str) -> dict[str, float]:
+        return {"groundedness": 4.5, "relevance": 4.0}
+
+    experiment = PromptExperiment(
+        name="rag-system-prompt-v2",
+        variants=[
+            PromptVariant("control", "You are a helpful assistant."),
+            PromptVariant("concise", "You are a concise assistant. Answer in 2 sentences max."),
+            PromptVariant("expert", "You are an Azure AI expert. Cite sources."),
+        ],
+        metrics=["groundedness", "relevance", "latency"],
+    )
+
+    results = experiment.run(
+        test_queries=["What is RAG?", "Explain embeddings"],
+        model_fn=my_model,
+        scorer_fn=my_scorer,
+    )
+    winner = experiment.pick_winner(results)
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Callable
+import json
+import time
+
+
+@dataclass
+class PromptVariant:
+    """A single prompt variant in an A/B test."""
+    name: str
+    system_prompt: str
+    weight: float = 1.0
+
+
+@dataclass
+class ExperimentResult:
+    """Result of running one variant against one query."""
+    variant: str
+    query: str
+    response: str
+    latency_ms: float
+    scores: dict[str, float] = field(default_factory=dict)
+
+
+@dataclass
+class PromptExperiment:
+    """A/B testing experiment for prompt variants.
+
+    Attributes:
+        name: Experiment identifier
+        variants: List of prompt variants to test
+        metrics: Quality metrics to measure
+    """
+    name: str
+    variants: list[PromptVariant]
+    metrics: list[str] = field(default_factory=lambda: ["groundedness", "relevance", "coherence"])
+
+    def run(
+        self,
+        test_queries: list[str],
+        model_fn: Callable[[str, str], str],
+        scorer_fn: Optional[Callable[[str, str], dict[str, float]]] = None,
+        rounds: int = 1,
+    ) -> list[ExperimentResult]:
+        """Run the experiment using provided model and scorer functions.
+
+        Args:
+            test_queries: Questions to test each variant against.
+            model_fn: Callable(system_prompt, query) -> response string.
+            scorer_fn: Optional Callable(query, response) -> {metric: score}.
+                        If not provided, only latency is measured.
+            rounds: Number of rounds to repeat (for statistical stability).
+        """
+        results = []
+        for _ in range(rounds):
+            for query in test_queries:
+                for variant in self.variants:
+                    start = time.perf_counter()
+                    response = model_fn(variant.system_prompt, query)
+                    latency = (time.perf_counter() - start) * 1000
+
+                    scores = {}
+                    if scorer_fn is not None:
+                        scores = scorer_fn(query, response)
+                    scores["latency_ms"] = round(latency, 1)
+
+                    result = ExperimentResult(
+                        variant=variant.name,
+                        query=query,
+                        response=response,
+                        latency_ms=round(latency, 1),
+                        scores=scores,
+                    )
+                    results.append(result)
+        return results
+
+    def pick_winner(self, results: list[ExperimentResult]) -> str:
+        """Pick the best variant based on average scores (excluding latency)."""
+        variant_scores: dict[str, list[float]] = {}
+        for r in results:
+            if r.variant not in variant_scores:
+                variant_scores[r.variant] = []
+            quality_scores = {k: v for k, v in r.scores.items() if k != "latency_ms"}
+            if quality_scores:
+                avg = sum(quality_scores.values()) / len(quality_scores)
+                variant_scores[r.variant].append(avg)
+
+        if not variant_scores or all(len(v) == 0 for v in variant_scores.values()):
+            # Fall back to lowest latency if no quality scores
+            latencies: dict[str, list[float]] = {}
+            for r in results:
+                latencies.setdefault(r.variant, []).append(r.latency_ms)
+            return min(latencies, key=lambda v: sum(latencies[v]) / len(latencies[v]))
+
+        averages = {v: sum(s) / len(s) for v, s in variant_scores.items() if s}
+        return max(averages, key=averages.get)
+
+    def summary(self, results: list[ExperimentResult]) -> str:
+        """Generate experiment summary."""
+        lines = [f"Experiment: {self.name}", "=" * 50]
+        variant_data: dict[str, list] = {}
+        for r in results:
+            variant_data.setdefault(r.variant, []).append(r)
+
+        for variant, data in variant_data.items():
+            avg_scores: dict[str, float] = {}
+            all_metrics = set()
+            for r in data:
+                all_metrics.update(r.scores.keys())
+            for m in sorted(all_metrics):
+                vals = [r.scores[m] for r in data if m in r.scores]
+                if vals:
+                    avg_scores[m] = sum(vals) / len(vals)
+            lines.append(f"\n  Variant: {variant}")
+            lines.append(f"  Samples: {len(data)}")
+            for m, s in avg_scores.items():
+                lines.append(f"    {m}: {s:.2f}")
+
+        winner = self.pick_winner(results)
+        lines.append(f"\n  Winner: {winner}")
+        return "\n".join(lines)
+
+    def to_json(self) -> str:
+        """Export experiment config as JSON."""
+        return json.dumps({
+            "name": self.name,
+            "variants": [{"name": v.name, "system_prompt": v.system_prompt, "weight": v.weight} for v in self.variants],
+            "metrics": self.metrics,
+        }, indent=2)