schematico 0.1.0__py3-none-any.whl

schematico/cli/runner.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from schematico.cli.progress import ProgressReporter
+from schematico.cli.projects import ProjectConfig
+from schematico.discovery import run_discovery
+from schematico.generator import run_generation
+from schematico.logging import get_logger
+from schematico.models import model_from_dict, model_from_json
+from schematico.providers import DEFAULT_MODEL, SchematicoModel, get_llm_model
+
+logger = get_logger("cli.runner")
+
+
+def run(
+    config: ProjectConfig,
+    *,
+    output_override: str | None = None,
+    count_override: int | None = None,
+    model_override: str | None = None,
+) -> None:
+    if not config.record_schema and not config.schema_path:
+        print(
+            f"schematico: error: config '{config.name}.{config.mode}.toml' has no "
+            "schema. Either fill the [schema] table or set schema_path "
+            f"(e.g. `schematico {config.mode} schema import ./schema.json`).",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    if config.mode == "discover" and not os.environ.get("TAVILY_API_KEY"):
+        print(
+            "schematico: error: `discover` mode searches the live web and needs a "
+            "Tavily API key. Set TAVILY_API_KEY (get one free at "
+            "https://tavily.com), or use `generate` mode to synthesize records "
+            "without web search.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    model_str = model_override or config.model or DEFAULT_MODEL
+
+    api_key: str | None = None
+    if config.env_key:
+        api_key = os.environ.get(config.env_key)
+        if not api_key:
+            print(
+                f"schematico: error: env var '{config.env_key}' is not set "
+                f"(required by config '{config.name}').",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    llm_model = get_llm_model(
+        SchematicoModel(
+            model=model_str,
+            api_key=api_key,
+            base_url=config.base_url or None,
+        )
+    )
+
+    output_path = output_override or config.output_path
+
+    try:
+        if config.schema_path:
+            record_model, rows, instructions = model_from_json(config.schema_path)
+        else:
+            record_model, rows, instructions = model_from_dict(config.record_schema)
+    except (FileNotFoundError, ValueError) as e:
+        print(f"schematico: error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    samples = count_override if count_override is not None else rows
+    table = record_model.__name__.removesuffix("Record")
+
+    logger.info(
+        "Running %s for '%s': %d fields, %d records, model=%s",
+        config.mode,
+        table,
+        len(record_model.model_fields),
+        samples,
+        model_str,
+    )
+
+    from pydantic_ai.exceptions import UserError
+
+    run_fn = run_discovery if config.mode == "discover" else run_generation
+
+    reporter = ProgressReporter(table)
+    try:
+        records = run_fn(
+            record_model,
+            samples,
+            instructions,
+            model=llm_model,
+            logfire_token=config.logfire_token or None,
+            progress_cb=reporter.update,
+        )
+    except UserError as e:
+        print(f"schematico: error: {e}", file=sys.stderr)
+        sys.exit(1)
+    reporter.done(len(records))
+
+    out_path = Path(output_path)
+    if out_path.suffix:
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out = out_path
+    else:
+        out_path.mkdir(parents=True, exist_ok=True)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        out = out_path / f"{config.name}_{timestamp}.json"
+    try:
+        with out.open("w", encoding="utf-8") as f:
+            json.dump(records, f, indent=2, ensure_ascii=False)
+    except OSError as e:
+        print(f"schematico: error writing output: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    logger.info("Wrote %d records to %s", len(records), out)
+    print(f"Generated {len(records)} records from schema '{table}' -> {out}")

schematico/cli/wizard.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from schematico.cli.projects import (
+    DEFAULT_COUNT,
+    DEFAULT_ENV_KEY,
+    Mode,
+    ProjectConfig,
+    ensure_config_dir,
+    next_available_name,
+    save_project,
+    set_default,
+)
+from schematico.models import model_from_dict
+
+_TEMPLATE_SCHEMA: dict[str, Any] = {
+    "table": "REPLACE_WITH_TABLE_NAME",
+    "rows": 25,
+    "instructions": "",
+    "fields": [
+        {"name": "id", "type": "string", "description": "UUID v4"},
+        {"name": "full_name", "type": "string", "description": "realistic full name"},
+        {"name": "email", "type": "string", "description": "unique work email"},
+        {
+            "name": "role",
+            "type": "enum",
+            "values": ["admin", "editor", "viewer"],
+        },
+        {
+            "name": "country",
+            "type": "string",
+            "description": "ISO 3166-1 alpha-2 country code",
+        },
+        {
+            "name": "created_at",
+            "type": "string",
+            "description": "ISO 8601 UTC timestamp",
+        },
+    ],
+}
+
+
+def _setup_schema(name: str, mode: Mode) -> tuple[dict[str, Any], str]:
+    """Return (record_schema, schema_path) — exactly one is populated."""
+    if typer.confirm("Do you have an existing JSON schema file to use?", default=False):
+        path_str = typer.prompt("Path to JSON schema file")
+        p = Path(path_str).expanduser()
+        if not p.exists():
+            typer.echo(f"schematico: error: file not found: {p}", err=True)
+            raise typer.Exit(1)
+        try:
+            raw = json.loads(p.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as e:
+            typer.echo(f"schematico: error: invalid JSON in '{p}': {e}", err=True)
+            raise typer.Exit(1)
+        try:
+            model_from_dict(raw)
+        except ValueError as e:
+            typer.echo(f"schematico: error: {e}", err=True)
+            raise typer.Exit(1)
+
+        embed = typer.confirm(
+            "Embed the schema into the config (vs. reference it by path)?",
+            default=True,
+        )
+        if embed:
+            typer.echo(
+                f"Imported schema from '{p}' ({len(raw.get('fields', []))} fields)."
+            )
+            return raw, ""
+        typer.echo(f"Referencing schema at '{p}'.")
+        return {}, str(p)
+
+    tpl = ensure_config_dir() / f"{name}.{mode}.schema.json"
+    if tpl.exists() and not typer.confirm(
+        f"{tpl} already exists. Overwrite with a fresh template?", default=False
+    ):
+        typer.echo(f"Keeping existing template at '{tpl}'. Edit it before running.")
+        return {}, str(tpl)
+    tpl.write_text(json.dumps(_TEMPLATE_SCHEMA, indent=2) + "\n", encoding="utf-8")
+    typer.echo(f"Created template schema at '{tpl}'.")
+    typer.echo(
+        "Edit it (replace REPLACE_WITH_TABLE_NAME, adjust fields, "
+        "optionally add 'instructions') before running."
+    )
+    return {}, str(tpl)
+
+
+def run_wizard() -> Path:
+    name_input = typer.prompt("Project name", default="project_1")
+    mode_input = (
+        typer.prompt("Mode ([d]iscover/[g]enerate)", default="discover").strip().lower()
+    )
+    if mode_input in ("d", "discover"):
+        mode: Mode = "discover"
+    elif mode_input in ("g", "generate"):
+        mode = "generate"
+    else:
+        typer.echo(
+            f"Invalid mode '{mode_input}'. Use 'd'/'discover' or 'g'/'generate'.",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    name = next_available_name(name_input, mode)
+    if name != name_input:
+        typer.echo(f"Name '{name_input}' is taken; using '{name}' instead.")
+
+    output_path = typer.prompt(
+        "Default output directory (filename will be <project>_<timestamp>.json)",
+        default="./.schematico/output",
+    )
+    count = typer.prompt("Count (records to generate)", default=DEFAULT_COUNT, type=int)
+    model = typer.prompt(
+        "Model (e.g. gateway/anthropic:claude-sonnet-4-6; "
+        "leave blank to inherit PAI_MODEL)",
+        default=os.environ.get("PAI_MODEL", ""),
+        show_default=True,
+    )
+    env_key = typer.prompt(
+        "Name of env var that holds your API key (not the key itself; "
+        "leave blank for local/keyless models like ollama)",
+        default=DEFAULT_ENV_KEY,
+    )
+    base_url = typer.prompt(
+        "Custom base URL (leave blank to use the provider's default; "
+        "e.g. http://localhost:11434/v1 for ollama)",
+        default="",
+        show_default=False,
+    )
+    logfire_token = ""
+    if typer.confirm(
+        "Enable Logfire observability? (You'll need a Logfire write token. "
+        "Decline to log to stdout only.)",
+        default=False,
+    ):
+        logfire_token = typer.prompt(
+            "Paste your Logfire write token (e.g. pylf_v1_...)",
+        )
+
+    record_schema, schema_path = _setup_schema(name, mode)
+
+    cfg = ProjectConfig(
+        name=name,
+        mode=mode,
+        model=model,
+        env_key=env_key,
+        base_url=base_url,
+        output_path=output_path,
+        count=count,
+        logfire_token=logfire_token,
+        schema_path=schema_path,
+        record_schema=record_schema,
+    )
+    path = save_project(cfg)
+    set_default(mode, name)
+
+    typer.echo("")
+    typer.echo(f"Created {path}")
+    typer.echo(f"Set as default for `schematico {mode}`.")
+    if record_schema:
+        typer.echo(f"Run `schematico {mode}` from this directory when ready.")
+    else:
+        typer.echo(
+            f"Edit '{schema_path}', then run `schematico {mode}` from this directory."
+        )
+    return path
+
+
+if __name__ == "__main__":
+    typer.run(run_wizard)

schematico/discovery.py

@@ -0,0 +1,102 @@
+import logfire
+from pydantic import BaseModel
+from pydantic_ai.models import Model
+from pydantic_ai.agent import Agent
+from typing import Callable
+
+from schematico.helpers import _table_name, _hash_record, _describe_fields
+from schematico.logging import get_logger
+from schematico.models import build_batch_model
+from schematico.providers import DEFAULT_MODEL
+from schematico.tools.tavily_tools import (
+    search_web,
+    extract_web_content,
+    crawl_paths,
+    map_website,
+)
+
+logger = get_logger("core.discovery")
+
+
+def _build_prompt(schema: type[BaseModel], samples: int, instructions: str) -> str:
+    table = _table_name(schema)
+    field_lines = _describe_fields(schema)
+    prompt = (
+        f"You are a data discovery agent for the '{table}' table.\n"
+        f"Find exactly {samples} realistic, unique records with "
+        "these fields:\n" + "\n".join(field_lines) + "\n\nRules:\n"
+        "- Every record must be unique across all fields.\n"
+        "- Enum fields must use only the declared values.\n"
+        "- Numeric fields must respect any declared min/max range.\n"
+        "- Return exactly the requested number of records.\n"
+        "- Use the tavily tools to find the records."
+    )
+    if instructions:
+        prompt += f"\n\nAdditional instructions:\n{instructions}"
+    return prompt
+
+
+def build_agent(
+    schema: type[BaseModel],
+    samples: int,
+    instructions: str = "",
+    model: str | Model | None = None,
+) -> Agent:
+    resolved: str | Model = model if model is not None else DEFAULT_MODEL
+    table = _table_name(schema)
+    logger.debug("Building agent for '%s' with model %r", table, resolved)
+    batch_model = build_batch_model(schema)
+
+    agent = Agent(
+        model=resolved,
+        output_type=batch_model,
+        system_prompt=_build_prompt(schema, samples, instructions),
+        tools=[search_web, extract_web_content, crawl_paths, map_website],
+    )
+    return agent
+
+
+def run_discovery(
+    schema: type[BaseModel],
+    samples: int,
+    instructions: str = "",
+    model: str | Model | None = None,
+    logfire_token: str | None = None,
+    progress_cb: Callable[[int, int, str], None] | None = None,
+) -> list[dict]:
+    if logfire_token:
+        logfire.configure(token=logfire_token, send_to_logfire=True, scrubbing=False)
+    else:
+        logfire.configure(send_to_logfire=False, scrubbing=False)
+    logfire.instrument_pydantic_ai()
+
+    table = _table_name(schema)
+    logger.info(
+        "Starting discovery run for '%s' (%d records requested)", table, samples
+    )
+    agent = build_agent(schema, samples, instructions, model=model)
+    result = agent.run_sync(
+        f"Find exactly {samples} unique records for the '{table}' table."
+    )
+    logger.debug("Agent returned %d raw records", len(result.output.records))
+
+    seen: dict[str, dict] = {}
+    duplicates = 0
+    for record in result.output.records:
+        record_dict = record.model_dump()
+        h = _hash_record(record_dict)
+        if h in seen:
+            duplicates += 1
+            if progress_cb:
+                progress_cb(len(seen), samples, "duplicate")
+            continue
+        seen[h] = record_dict
+        if progress_cb:
+            progress_cb(len(seen), samples, "found")
+
+    logger.info(
+        "Generation run complete: %d unique records (%d duplicates discarded)",
+        len(seen),
+        duplicates,
+    )
+    return list(seen.values())

schematico/generator.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import Callable
+
+import logfire
+from pydantic import BaseModel
+from pydantic_ai import Agent
+from pydantic_ai.models import Model
+
+from schematico.helpers import _table_name, _hash_record, _describe_fields
+from schematico.logging import get_logger
+from schematico.models import build_batch_model
+from schematico.providers import DEFAULT_MODEL
+
+logger = get_logger("core.generator")
+
+
+def _build_prompt(schema: type[BaseModel], samples: int, instructions: str) -> str:
+    table = _table_name(schema)
+    field_lines = _describe_fields(schema)
+    prompt = (
+        f"You are a data generation agent for the '{table}' table.\n"
+        f"Generate exactly {samples} realistic, unique records with "
+        "these fields:\n" + "\n".join(field_lines) + "\n\nRules:\n"
+        "- Every record must be unique across all fields.\n"
+        "- Enum fields must use only the declared values.\n"
+        "- Numeric fields must respect any declared min/max range.\n"
+        "- Return exactly the requested number of records."
+    )
+    if instructions:
+        prompt += f"\n\nAdditional instructions:\n{instructions}"
+    return prompt
+
+
+def build_agent(
+    schema: type[BaseModel],
+    samples: int,
+    instructions: str = "",
+    model: str | Model | None = None,
+) -> Agent:
+    resolved: str | Model = model if model is not None else DEFAULT_MODEL
+    table = _table_name(schema)
+    logger.debug("Building agent for '%s' with model %r", table, resolved)
+    batch_model = build_batch_model(schema)
+
+    agent = Agent(
+        model=resolved,
+        output_type=batch_model,
+        system_prompt=_build_prompt(schema, samples, instructions),
+    )
+    return agent
+
+
+def run_generation(
+    schema: type[BaseModel],
+    samples: int,
+    instructions: str = "",
+    model: str | Model | None = None,
+    logfire_token: str | None = None,
+    progress_cb: Callable[[int, int, str], None] | None = None,
+) -> list[dict]:
+    if logfire_token:
+        logfire.configure(token=logfire_token, send_to_logfire=True)
+    else:
+        logfire.configure(send_to_logfire=False)
+    logfire.instrument_pydantic_ai()
+
+    table = _table_name(schema)
+    logger.info(
+        "Starting generation run for '%s' (%d records requested)", table, samples
+    )
+    agent = build_agent(schema, samples, instructions, model=model)
+    result = agent.run_sync(
+        f"Generate exactly {samples} unique records for the '{table}' table."
+    )
+    logger.debug("Agent returned %d raw records", len(result.output.records))
+
+    seen: dict[str, dict] = {}
+    duplicates = 0
+    for record in result.output.records:
+        record_dict = record.model_dump()
+        h = _hash_record(record_dict)
+        if h in seen:
+            duplicates += 1
+            if progress_cb:
+                progress_cb(len(seen), samples, "duplicate")
+            continue
+        seen[h] = record_dict
+        if progress_cb:
+            progress_cb(len(seen), samples, "found")
+
+    logger.info(
+        "Generation run complete: %d unique records (%d duplicates discarded)",
+        len(seen),
+        duplicates,
+    )
+    return list(seen.values())

schematico/helpers.py

@@ -0,0 +1,38 @@
+import hashlib
+import json
+import re
+from typing import Any
+
+from pydantic import BaseModel
+
+
+def _table_name(schema: type[BaseModel]) -> str:
+    name = schema.__name__
+    if name.endswith("Record"):
+        name = name[: -len("Record")]
+    s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name)
+    return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
+
+
+def _hash_record(record: dict) -> str:
+    serialized = json.dumps(record, sort_keys=True, default=str)
+    return hashlib.sha256(serialized.encode()).hexdigest()
+
+
+def _describe_fields(schema: type[BaseModel]) -> list[str]:
+    json_schema = schema.model_json_schema()
+    properties: dict[str, dict[str, Any]] = json_schema.get("properties", {})
+    lines: list[str] = []
+    for name, prop in properties.items():
+        ptype = prop.get("type") or prop.get("anyOf") or "any"
+        line = f"- {name}: {ptype}"
+        desc = prop.get("description")
+        if desc:
+            line += f" — {desc}"
+        if "enum" in prop:
+            line += f" (must be exactly one of: {prop['enum']})"
+        lo, hi = prop.get("minimum"), prop.get("maximum")
+        if lo is not None or hi is not None:
+            line += f" (range: {lo} to {hi})"
+        lines.append(line)
+    return lines

schematico/logging.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+import os
+import sys
+
+_ROOT_NAME = "schematico"
+_configured = False
+
+
+def configure_logging() -> None:
+    global _configured
+    if _configured:
+        return
+
+    level_name = os.environ.get("LOG_LEVEL", "WARNING").upper()
+    level = getattr(logging, level_name, logging.WARNING)
+
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s %(levelname)-8s %(name)s: %(message)s")
+    )
+
+    root = logging.getLogger(_ROOT_NAME)
+    root.setLevel(level)
+    root.addHandler(handler)
+    root.propagate = False
+
+    _configured = True
+
+
+def get_logger(name: str) -> logging.Logger:
+    configure_logging()
+    return logging.getLogger(f"{_ROOT_NAME}.{name}")