scorebook 0.0.3__tar.gz → 0.0.5__tar.gz

{scorebook-0.0.3 → scorebook-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scorebook
-Version: 0.0.3
+Version: 0.0.5
 Summary: A Python project for LLM evaluation.
 Author: Euan Campbell
 Author-email: euan@trismik.com
@@ -35,6 +35,7 @@ Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"
 Requires-Dist: transformers ; extra == "examples"
+Requires-Dist: trismik
 Description-Content-Type: text/markdown
 
 # Scorebook

{scorebook-0.0.3 → scorebook-0.0.5}/pyproject.toml

@@ -11,12 +11,20 @@ requires-python = ">=3.9"
 dependencies = [
     "datasets>=3.6.0",
     "notebook (>=7.4.5,<8.0.0)",
+    "trismik",
 ]
 
+[project.scripts]
+scorebook = "scorebook.cli.main:main"
+
 [tool.poetry]
-version = "0.0.3"  # base version
+version = "0.0.5"  # base version
 packages = [{ include = "scorebook", from = "src" }]
 
+[[tool.poetry.source]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+priority = "supplemental"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.2"

scorebook-0.0.5/src/scorebook/__init__.py

@@ -0,0 +1,18 @@
+"""
+Scorebook package.
+
+A Python project for scorebook functionality.
+"""
+
+import importlib.metadata
+
+# get version from pyproject.toml
+__version__ = importlib.metadata.version(__package__ or __name__)
+
+from scorebook.eval_dataset import EvalDataset
+from scorebook.evaluate import evaluate
+from scorebook.inference_pipeline import InferencePipeline
+from scorebook.trismik.login import login, whoami
+from scorebook.utils.build_prompt import build_prompt
+
+__all__ = ["EvalDataset", "evaluate", "build_prompt", "login", "whoami", "InferencePipeline"]

scorebook-0.0.5/src/scorebook/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI module for scorebook."""

scorebook-0.0.5/src/scorebook/cli/auth.py

@@ -0,0 +1,98 @@
+"""Authentication CLI commands."""
+
+import argparse
+import getpass
+import sys
+
+from scorebook.trismik.login import get_stored_token, get_token_path, login, logout, whoami
+
+
+def auth_command(args: argparse.Namespace) -> int:
+    """Handle auth subcommands."""
+    if args.auth_command == "login":
+        return login_command(args)
+    elif args.auth_command == "logout":
+        return logout_command(args)
+    elif args.auth_command == "whoami":
+        return whoami_command(args)
+    else:
+        print(
+            "Error: No auth command specified. Use 'login', 'logout', or 'whoami'.", file=sys.stderr
+        )
+        return 1
+
+
+def login_command(args: argparse.Namespace) -> int:
+    """Handle login command."""
+    try:
+        token = args.token
+
+        if not token:
+            # Check if we're already logged in
+            stored_token = get_stored_token()
+
+            if stored_token:
+                print("You are already logged in.")
+                overwrite = (
+                    input("Do you want to overwrite the existing token? (y/N): ").lower().strip()
+                )
+                if overwrite not in ("y", "yes"):
+                    print("Login cancelled.")
+                    return 0
+
+            # Prompt for token securely
+            print("Enter your Trismik API token:")
+            print("(You can find your token at: https://trismik.com/settings/tokens)")
+            token = getpass.getpass("Token: ").strip()
+
+        if not token:
+            print("Error: No token provided.", file=sys.stderr)
+            return 1
+
+        # Login
+        login(token)
+
+        # Success message
+        print(f"Successfully logged in! Token saved to {get_token_path()}")
+        return 0
+
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("\nLogin cancelled.")
+        return 130
+    except Exception as e:
+        print(f"Unexpected error: {e}", file=sys.stderr)
+        return 1
+
+
+def logout_command(args: argparse.Namespace) -> int:
+    """Handle logout command."""
+    try:
+        success = logout()
+        if success:
+            print("Successfully logged out!")
+        else:
+            print("Not currently logged in.")
+        return 0
+    except Exception as e:
+        print(f"Error during logout: {e}", file=sys.stderr)
+        return 1
+
+
+def whoami_command(args: argparse.Namespace) -> int:
+    """Handle whoami command."""
+    try:
+        token = whoami()
+        if token is None:
+            print("Not logged in. Run 'scorebook auth login' first.")
+            return 1
+        else:
+            # TODO: Make actual API call to get user info
+            # For now, just confirm we have a token
+            print(f"Logged in with token: {token[:8]}...")
+            return 0
+    except Exception as e:
+        print(f"Error checking login status: {e}", file=sys.stderr)
+        return 1

scorebook-0.0.5/src/scorebook/cli/main.py

@@ -0,0 +1,57 @@
+"""Main CLI entry point for scorebook."""
+
+import argparse
+import sys
+from typing import List, Optional
+
+from .auth import auth_command
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create the main argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="scorebook",
+        description="Scorebook CLI - A Python project for LLM evaluation",
+    )
+
+    # Add subcommands
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Auth subcommand
+    auth_parser = subparsers.add_parser("auth", help="Authentication commands")
+    auth_subparsers = auth_parser.add_subparsers(dest="auth_command", help="Auth commands")
+
+    # Auth login
+    login_parser = auth_subparsers.add_parser("login", help="Login to scorebook")
+    login_parser.add_argument("--token", help="API token to use for login")
+
+    # Auth logout
+    auth_subparsers.add_parser("logout", help="Logout from scorebook")
+
+    # Auth whoami
+    auth_subparsers.add_parser("whoami", help="Show current login status")
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Run the main CLI entry point."""
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "auth":
+            return auth_command(args)
+        else:
+            parser.print_help()
+            return 1
+    except KeyboardInterrupt:
+        print("\nOperation cancelled.")
+        return 130
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{scorebook-0.0.3/src/scorebook/types → scorebook-0.0.5/src/scorebook}/eval_dataset.py

@@ -2,6 +2,7 @@
 
 import csv
 import json
+import random
 from typing import Any, Dict, Iterator, List, Optional, Type, Union
 
 import yaml
@@ -364,3 +365,40 @@ class EvalDataset:
                 resolved.append(MetricRegistry.get(m))  # Use registry for str or class
 
         return resolved
+
+    def sample(self, sample_size: int) -> "EvalDataset":
+        """Create a new dataset with randomly sampled items from this dataset.
+
+        Args:
+            sample_size: The number of items to sample from the dataset
+
+        Returns:
+            A new EvalDataset with randomly sampled items
+
+        Raises:
+            ValueError: If sample_size is larger than the dataset size
+        """
+        dataset_size = len(self.items)
+
+        if sample_size > dataset_size:
+            raise ValueError(
+                f"Sample size {sample_size} is larger than dataset size {dataset_size} "
+                f"for dataset '{self.name}'"
+            )
+
+        # Create randomly sampled items
+        sampled_items = random.sample(self.items, sample_size)
+
+        # Create a new EvalDataset instance with sampled items using from_list
+        sampled_dataset = self.from_list(
+            name=self.name,
+            label=self.label,
+            metrics=self.metrics,
+            data=sampled_items,
+        )
+
+        # Preserve the prompt template if it exists
+        if self.prompt_template is not None:
+            sampled_dataset.prompt_template = self.prompt_template
+
+        return sampled_dataset