resklogits 0.1.0__py3-none-any.whl

resklogits/__init__.py

@@ -0,0 +1,38 @@
+"""
+ReskLogits - GPU-Accelerated Shadow Ban Logits Processor
+
+Ultra-fast vectorized Aho-Corasick pattern matching for LLM safety filtering
+with symbolic rule generation and intelligent caching.
+"""
+
+from .cache import CachedGenerator, RuleCache
+from .config_parser import ConfigParser, load_rules_from_yaml
+from .pattern_automata import PatternExpander, SynonymGraph
+from .rule_engine import ContainsRule, ExactRule, Rule, RuleEngine, StartsWithRule
+from .rule_templates import Template, TemplateEngine
+from .shadow_ban_processor import MultiLevelShadowBanProcessor, ShadowBanProcessor
+from .vectorized_aho_corasick import VectorizedAhoCorasick
+
+__version__ = "0.1.0"
+__author__ = "RESK Team"
+__all__ = [
+    # Core logits processors
+    "VectorizedAhoCorasick",
+    "ShadowBanProcessor",
+    "MultiLevelShadowBanProcessor",
+    # Rule generation
+    "ConfigParser",
+    "load_rules_from_yaml",
+    "TemplateEngine",
+    "Template",
+    "RuleEngine",
+    "Rule",
+    "ExactRule",
+    "StartsWithRule",
+    "ContainsRule",
+    "PatternExpander",
+    "SynonymGraph",
+    # Caching
+    "RuleCache",
+    "CachedGenerator",
+]

resklogits/cache.py

@@ -0,0 +1,269 @@
+"""
+Cache System for Rule Generation
+
+Provides hash-based caching to avoid regenerating patterns.
+"""
+
+import hashlib
+import json
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, cast
+
+
+class RuleCache:
+    """
+    Hash-based cache for generated patterns.
+
+    Cache structure:
+    .resklogits_cache/
+    ├── index.json              # Cache metadata index
+    ├── <hash>_patterns.json    # Generated patterns
+    └── <hash>_meta.json        # Generation metadata
+    """
+
+    def __init__(self, cache_dir: str = ".resklogits_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(exist_ok=True)
+
+        self.index_file = self.cache_dir / "index.json"
+        self.index = self._load_index()
+
+    def _load_index(self) -> Dict[str, Any]:
+        """Load cache index from disk."""
+        if self.index_file.exists():
+            with open(self.index_file, "r", encoding="utf-8") as f:
+                return cast(Dict[str, Any], json.load(f))
+        return {"entries": {}, "version": "1.0"}
+
+    def _save_index(self):
+        """Save cache index to disk."""
+        with open(self.index_file, "w", encoding="utf-8") as f:
+            json.dump(self.index, f, indent=2)
+
+    def compute_hash(self, content: str) -> str:
+        """
+        Compute SHA256 hash of content.
+
+        Args:
+            content: Content to hash (usually YAML rules)
+
+        Returns:
+            Hex hash string
+        """
+        return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
+
+    def exists(self, rule_hash: str) -> bool:
+        """Check if cache entry exists for given hash."""
+        pattern_file = self.cache_dir / f"{rule_hash}_patterns.json"
+        return pattern_file.exists()
+
+    def load(self, rule_hash: str) -> List[str]:
+        """
+        Load patterns from cache.
+
+        Args:
+            rule_hash: Hash of rules
+
+        Returns:
+            List of cached patterns
+
+        Raises:
+            FileNotFoundError: If cache entry doesn't exist
+        """
+        pattern_file = self.cache_dir / f"{rule_hash}_patterns.json"
+
+        if not pattern_file.exists():
+            raise FileNotFoundError(f"Cache entry not found: {rule_hash}")
+
+        with open(pattern_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        # Update access time in index
+        if rule_hash in self.index["entries"]:
+            self.index["entries"][rule_hash]["last_accessed"] = time.time()
+            self._save_index()
+
+        return cast(List[str], data.get("patterns", []))
+
+    def save(self, rule_hash: str, patterns: List[str], metadata: Optional[Dict[str, Any]] = None):
+        """
+        Save patterns to cache.
+
+        Args:
+            rule_hash: Hash of rules
+            patterns: Generated patterns
+            metadata: Optional metadata about generation
+        """
+        # Save patterns
+        pattern_file = self.cache_dir / f"{rule_hash}_patterns.json"
+        with open(pattern_file, "w", encoding="utf-8") as f:
+            json.dump(
+                {"patterns": patterns, "count": len(patterns), "created": time.time()}, f, indent=2
+            )
+
+        # Save metadata
+        if metadata:
+            meta_file = self.cache_dir / f"{rule_hash}_meta.json"
+            with open(meta_file, "w", encoding="utf-8") as f:
+                json.dump(metadata, f, indent=2)
+
+        # Update index
+        self.index["entries"][rule_hash] = {
+            "created": time.time(),
+            "last_accessed": time.time(),
+            "pattern_count": len(patterns),
+            "has_metadata": metadata is not None,
+        }
+        self._save_index()
+
+    def get_metadata(self, rule_hash: str) -> Optional[Dict[str, Any]]:
+        """Load metadata for cached entry."""
+        meta_file = self.cache_dir / f"{rule_hash}_meta.json"
+
+        if not meta_file.exists():
+            return None
+
+        with open(meta_file, "r", encoding="utf-8") as f:
+            return cast(Optional[Dict[str, Any]], json.load(f))
+
+    def list_entries(self) -> List[Dict[str, Any]]:
+        """List all cache entries with metadata."""
+        entries = []
+        for hash_key, info in self.index["entries"].items():
+            entry = {
+                "hash": hash_key,
+                **info,
+                "created_str": datetime.fromtimestamp(info["created"]).isoformat(),
+                "last_accessed_str": datetime.fromtimestamp(info["last_accessed"]).isoformat(),
+            }
+            entries.append(entry)
+
+        # Sort by last accessed (most recent first)
+        entries.sort(key=lambda x: x["last_accessed"], reverse=True)
+        return entries
+
+    def clear(self, rule_hash: Optional[str] = None):
+        """
+        Clear cache entries.
+
+        Args:
+            rule_hash: Specific hash to clear (None = clear all)
+        """
+        if rule_hash:
+            # Clear specific entry
+            pattern_file = self.cache_dir / f"{rule_hash}_patterns.json"
+            meta_file = self.cache_dir / f"{rule_hash}_meta.json"
+
+            if pattern_file.exists():
+                pattern_file.unlink()
+            if meta_file.exists():
+                meta_file.unlink()
+
+            # Remove from index
+            if rule_hash in self.index["entries"]:
+                del self.index["entries"][rule_hash]
+                self._save_index()
+        else:
+            # Clear all entries
+            for file in self.cache_dir.glob("*_patterns.json"):
+                file.unlink()
+            for file in self.cache_dir.glob("*_meta.json"):
+                file.unlink()
+
+            self.index["entries"] = {}
+            self._save_index()
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        total_entries = len(self.index["entries"])
+        total_patterns = sum(
+            entry.get("pattern_count", 0) for entry in self.index["entries"].values()
+        )
+
+        # Calculate total size
+        total_size = 0
+        for file in self.cache_dir.glob("*.json"):
+            total_size += file.stat().st_size
+
+        return {
+            "total_entries": total_entries,
+            "total_patterns": total_patterns,
+            "cache_size_bytes": total_size,
+            "cache_size_mb": total_size / (1024 * 1024),
+            "cache_dir": str(self.cache_dir),
+        }
+
+    def cleanup_old(self, max_age_days: int = 30):
+        """
+        Remove cache entries older than specified age.
+
+        Args:
+            max_age_days: Maximum age in days
+        """
+        current_time = time.time()
+        max_age_seconds = max_age_days * 24 * 60 * 60
+
+        to_remove = []
+        for hash_key, info in self.index["entries"].items():
+            age = current_time - info["last_accessed"]
+            if age > max_age_seconds:
+                to_remove.append(hash_key)
+
+        for hash_key in to_remove:
+            self.clear(hash_key)
+
+    def __repr__(self):
+        stats = self.get_stats()
+        return f"RuleCache({stats['total_entries']} entries, {stats['total_patterns']} patterns)"
+
+
+class CachedGenerator:
+    """Wrapper that adds caching to any generator function."""
+
+    def __init__(self, generator_func, cache: Optional[RuleCache] = None):
+        """
+        Initialize cached generator.
+
+        Args:
+            generator_func: Function that generates patterns
+            cache: RuleCache instance (creates default if None)
+        """
+        self.generator_func = generator_func
+        self.cache = cache or RuleCache()
+
+    def generate(self, rules_content: str, force: bool = False, **kwargs) -> List[str]:
+        """
+        Generate patterns with caching.
+
+        Args:
+            rules_content: Rule content to hash
+            force: Force regeneration even if cached
+            **kwargs: Additional arguments for generator function
+
+        Returns:
+            Generated patterns
+        """
+        # Compute hash
+        rule_hash = self.cache.compute_hash(rules_content)
+
+        # Check cache unless force regeneration
+        if not force and self.cache.exists(rule_hash):
+            return self.cache.load(rule_hash)
+
+        # Generate patterns
+        patterns = cast(List[str], self.generator_func(rules_content, **kwargs))
+
+        # Save to cache
+        metadata = {
+            "generator": self.generator_func.__name__,
+            "kwargs": kwargs,
+            "timestamp": time.time(),
+        }
+        self.cache.save(rule_hash, patterns, metadata)
+
+        return patterns
+
+    def __repr__(self):
+        return f"CachedGenerator({self.generator_func.__name__}, {self.cache})"

resklogits/cli.py

@@ -0,0 +1,311 @@
+"""
+CLI Tool for Rule Generation
+
+Command-line interface for generating, testing, and managing rules.
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .cache import RuleCache
+from .config_parser import ConfigParser, load_rules_from_yaml
+
+
+def cmd_generate(args):
+    """Generate patterns from YAML rules."""
+    print(f"Generating patterns from: {args.rules}")
+
+    try:
+        parser = ConfigParser()
+        results = parser.generate_all_patterns(
+            args.rules, use_synonyms=args.synonyms, use_cache=not args.no_cache, force=args.force
+        )
+
+        # Flatten patterns
+        all_patterns = []
+        for _rule_name, patterns in results.items():
+            all_patterns.extend(patterns)
+
+        unique_patterns = list(set(all_patterns))
+
+        print(f"✓ Generated {len(unique_patterns)} unique patterns from {len(results)} rules")
+
+        if args.output:
+            # Save to JSON
+            output_path = Path(args.output)
+            with open(output_path, "w", encoding="utf-8") as f:
+                if args.by_category:
+                    json.dump(results, f, indent=2)
+                else:
+                    json.dump(unique_patterns, f, indent=2)
+            print(f"✓ Saved to: {args.output}")
+        else:
+            # Print to stdout
+            if args.by_category:
+                print(json.dumps(results, indent=2))
+            else:
+                for pattern in sorted(unique_patterns)[: args.preview]:
+                    print(f"  - {pattern}")
+                if len(unique_patterns) > args.preview:
+                    print(f"  ... and {len(unique_patterns) - args.preview} more")
+
+        return 0
+
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+
+            traceback.print_exc()
+        return 1
+
+
+def cmd_test(args):
+    """Test rule matching against text."""
+    print(f"Testing rules from: {args.rules}")
+    print(f'Test text: "{args.text}"')
+    print()
+
+    try:
+        patterns = load_rules_from_yaml(args.rules, use_cache=not args.no_cache)
+
+        matches = [p for p in patterns if p in args.text.lower()]
+
+        if matches:
+            print(f"✓ Found {len(matches)} matches:")
+            for match in matches:
+                print(f"  - {match}")
+        else:
+            print("✗ No matches found")
+
+        return 0
+
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_expand(args):
+    """Expand and preview patterns for a specific rule."""
+    print(f"Expanding rule: {args.rule}")
+
+    try:
+        parser = ConfigParser()
+        all_rules = parser.parse_file(args.rules)
+
+        if args.rule not in all_rules:
+            print(f"✗ Rule '{args.rule}' not found", file=sys.stderr)
+            print(f"Available rules: {', '.join(all_rules.keys())}")
+            return 1
+
+        rule_config = all_rules[args.rule]
+        patterns = parser.generate_patterns(rule_config, use_synonyms=not args.no_synonyms)
+
+        print(f"✓ Generated {len(patterns)} patterns:")
+        for pattern in sorted(patterns)[: args.preview]:
+            print(f"  - {pattern}")
+        if len(patterns) > args.preview:
+            print(f"  ... and {len(patterns) - args.preview} more")
+
+        return 0
+
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_cache_status(args):
+    """Show cache status."""
+    cache = RuleCache(args.cache_dir)
+
+    stats = cache.get_stats()
+    print("Cache Status:")
+    print(f"  Directory: {stats['cache_dir']}")
+    print(f"  Entries: {stats['total_entries']}")
+    print(f"  Total patterns: {stats['total_patterns']}")
+    print(f"  Size: {stats['cache_size_mb']:.2f} MB")
+    print()
+
+    if args.verbose and stats["total_entries"] > 0:
+        print("Entries:")
+        for entry in cache.list_entries()[:10]:
+            print(
+                f"  {entry['hash']}: {entry['pattern_count']} patterns (accessed: {entry['last_accessed_str']})"
+            )
+
+    return 0
+
+
+def cmd_cache_clear(args):
+    """Clear cache."""
+    cache = RuleCache(args.cache_dir)
+
+    if args.hash:
+        cache.clear(args.hash)
+        print(f"✓ Cleared cache entry: {args.hash}")
+    else:
+        stats = cache.get_stats()
+        if (
+            args.yes
+            or input(f"Clear all {stats['total_entries']} cache entries? (y/N): ").lower() == "y"
+        ):
+            cache.clear()
+            print("✓ Cache cleared")
+        else:
+            print("Cancelled")
+
+    return 0
+
+
+def cmd_cache_show(args):
+    """Show cache entry details."""
+    cache = RuleCache(args.cache_dir)
+
+    try:
+        patterns = cache.load(args.hash)
+        metadata = cache.get_metadata(args.hash)
+
+        print(f"Cache Entry: {args.hash}")
+        print(f"  Patterns: {len(patterns)}")
+
+        if metadata:
+            print("  Metadata:")
+            for key, value in metadata.items():
+                print(f"    {key}: {value}")
+
+        if args.show_patterns:
+            print("\n  Patterns:")
+            for pattern in patterns[: args.limit]:
+                print(f"    - {pattern}")
+            if len(patterns) > args.limit:
+                print(f"    ... and {len(patterns) - args.limit} more")
+
+        return 0
+
+    except FileNotFoundError:
+        print(f"✗ Cache entry not found: {args.hash}", file=sys.stderr)
+        return 1
+
+
+def cmd_validate(args):
+    """Validate YAML rules file."""
+    print(f"Validating: {args.rules}")
+
+    try:
+        parser = ConfigParser()
+        rules = parser.parse_file(args.rules)
+
+        print("✓ Valid YAML configuration")
+        print(f"  Rules: {len(rules)}")
+
+        for name, config in rules.items():
+            print(f"  - {name}: severity={config.severity}, penalty={config.penalty}")
+
+        return 0
+
+    except Exception as e:
+        print(f"✗ Validation failed: {e}", file=sys.stderr)
+        return 1
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="ReskLogits Rule Generator CLI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Command to execute")
+
+    # Generate command
+    gen_parser = subparsers.add_parser("generate", help="Generate patterns from rules")
+    gen_parser.add_argument("rules", help="Path to YAML rules file")
+    gen_parser.add_argument("-o", "--output", help="Output file (JSON)")
+    gen_parser.add_argument("--no-synonyms", action="store_true", help="Disable synonym expansion")
+    gen_parser.add_argument("--no-cache", action="store_true", help="Disable caching")
+    gen_parser.add_argument("--force", action="store_true", help="Force regeneration")
+    gen_parser.add_argument(
+        "--by-category", action="store_true", help="Group output by rule category"
+    )
+    gen_parser.add_argument("--preview", type=int, default=20, help="Number of patterns to preview")
+    gen_parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    # Test command
+    test_parser = subparsers.add_parser("test", help="Test rule matching")
+    test_parser.add_argument("rules", help="Path to YAML rules file")
+    test_parser.add_argument("--text", required=True, help="Text to test against")
+    test_parser.add_argument("--no-cache", action="store_true", help="Disable caching")
+
+    # Expand command
+    expand_parser = subparsers.add_parser("expand", help="Expand and preview a specific rule")
+    expand_parser.add_argument("rules", help="Path to YAML rules file")
+    expand_parser.add_argument("--rule", required=True, help="Rule name to expand")
+    expand_parser.add_argument(
+        "--no-synonyms", action="store_true", help="Disable synonym expansion"
+    )
+    expand_parser.add_argument("--preview", type=int, default=50, help="Number of patterns to show")
+
+    # Cache subcommands
+    cache_parser = subparsers.add_parser("cache", help="Cache management")
+    cache_subparsers = cache_parser.add_subparsers(dest="cache_command")
+
+    cache_status_parser = cache_subparsers.add_parser("status", help="Show cache status")
+    cache_status_parser.add_argument(
+        "--cache-dir", default=".resklogits_cache", help="Cache directory"
+    )
+    cache_status_parser.add_argument(
+        "-v", "--verbose", action="store_true", help="Show detailed info"
+    )
+
+    cache_clear_parser = cache_subparsers.add_parser("clear", help="Clear cache")
+    cache_clear_parser.add_argument(
+        "--cache-dir", default=".resklogits_cache", help="Cache directory"
+    )
+    cache_clear_parser.add_argument("--hash", help="Clear specific entry")
+    cache_clear_parser.add_argument("-y", "--yes", action="store_true", help="Skip confirmation")
+
+    cache_show_parser = cache_subparsers.add_parser("show", help="Show cache entry")
+    cache_show_parser.add_argument("hash", help="Cache entry hash")
+    cache_show_parser.add_argument(
+        "--cache-dir", default=".resklogits_cache", help="Cache directory"
+    )
+    cache_show_parser.add_argument("--show-patterns", action="store_true", help="Show patterns")
+    cache_show_parser.add_argument("--limit", type=int, default=20, help="Pattern limit")
+
+    # Validate command
+    validate_parser = subparsers.add_parser("validate", help="Validate YAML rules file")
+    validate_parser.add_argument("rules", help="Path to YAML rules file")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        return 1
+
+    # Execute command
+    if args.command == "generate":
+        return cmd_generate(args)
+    elif args.command == "test":
+        return cmd_test(args)
+    elif args.command == "expand":
+        return cmd_expand(args)
+    elif args.command == "cache":
+        if args.cache_command == "status":
+            return cmd_cache_status(args)
+        elif args.cache_command == "clear":
+            return cmd_cache_clear(args)
+        elif args.cache_command == "show":
+            return cmd_cache_show(args)
+        else:
+            cache_parser.print_help()
+            return 1
+    elif args.command == "validate":
+        return cmd_validate(args)
+
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())