provide-foundation 0.0.0.dev0__py3-none-any.whl → 0.0.0.dev2__py3-none-any.whl

provide/foundation/cli/commands/logs/generate_old.py

@@ -1,569 +0,0 @@
-"""
-Generate test logs command for Foundation CLI.
-
-Incorporates creative prose inspired by William S. Burroughs and the cut-up technique.
-"""
-
-import random
-import time
-from typing import Any
-
-try:
-    import click
-
-    _HAS_CLICK = True
-except ImportError:
-    click = None
-    _HAS_CLICK = False
-
-from provide.foundation.logger import get_logger
-
-log = get_logger(__name__)
-
-
-# Cut-up phrases inspired by Burroughs
-BURROUGHS_PHRASES = [
-    "mutated Soft Machine prescribed within data stream",
-    "pre-recorded talking asshole dissolved into under neon hum",
-    "the viral Word carrying a new strain of reality",
-    "equations of control flickering on a broken monitor",
-    "memory banks spilling future-pasts onto the terminal floor",
-    "a thousand junk units screaming in unison",
-    "the algebra of need computed by the Nova Mob",
-    "subliminal commands embedded in the white noise",
-    "the Biologic Courts passing sentence in a dream",
-    "Nova Police raiding the reality studio",
-    "the soft typewriter of the Other Half",
-    "a flickering hologram of Hassan i Sabbah",
-    "contaminated data feed from the Crab Nebula",
-    "thought-forms materializing in the Interzone",
-    "frequency shift reported by Sector 5",
-]
-
-# Standard technical messages
-TECHNICAL_MESSAGES = [
-    "Request processed successfully",
-    "Database connection established",
-    "Cache invalidated",
-    "User authenticated",
-    "Session initiated",
-    "Transaction completed",
-    "Queue message processed",
-    "Health check passed",
-    "Metrics exported",
-    "Configuration reloaded",
-    "Backup completed",
-    "Index rebuilt",
-    "Connection pool recycled",
-    "Rate limit enforced",
-    "Circuit breaker triggered",
-]
-
-# Services and operations for realistic logs
-SERVICES = [
-    "api-gateway",
-    "auth-service",
-    "payment-processor",
-    "user-service",
-    "notification-engine",
-    "data-pipeline",
-    "cache-layer",
-    "search-index",
-    "reality-studio",
-    "interzone-terminal",
-    "nova-police",
-    "soft-machine",
-]
-
-OPERATIONS = [
-    "handle_request",
-    "process_data",
-    "validate_input",
-    "execute_query",
-    "send_notification",
-    "update_cache",
-    "compute_metrics",
-    "sync_state",
-    "transmit_signal",
-    "decode_reality",
-    "intercept_word",
-    "scan_frequency",
-]
-
-DOMAINS = [
-    "transmission",
-    "control",
-    "reality",
-    "system",
-    "network",
-    "quantum",
-    "temporal",
-    "dimensional",
-    "biologic",
-    "viral",
-]
-
-ACTIONS = [
-    "broadcast",
-    "receive",
-    "process",
-    "analyze",
-    "detect",
-    "mutate",
-    "dissolve",
-    "compute",
-    "raid",
-    "intercept",
-]
-
-STATUSES = [
-    "nominal",
-    "degraded",
-    "critical",
-    "optimal",
-    "unstable",
-    "fluctuating",
-    "synchronized",
-    "divergent",
-    "contaminated",
-    "clean",
-]
-
-
-if _HAS_CLICK:
-
-    @click.command("generate")
-    @click.option(
-        "--count",
-        "-n",
-        type=int,
-        default=100,
-        help="Number of logs to generate (0 for continuous)",
-    )
-    @click.option(
-        "--rate",
-        "-r",
-        type=float,
-        default=10.0,
-        help="Target logs per second (can go up to 10000/s)",
-    )
-    @click.option(
-        "--style",
-        type=click.Choice(["technical", "burroughs", "mixed"]),
-        default="mixed",
-        help="Log message style",
-    )
-    @click.option(
-        "--error-rate",
-        type=float,
-        default=0.1,
-        help="Percentage of error logs (0.0 to 1.0)",
-    )
-    @click.option(
-        "--services",
-        help="Comma-separated list of services (uses defaults if not provided)",
-    )
-    @click.option(
-        "--stream",
-        default="default",
-        help="Target stream for logs",
-    )
-    @click.option(
-        "--batch-size",
-        type=int,
-        default=10,
-        help="Number of logs to send in each batch",
-    )
-    @click.option(
-        "--with-traces",
-        is_flag=True,
-        default=True,
-        help="Generate trace IDs for correlation",
-    )
-    @click.pass_context
-    def generate_command(
-        ctx, count, rate, style, error_rate, services, stream, batch_size, with_traces
-    ):
-        """Generate test logs with optional Burroughs-inspired prose.
-
-        Examples:
-            # Generate 100 test logs
-            foundation logs generate -n 100
-
-            # Generate continuous logs at 5/second
-            foundation logs generate -n 0 -r 5
-
-            # Generate with Burroughs-style messages
-            foundation logs generate --style burroughs
-
-            # Generate with 20% error rate
-            foundation logs generate --error-rate 0.2
-
-            # Generate for specific services
-            foundation logs generate --services "api,auth,payment"
-        """
-
-        client = ctx.obj.get("client")
-
-        # Parse services
-        if services:
-            service_list = [s.strip() for s in services.split(",")]
-        else:
-            service_list = SERVICES
-
-        click.echo("🚀 Starting log generation...")
-        click.echo(f"   Style: {style}")
-        click.echo(f"   Error rate: {error_rate * 100:.0f}%")
-        click.echo(f"   Target stream: {stream}")
-        if count == 0:
-            click.echo(f"   Mode: Continuous at {rate} logs/second")
-        else:
-            click.echo(f"   Count: {count} logs")
-        click.echo("   Press Ctrl+C to stop\n")
-
-        def generate_message(style: str, index: int) -> tuple[str, str]:
-            """Generate a log message based on style."""
-            if style == "burroughs":
-                message = random.choice(BURROUGHS_PHRASES)
-                level = random.choice(["TRACE", "DEBUG", "INFO", "WARN", "ERROR"])
-            elif style == "technical":
-                message = random.choice(TECHNICAL_MESSAGES)
-                level = random.choice(["DEBUG", "INFO", "WARN"] * 3 + ["ERROR"])
-            else:  # mixed
-                if random.random() > 0.7:
-                    message = random.choice(BURROUGHS_PHRASES)
-                    level = random.choice(["TRACE", "DEBUG", "INFO", "WARN", "ERROR"])
-                else:
-                    message = random.choice(TECHNICAL_MESSAGES)
-                    level = random.choice(["DEBUG", "INFO", "WARN"] * 3 + ["ERROR"])
-
-            # Override level based on error rate
-            if random.random() < error_rate:
-                level = "ERROR"
-                if style != "burroughs":
-                    message = f"Error: {message}"
-
-            return message, level
-
-        def generate_log_entry(index: int) -> dict[str, Any]:
-            """Generate a single log entry."""
-            message, level = generate_message(style, index)
-            service = random.choice(service_list)
-            operation = random.choice(OPERATIONS)
-
-            entry = {
-                "message": f"[{service}] {message}",
-                "level": level,
-                "service": service,
-                "operation": operation,
-                "domain": random.choice(DOMAINS),
-                "action": random.choice(ACTIONS),
-                "status": "degraded" if level == "ERROR" else random.choice(STATUSES),
-                "duration_ms": random.randint(10, 5000),
-                "iteration": index,
-            }
-
-            # Add trace correlation
-            if with_traces:
-                # Group logs by trace (5-10 logs per trace)
-                trace_group = index // random.randint(5, 10)
-                entry["trace_id"] = f"trace_{trace_group:08x}"
-                entry["span_id"] = f"span_{index:08x}"
-
-            # Add error details
-            if level == "ERROR":
-                entry["error_code"] = random.choice([400, 404, 500, 502, 503])
-                entry["error_type"] = random.choice(
-                    [
-                        "ConnectionTimeout",
-                        "ValidationError",
-                        "DatabaseError",
-                        "ServiceUnavailable",
-                    ]
-                )
-
-            return entry
-
-        try:
-            logs_sent = 0
-            logs_failed = 0
-            logs_rate_limited = 0
-            start_time = time.time()
-            last_stats_time = start_time
-            last_stats_sent = 0
-
-            # Set up rate limiter
-            # Configure with the target rate and a reasonable burst size
-            rate_limiter = SimpleSyncRateLimiter(
-                capacity=min(
-                    rate * 2, 1000
-                ),  # Allow burst up to 2 seconds worth or 1000
-                refill_rate=rate,  # tokens per second
-            )
-
-            # Track rate limiting
-            rate_limit_detected = False
-            consecutive_failures = 0
-
-            def send_log_with_tracking(entry):
-                """Send log using Foundation's logger and track success/failure."""
-                nonlocal \
-                    logs_sent, \
-                    logs_failed, \
-                    logs_rate_limited, \
-                    rate_limit_detected, \
-                    consecutive_failures
-
-                try:
-                    # Get a logger for the service
-                    service_logger = get_logger(f"generated.{entry['service']}")
-
-                    # Use Foundation's logger with appropriate level
-                    level = entry["level"].lower()
-                    message = entry["message"]
-
-                    # Remove message and level from attributes since they're passed separately
-                    attrs = {
-                        k: v for k, v in entry.items() if k not in ["message", "level"]
-                    }
-
-                    # Call the appropriate log level method
-                    if level == "trace":
-                        service_logger.trace(message, **attrs)
-                    elif level == "debug":
-                        service_logger.debug(message, **attrs)
-                    elif level == "info":
-                        service_logger.info(message, **attrs)
-                    elif level == "warn" or level == "warning":
-                        service_logger.warning(message, **attrs)
-                    elif level == "error":
-                        service_logger.error(message, **attrs)
-                    elif level == "critical":
-                        service_logger.critical(message, **attrs)
-                    else:
-                        service_logger.info(message, **attrs)
-
-                    # Also send to OpenObserve if configured
-                    if client:
-                        from provide.foundation.observability.openobserve.otlp import (
-                            send_log_bulk,
-                        )
-
-                        success = send_log_bulk(
-                            message=message,
-                            level=entry["level"],
-                            service=entry["service"],
-                            attributes=attrs,
-                            client=client,
-                        )
-                        if not success:
-                            logs_failed += 1
-                            consecutive_failures += 1
-                            if consecutive_failures >= 5 and not rate_limit_detected:
-                                rate_limit_detected = True
-                                logs_rate_limited = logs_failed
-                            elif rate_limit_detected:
-                                logs_rate_limited += 1
-                            return False
-
-                    logs_sent += 1
-                    consecutive_failures = 0
-                    return True
-
-                except Exception as e:
-                    log.debug(f"Failed to send log: {e}")
-                    logs_failed += 1
-                    consecutive_failures += 1
-                    return False
-
-            if count == 0:
-                # Continuous mode using Foundation's rate limiter with async workers
-                index = 0
-                import concurrent.futures
-                import queue
-
-                # Work queue for async processing
-                work_queue = queue.Queue(
-                    maxsize=int(rate * 2)
-                )  # Buffer up to 2 seconds of logs
-
-                # Use thread pool for high-speed sending
-                # More workers for higher rates
-                num_workers = min(50, max(4, int(rate / 100)))
-
-                with concurrent.futures.ThreadPoolExecutor(
-                    max_workers=num_workers
-                ) as executor:
-                    # Start worker threads
-                    futures = []
-
-                    def worker():
-                        """Worker thread that processes logs from queue."""
-                        while True:
-                            try:
-                                entry = work_queue.get(timeout=1)
-                                if entry is None:  # Shutdown signal
-                                    break
-                                send_log_with_tracking(entry)
-                                work_queue.task_done()
-                            except queue.Empty:
-                                continue
-
-                    # Start workers
-                    for _ in range(num_workers):
-                        futures.append(executor.submit(worker))
-
-                    while True:
-                        current_time = time.time()
-
-                        # Try to acquire a token from the rate limiter
-                        if rate_limiter.acquire():
-                            # Token acquired, generate and queue log
-                            entry = generate_log_entry(index)
-                            try:
-                                work_queue.put_nowait(entry)
-                                index += 1
-                            except queue.Full:
-                                # Queue is full, we're generating faster than sending
-                                logs_rate_limited += 1
-                                if not rate_limit_detected:
-                                    rate_limit_detected = True
-                                    log.warning(
-                                        "⚠️ Queue full - cannot keep up with target rate"
-                                    )
-                        else:
-                            # Rate limited - track it
-                            logs_rate_limited += 1
-                            if not rate_limit_detected:
-                                rate_limit_detected = True
-                                log.debug(
-                                    "⚠️ Rate limiter activated - target rate exceeded"
-                                )
-
-                            # Small sleep to prevent busy waiting
-                            time.sleep(0.0001)
-
-                        # Print stats every second
-                        if current_time - last_stats_time >= 1.0:
-                            current_sent = logs_sent
-                            current_rate = (current_sent - last_stats_sent) / (
-                                current_time - last_stats_time
-                            )
-                            tokens_available = rate_limiter.tokens
-                            queue_size = work_queue.qsize()
-
-                            status = f"📊 Sent: {logs_sent:,} | Rate: {current_rate:.0f}/s | Tokens: {tokens_available:.0f}/{rate_limiter.capacity} | Queue: {queue_size}"
-                            if logs_failed > 0:
-                                status += f" | Failed: {logs_failed:,}"
-                            if rate_limit_detected:
-                                status += f" | ⚠️ RATE LIMITED ({logs_rate_limited:,} throttled)"
-
-                            click.echo(status)
-                            last_stats_time = current_time
-                            last_stats_sent = current_sent
-
-            else:
-                # Fixed count mode using Foundation's rate limiter with async workers
-                import concurrent.futures
-                import queue
-
-                # Work queue for async processing
-                work_queue = queue.Queue(maxsize=min(1000, count))
-
-                # Use thread pool for high-speed sending
-                num_workers = min(50, max(4, int(rate / 100)))
-
-                with concurrent.futures.ThreadPoolExecutor(
-                    max_workers=num_workers
-                ) as executor:
-                    # Start worker threads
-                    def worker():
-                        """Worker thread that processes logs from queue."""
-                        while True:
-                            try:
-                                entry = work_queue.get(timeout=1)
-                                if entry is None:  # Shutdown signal
-                                    break
-                                send_log_with_tracking(entry)
-                                work_queue.task_done()
-                            except queue.Empty:
-                                continue
-
-                    # Start workers
-                    workers = [executor.submit(worker) for _ in range(num_workers)]
-
-                    # Generate and queue logs
-                    for i in range(count):
-                        # Wait for rate limiter token
-                        while not rate_limiter.acquire():
-                            logs_rate_limited += 1
-                            if not rate_limit_detected:
-                                rate_limit_detected = True
-                                log.debug(
-                                    "⚠️ Rate limiter activated - target rate exceeded"
-                                )
-                            time.sleep(
-                                0.0001
-                            )  # Very small sleep to prevent busy waiting
-
-                        # Generate and queue log
-                        entry = generate_log_entry(i)
-                        work_queue.put(entry)
-
-                        # Print progress
-                        if (i + 1) % 100 == 0:
-                            current_time = time.time()
-                            elapsed = current_time - start_time
-                            current_rate = logs_sent / elapsed if elapsed > 0 else 0
-                            tokens_available = rate_limiter.tokens
-                            queue_size = work_queue.qsize()
-
-                            status = f"📊 Progress: {i + 1}/{count} | Sent: {logs_sent:,} | Rate: {current_rate:.0f}/s | Queue: {queue_size} | Tokens: {tokens_available:.0f}"
-                            if logs_failed > 0:
-                                status += f" | Failed: {logs_failed}"
-                            if rate_limit_detected:
-                                status += f" | ⚠️ THROTTLED ({logs_rate_limited:,})"
-
-                            click.echo(status)
-
-                    # Wait for queue to empty
-                    click.echo("⏳ Waiting for queue to empty...")
-                    work_queue.join()
-
-                    # Shutdown workers
-                    for _ in range(num_workers):
-                        work_queue.put(None)
-
-                    # Wait for workers to finish
-                    concurrent.futures.wait(workers)
-
-            elapsed = time.time() - start_time
-            rate_actual = logs_sent / elapsed if elapsed > 0 else 0
-
-            click.echo("\n📊 Generation complete:")
-            click.echo(f"   Total sent: {logs_sent:,} logs")
-            click.echo(f"   Total failed: {logs_failed:,} logs")
-            if rate_limit_detected:
-                click.echo(f"   ⚠️  Rate limited: {logs_rate_limited:,} logs")
-            click.echo(f"   Time: {elapsed:.2f}s")
-            click.echo(f"   Target rate: {rate:.0f} logs/second")
-            click.echo(f"   Actual rate: {rate_actual:.1f} logs/second")
-            if rate_limit_detected and rate_actual < rate * 0.5:
-                click.echo(
-                    f"   ⚠️  Rate limiting detected - actual rate is {(rate_actual / rate) * 100:.0f}% of target"
-                )
-
-        except KeyboardInterrupt:
-            click.echo(f"\n✋ Stopped. Generated {logs_sent} logs.")
-        except Exception as e:
-            click.echo(f"Generation failed: {e}", err=True)
-            return 1
-
-else:
-
-    def generate_command(*args, **kwargs):
-        """Generate command stub when click is not available."""
-        raise ImportError(
-            "CLI commands require optional dependencies. "
-            "Install with: pip install 'provide-foundation[cli]'"
-        )