loggerbuf 0.9.0__py3-none-any.whl

loggerbuf/cli/handlers/decode.py

@@ -0,0 +1,234 @@
+import sys
+import os
+import gzip
+import json
+import re
+from google.protobuf.json_format import MessageToDict
+
+try:
+    from ... import schema_loader
+    main_data_pb2 = schema_loader.get_main_data_pb2()
+    Event = main_data_pb2.Event
+    CounterEvent = main_data_pb2.CounterEvent
+except ImportError:
+    Event = None
+    CounterEvent = None
+
+def decode_file(filepath, verify_key=None, skip_integrity=False, is_counter=False):
+    """
+    Reads a length-prefixed binary log file (raw or gzipped)
+    and yields deserialized Event objects.
+    """
+    record_class = CounterEvent if is_counter else Event
+    if not record_class:
+        raise ImportError(f"{record_class.__name__ if record_class else 'Event class'} is not available. Please run 'loggerbuf build' first.")
+        
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"File not found: {filepath}")
+
+    open_func = gzip.open if filepath.endswith('.gz') else open
+    
+    import hmac
+    import hashlib
+    import click
+
+    current_hash = None
+    event_index = 0
+
+    with open_func(filepath, 'rb') as f:
+        while True:
+            header = f.read(4)
+            if not header:
+                break
+            if len(header) < 4:
+                click.secho(f"Warning: Truncated header found in {filepath} at position {f.tell() - len(header)}", fg='red', err=True)
+                break
+            
+            size = int.from_bytes(header, byteorder='big')
+            payload = f.read(size)
+            if len(payload) < size:
+                click.secho(f"Warning: Truncated payload of size {size} in {filepath}", fg='red', err=True)
+                break
+            
+            event_index += 1
+            
+            try:
+                event = record_class.FromString(payload)
+                
+                # --- HMAC Verification ---
+                if not skip_integrity and verify_key:
+                    if hasattr(event, "hmac_signature") and event.hmac_signature:
+                        stored_sig = event.hmac_signature
+                        
+                        # Prepare payload for hash
+                        event.ClearField("hmac_signature")
+                        
+                        if getattr(event, "is_chain_start", False):
+                            # Start of a new chain
+                            prev = event.previous_file_hash if getattr(event, "previous_file_hash", b'') else b''
+                        else:
+                            prev = current_hash if current_hash else b''
+                            
+                        clean_payload = event.SerializeToString()
+                        
+                        expected_hash = hmac.new(verify_key.encode('utf-8'), clean_payload + prev, hashlib.sha256).digest()
+                        
+                        if expected_hash != stored_sig:
+                            click.secho(f"\n[!] CRITICAL ALERT: Integrity Compromised at Event #{event_index} (offset {f.tell() - size})", fg='red', err=True)
+                            click.secho(f"    Calculated signature does not match stored signature.", fg='red', err=True)
+                        
+                        # Update running hash
+                        current_hash = stored_sig
+                        # Restore signature for output consistency if needed
+                        event.hmac_signature = stored_sig
+
+                yield event
+            except Exception as e:
+                click.secho(f"Warning: Failed to deserialize event of size {size} at index {event_index}: {e}", fg='red', err=True)
+
+def run_decode(input_file: str, output_file: str, format: str, stats: bool, head: int, tail: int, verify_key: str = None, skip_integrity: bool = False, is_counter: bool = False):
+    """Main decoding logic separated from click."""
+    
+    try:
+        event_generator = decode_file(input_file, verify_key, skip_integrity, is_counter)
+    except Exception as e:
+        print(f"Error reading file: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if head:
+        import itertools
+        event_generator = itertools.islice(event_generator, head)
+    elif tail:
+        import collections
+        event_generator = collections.deque(event_generator, maxlen=tail)
+
+    total_events = 0
+
+    if stats:
+        event_types = {}
+        statuses = {}
+        
+        try:
+            for ev in event_generator:
+                total_events += 1
+                if is_counter:
+                    ev_type_name = str(ev.counter_type)
+                    status_name = "N/A"
+                else:
+                    ev_type_name = str(ev.event_type)
+                    status_name = str(ev.status)
+                
+                event_types[ev_type_name] = event_types.get(ev_type_name, 0) + (ev.count if is_counter else 1)
+                statuses[status_name] = statuses.get(status_name, 0) + 1
+        except Exception as e:
+            print(f"Error decoding events: {e}", file=sys.stderr)
+
+        print(f"=== LoggerBuf Telemetry Statistics ===")
+        print(f"File: {input_file}")
+        print(f"Total events decoded: {total_events}")
+        
+        if is_counter:
+            print("\nCounters Breakdown:")
+            for k, v in event_types.items():
+                print(f"  - {k}: {v} (total count)")
+        else:
+            print("\nEvent Types Breakdown:")
+            for k, v in event_types.items():
+                print(f"  - {k}: {v}")
+            print("\nStatuses Breakdown:")
+            for k, v in statuses.items():
+                print(f"  - {k}: {v}")
+        return
+
+    out_f = open(output_file, "w") if output_file else sys.stdout
+    try:
+        for ev in event_generator:
+            total_events += 1
+            ev_dict = MessageToDict(ev, always_print_fields_with_no_presence=True)
+            if format == "pretty":
+                out_f.write(json.dumps(ev_dict, indent=2, ensure_ascii=False) + "\n")
+            else:
+                out_f.write(json.dumps(ev_dict, ensure_ascii=False) + "\n")
+    except Exception as e:
+        print(f"Error decoding events: {e}", file=sys.stderr)
+    finally:
+        if output_file:
+            out_f.close()
+            print(f"Successfully decoded {total_events} events and exported to {output_file}")
+
+
+import collections
+import itertools
+
+def decode_debug_file(filepath):
+    """
+    Reads a jsonl log file (raw or gzipped) and yields parsed JSON objects.
+    """
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"File not found: {filepath}")
+
+    open_func = gzip.open if filepath.endswith('.gz') else open
+    
+    with open_func(filepath, 'rt', encoding='utf-8') as f:
+        for line_num, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError as e:
+                print(f"Warning: Failed to decode JSON at {filepath}:{line_num}: {e}", file=sys.stderr)
+
+def run_decode_debug(input_file: str, grep_keyword: str = None, head: int = None, tail: int = None):
+    # Warning if it is the base file without rotation ext
+    basename = os.path.basename(input_file)
+    if basename.endswith(".log") and not re.search(r'\d{4}-\d{2}-\d{2}', basename):
+        print(f"\nWarning: You are decoding what appears to be the live active file '{basename}'. "
+              f"Some final bytes might be incomplete if written concurrently.\n", file=sys.stderr)
+
+    try:
+        log_generator = decode_debug_file(input_file)
+    except Exception as e:
+        print(f"Error reading file: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    total_logs = 0
+    matched_logs = 0
+    
+    grep_lower = grep_keyword.lower() if grep_keyword else None
+
+    if tail is not None and tail > 0:
+        log_generator = collections.deque(log_generator, maxlen=tail)
+
+    # Original visual format: '[{asctime}] >>{name}<< ({filename}::{caller_class}::{funcName}->{lineno}) - *{levelname}* - message::>{message}'
+    try:
+        for log_obj in log_generator:
+            if head is not None and matched_logs >= head:
+                break
+                
+            total_logs += 1
+            
+            timestamp = log_obj.get("timestamp", "Unknown")
+            logger_name = log_obj.get("logger", "Unknown")
+            filename = log_obj.get("file", "Unknown")
+            caller_class = log_obj.get("class", "None")
+            func_name = log_obj.get("function", "Unknown")
+            lineno = log_obj.get("line", 0)
+            level = log_obj.get("level", "UNKNOWN")
+            message = log_obj.get("message", "")
+            
+            formatted_msg = f"[{timestamp}] >>{logger_name}<< ({filename}::{caller_class}::{func_name}->{lineno}) - *{level}* - message::>{message}"
+            
+            if grep_lower:
+                if grep_lower not in formatted_msg.lower():
+                    continue
+            
+            matched_logs += 1
+            print(formatted_msg)
+            
+    except Exception as e:
+        print(f"Error decoding debug logs: {e}", file=sys.stderr)
+    finally:
+        if grep_keyword:
+            print(f"--- Filtered {matched_logs} matches out of {total_logs} total logs in {input_file} ---", file=sys.stderr)
+

loggerbuf/cli/handlers/events.py

@@ -0,0 +1,193 @@
+import os
+import re
+import click
+from typing import List
+from loggerbuf.cli.handlers.protos import get_protos_dir
+
+def _get_registry_proto() -> str:
+    return os.path.join(get_protos_dir(), "registry.proto")
+
+def _read_proto() -> str:
+    with open(_get_registry_proto(), "r") as f:
+        return f.read()
+
+def _write_proto(content: str):
+    with open(_get_registry_proto(), "w") as f:
+        f.write(content)
+
+def _get_max_value(enum_content: str) -> int:
+    """Finds the absolute maximum integer value used in the enum body."""
+    matches = re.findall(r'=\s*(\d+)\s*;', enum_content)
+    if not matches:
+        return 0
+    return max(int(m) for m in matches)
+
+def _append_to_enum(proto_content: str, enum_name: str, block_name: str, items: List[str], reserve: int) -> str:
+    """
+    Appends a new block to an enum.
+    """
+    enum_pattern = re.compile(rf'(enum\s+{enum_name}\s+{{)(.*?)(^\}})', re.DOTALL | re.MULTILINE)
+    match = enum_pattern.search(proto_content)
+    if not match:
+        raise ValueError(f"Enum {enum_name} not found in {REGISTRY_PROTO}")
+    
+    enum_start = match.group(1)
+    enum_body = match.group(2)
+    enum_end = match.group(3)
+    
+    max_val = _get_max_value(enum_body)
+    
+    start_val = max_val + 1
+    if start_val % 10 != 0:
+        start_val = ((start_val // 10) + 1) * 10
+    
+    next_val = start_val
+    new_block = f"\n    // Specific {enum_name} for {block_name}\n"
+    
+    added_lines = []
+    for item in items:
+        added_lines.append(f"    {item} = {next_val};")
+        next_val += 1
+        
+    end_range = next_val + reserve - 1
+    if end_range < next_val:
+        end_range = next_val
+        
+    new_block += f"    // Range: {start_val}-{end_range}\n"
+    new_block += f"    // Next value: {next_val}\n"
+    if added_lines:
+        new_block += "\n".join(added_lines) + "\n"
+    
+    new_enum_body = enum_body.rstrip() + "\n" + new_block
+    new_content = proto_content[:match.start()] + enum_start + new_enum_body + enum_end + proto_content[match.end():]
+    return new_content
+
+def _add_status_to_block(proto_content: str, enum_name: str, block_keyword: str, status_name: str) -> str:
+    enum_pattern = re.compile(rf'(enum\s+{enum_name}\s+{{)(.*?)(^\}})', re.DOTALL | re.MULTILINE)
+    match = enum_pattern.search(proto_content)
+    if not match:
+        raise ValueError(f"Enum {enum_name} not found.")
+    
+    enum_start = match.group(1)
+    enum_body = match.group(2)
+    enum_end = match.group(3)
+    
+    block_pattern = re.compile(rf'(//.*?{block_keyword}.*?\n\s*//\s*Range:\s*\d+-\d+\n\s*//\s*Next value:\s*)(\d+)(\n(?:.*?\n)*?)', re.IGNORECASE)
+    
+    block_match = None
+    for m in block_pattern.finditer(enum_body):
+        block_match = m
+        
+    if not block_match:
+        raise ValueError(f"Could not find reservation block for '{block_keyword}' in {enum_name}. Make sure it was created via CLI.")
+    
+    next_val = int(block_match.group(2))
+    
+    prefix = enum_body[:block_match.start(2)]
+    updated_next_val = str(next_val + 1)
+    suffix = enum_body[block_match.end(2):]
+    
+    lines = suffix.split('\n')
+    insert_idx = 0
+    for i, line in enumerate(lines):
+        if line.strip() == '' or line.strip().startswith('//'):
+            insert_idx = i
+            break
+    else:
+        insert_idx = len(lines)
+        
+    new_line = f"    {status_name} = {next_val};"
+    lines.insert(insert_idx, new_line)
+    
+    new_suffix = "\n".join(lines)
+    new_enum_body = prefix + updated_next_val + new_suffix
+    
+    new_content = proto_content[:match.start()] + enum_start + new_enum_body + enum_end + proto_content[match.end():]
+    return new_content
+
+def add_type(name: str, statuses: List[str], reserve: int):
+    proto_content = _read_proto()
+    
+    type_name = name.upper()
+    if not type_name.startswith("EVENT_"):
+        type_name = f"EVENT_{type_name}"
+        
+    proto_content = _append_to_enum(proto_content, "EventType", name, [type_name], reserve=reserve)
+    
+    status_names = []
+    prefix = name.upper()
+    if prefix.startswith("EVENT_"):
+        prefix = prefix[6:]
+        
+    for st in statuses:
+        st_name = st.upper()
+        if not st_name.startswith(f"{prefix}_STATUS_"):
+            st_name = f"{prefix}_STATUS_{st_name}"
+        status_names.append(st_name)
+        
+    proto_content = _append_to_enum(proto_content, "EventStatus", name, status_names, reserve=reserve)
+    _write_proto(proto_content)
+    
+    click.secho(f"Event type '{name}' and its statuses added successfully to registry.proto", fg="green")
+
+def add_status(type_name: str, status_name: str):
+    proto_content = _read_proto()
+    
+    prefix = type_name.upper()
+    if prefix.startswith("EVENT_"):
+        prefix = prefix[6:]
+        
+    st_name = status_name.upper()
+    if not st_name.startswith(f"{prefix}_STATUS_"):
+        st_name = f"{prefix}_STATUS_{st_name}"
+        
+    proto_content = _add_status_to_block(proto_content, "EventStatus", prefix, st_name)
+    _write_proto(proto_content)
+    
+    click.secho(f"Status '{st_name}' added successfully to registry.proto", fg="green")
+
+def list_events(type_name: str = None):
+    proto_content = _read_proto()
+    click.secho(f"--- EventType ---", fg="cyan")
+    
+    enum_pattern = re.compile(r'enum\s+EventType\s+\{(.*?)\}', re.DOTALL)
+    match = enum_pattern.search(proto_content)
+    if match:
+        for line in match.group(1).split('\n'):
+            if '=' in line and not line.strip().startswith('//'):
+                click.echo(line.strip())
+                
+    click.secho(f"\n--- EventStatus ---", fg="cyan")
+    enum_pattern = re.compile(r'enum\s+EventStatus\s+\{(.*?)\}', re.DOTALL)
+    match = enum_pattern.search(proto_content)
+    if match:
+        for line in match.group(1).split('\n'):
+            if '=' in line and not line.strip().startswith('//'):
+                if type_name:
+                    prefix = type_name.upper()
+                    if prefix.startswith("EVENT_"):
+                        prefix = prefix[6:]
+                    if prefix in line:
+                        click.echo(line.strip())
+                else:
+                    click.echo(line.strip())
+
+def add_counter_type(name: str, counters: List[str], reserve: int):
+    proto_content = _read_proto()
+    
+    prefix = name.upper()
+    if prefix.startswith("COUNTER_"):
+        prefix = prefix[8:]
+        
+    counter_names = []
+    for ct in counters:
+        ct_name = ct.upper()
+        if not ct_name.startswith(f"{prefix}_"):
+            ct_name = f"{prefix}_{ct_name}"
+        counter_names.append(ct_name)
+        
+    proto_content = _append_to_enum(proto_content, "CounterType", name, counter_names, reserve=reserve)
+    _write_proto(proto_content)
+    
+    click.secho(f"Counter block '{name}' with {len(counter_names)} counters added successfully to registry.proto", fg="green")
+

loggerbuf/cli/handlers/fields.py

@@ -0,0 +1,119 @@
+import os
+import glob
+import re
+import sys
+from loggerbuf.cli.handlers.protos import build, get_protos_dir
+
+def _find_file_for_message(message_name: str, file_name: str = None) -> str:
+    protos_dir = get_protos_dir()
+    if file_name:
+        path = os.path.join(protos_dir, file_name)
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"File '{file_name}' not found in {protos_dir}.")
+        return path
+        
+    files = glob.glob(os.path.join(protos_dir, "*.proto"))
+    matches = []
+    
+    for f in files:
+        with open(f, "r") as file:
+            content = file.read()
+            if f"message {message_name} " in content or f"message {message_name}{{" in content or f"message {message_name}\n" in content:
+                matches.append(f)
+                
+    if len(matches) == 0:
+        raise ValueError(f"Message '{message_name}' not found in any .proto file.")
+    elif len(matches) > 1:
+        basenames = [os.path.basename(m) for m in matches]
+        raise ValueError(f"Found message '{message_name}' in multiple files: {basenames}. Please use --file to specify.")
+        
+    return matches[0]
+
+def add_subfield(message_name: str, field_name: str, field_type: str, file_name: str = None):
+    """Injects a new field into an existing proto message."""
+    target_file = _find_file_for_message(message_name, file_name)
+    
+    with open(target_file, "r") as f:
+        lines = f.readlines()
+        
+    in_message = False
+    brace_count = 0
+    max_tag = 0
+    insert_idx = -1
+    
+    for i, line in enumerate(lines):
+        if re.match(rf"^message\s+{message_name}\s*{{", line) or re.match(rf"^message\s+{message_name}\n", line):
+            in_message = True
+            brace_count += line.count('{') - line.count('}')
+            continue
+            
+        if in_message:
+            brace_count += line.count('{') - line.count('}')
+            if brace_count == 0 and '}' in line:
+                insert_idx = i
+                break
+                
+            # Only match fields at the main message level (brace_count == 1)
+            if brace_count == 1:
+                if re.search(rf"\s+{field_name}\s*=", line):
+                    raise ValueError(f"Field '{field_name}' already exists in message '{message_name}'.")
+                    
+                match = re.search(r"=\s*(\d+)\s*;", line)
+                if match:
+                    tag = int(match.group(1))
+                    if tag > max_tag:
+                        max_tag = tag
+                    
+    if insert_idx == -1:
+        raise ValueError(f"Could not find the end of message '{message_name}'. Check syntax in {target_file}.")
+        
+    new_tag = max_tag + 1
+    new_field = f"    {field_type} {field_name} = {new_tag};\n"
+    
+    lines.insert(insert_idx, new_field)
+    
+    with open(target_file, "w") as f:
+        f.writelines(lines)
+        
+    print(f"Added field '{field_name}' (Tag {new_tag}) to '{message_name}' in {os.path.basename(target_file)}.")
+    build()
+
+def deprecate_subfield(message_name: str, field_name: str, file_name: str = None):
+    """Marks a specific field in a sub-message as deprecated."""
+    target_file = _find_file_for_message(message_name, file_name)
+    
+    with open(target_file, "r") as f:
+        lines = f.readlines()
+        
+    in_message = False
+    brace_count = 0
+    found = False
+    
+    for i, line in enumerate(lines):
+        if re.match(rf"^message\s+{message_name}\s*{{", line) or re.match(rf"^message\s+{message_name}\n", line):
+            in_message = True
+            brace_count += line.count('{') - line.count('}')
+            continue
+            
+        if in_message:
+            brace_count += line.count('{') - line.count('}')
+            if brace_count == 0 and '}' in line:
+                break
+                
+            if brace_count == 1:
+                if re.search(rf"\s+{field_name}\s*=", line):
+                    found = True
+                    if "[deprecated" in line.lower():
+                        print(f"Field '{field_name}' is already deprecated.")
+                        return
+                    lines[i] = re.sub(r";", " [deprecated = true];", line, count=1)
+                    break
+                
+    if not found:
+        raise ValueError(f"Field '{field_name}' not found in message '{message_name}'.")
+        
+    with open(target_file, "w") as f:
+        f.writelines(lines)
+        
+    print(f"Marked field '{field_name}' in '{message_name}' as deprecated in {os.path.basename(target_file)}.")
+    build()