runbooks 0.2.5__py3-none-any.whl → 0.6.1__py3-none-any.whl

jupyter-agent/utils.py

@@ -0,0 +1,409 @@
+"""
+utils.py
+
+This module provides helper functions to:
+- Generate and update Jupyter Notebook structures.
+- Execute code in a sandbox environment.
+- Parse and convert execution results to notebook cell outputs.
+- Export notebooks to HTML.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+import nbformat
+from e2b_code_interpreter import Sandbox
+from huggingface_hub import InferenceClient
+from nbconvert import HTMLExporter
+from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook
+from traitlets.config import Config
+from transformers import AutoTokenizer
+
+## --- Global Configuration and Template Loading ---
+
+## Create a configuration for nbconvert and set up the HTML exporter
+config = Config()
+html_exporter = HTMLExporter(config=config, template_name="classic")
+
+## Load the Jinja template for the LLaMA model
+TEMPLATE_PATH = Path("llama3_template.jinja")
+try:
+    with TEMPLATE_PATH.open("r", encoding="utf-8") as f:
+        llama_template = f.read()
+except FileNotFoundError:
+    raise FileNotFoundError(f"Template file {TEMPLATE_PATH} not found.")
+
+MAX_TURNS = 4
+
+## --- Code Execution Functions ---
+
+
+def parse_exec_result_nb(execution: Any) -> List[Dict[str, Any]]:
+    """
+    Convert an E2B execution object to a list of Jupyter notebook cell outputs format.
+
+    :param execution: Execution object from the sandbox.
+    :return: List of output dictionaries.
+    """
+    outputs: List[Dict[str, Any]] = []
+
+    if execution.logs.stdout:
+        outputs.append(
+            {
+                "output_type": "stream",
+                "name": "stdout",
+                "text": "".join(execution.logs.stdout),
+            }
+        )
+
+    if execution.logs.stderr:
+        outputs.append(
+            {
+                "output_type": "stream",
+                "name": "stderr",
+                "text": "".join(execution.logs.stderr),
+            }
+        )
+
+    if execution.error:
+        outputs.append(
+            {
+                "output_type": "error",
+                "ename": execution.error.name,
+                "evalue": execution.error.value,
+                "traceback": [line for line in execution.error.traceback.split("\n")],
+            }
+        )
+
+    for result in execution.results:
+        output = {
+            "output_type": (
+                "execute_result" if result.is_main_result else "display_data"
+            ),
+            "metadata": {},
+            "data": {},
+        }
+
+        if result.text:
+            output["data"]["text/plain"] = [result.text]  # Array for text/plain
+        if result.html:
+            output["data"]["text/html"] = result.html
+        if result.png:
+            output["data"]["image/png"] = result.png
+        if result.svg:
+            output["data"]["image/svg+xml"] = result.svg
+        if result.jpeg:
+            output["data"]["image/jpeg"] = result.jpeg
+        if result.pdf:
+            output["data"]["application/pdf"] = result.pdf
+        if result.latex:
+            output["data"]["text/latex"] = result.latex
+        if result.json:
+            output["data"]["application/json"] = result.json
+        if result.javascript:
+            output["data"]["application/javascript"] = result.javascript
+
+        if result.is_main_result and execution.execution_count is not None:
+            output["execution_count"] = execution.execution_count
+
+        if output["data"]:
+            outputs.append(output)
+
+    return outputs
+
+
+## HTML and CSS templates for notebook cells
+system_template = """\
+<details>
+  <summary style="display: flex; align-items: center;">
+    <div class="alert alert-block alert-info" style="margin: 0; width: 100%;">
+      <b>System: <span class="arrow">▶</span></b>
+    </div>
+  </summary>
+  <div class="alert alert-block alert-info">
+    {}
+  </div>
+</details>
+
+<style>
+details > summary .arrow {{
+  display: inline-block;
+  transition: transform 0.2s;
+}}
+details[open] > summary .arrow {{
+  transform: rotate(90deg);
+}}
+</style>
+"""
+
+user_template = """<div class="alert alert-block alert-success">
+<b>User:</b> {}
+</div>
+"""
+
+header_message = """<p align="center">
+  <img src="cloudops-agent.png" alt="Jupyter Agent" />
+</p>
+
+
+<p style="text-align:center;">Let a LLM agent write and execute code inside a notebook!</p>"""
+
+bad_html_bad = """input[type="file"] {
+  display: block;
+}"""
+
+
+## --- Notebook Creation and Update Functions ---
+
+
+def create_base_notebook(messages: List[Dict[str, Any]]) -> Tuple[Dict[str, Any], int]:
+    """
+    Create the base Jupyter Notebook structure with initial cells.
+
+    :param messages: List of conversation messages.
+    :return: A tuple of the notebook data dictionary and the current code cell counter.
+    """
+    base_notebook = {
+        "metadata": {
+            "kernel_info": {"name": "python3"},
+            "language_info": {
+                "name": "python",
+                "version": "3.12",
+            },
+        },
+        "nbformat": 4,
+        "nbformat_minor": 0,
+        "cells": [],
+    }
+    base_notebook["cells"].append(
+        {"cell_type": "markdown", "metadata": {}, "source": header_message}
+    )
+
+    if len(messages) == 0:
+        base_notebook["cells"].append(
+            {
+                "cell_type": "code",
+                "execution_count": None,
+                "metadata": {},
+                "source": "",
+                "outputs": [],
+            }
+        )
+
+    code_cell_counter = 0
+
+    for message in messages:
+        if message["role"] == "system":
+            text = system_template.format(message["content"].replace("\n", "<br>"))
+            base_notebook["cells"].append(
+                {"cell_type": "markdown", "metadata": {}, "source": text}
+            )
+        elif message["role"] == "user":
+            text = user_template.format(message["content"].replace("\n", "<br>"))
+            base_notebook["cells"].append(
+                {"cell_type": "markdown", "metadata": {}, "source": text}
+            )
+
+        elif message["role"] == "assistant" and "tool_calls" in message:
+            base_notebook["cells"].append(
+                {
+                    "cell_type": "code",
+                    "execution_count": None,
+                    "metadata": {},
+                    "source": message["content"],
+                    "outputs": [],
+                }
+            )
+
+        elif message["role"] == "ipython":
+            code_cell_counter += 1
+            base_notebook["cells"][-1]["outputs"] = message["nbformat"]
+            base_notebook["cells"][-1]["execution_count"] = code_cell_counter
+
+        elif message["role"] == "assistant" and "tool_calls" not in message:
+            base_notebook["cells"].append(
+                {"cell_type": "markdown", "metadata": {}, "source": message["content"]}
+            )
+
+        else:
+            raise ValueError(message)
+
+    return base_notebook, code_cell_counter
+
+
+def execute_code(sbx: Sandbox, code: str) -> Tuple[str, Any]:
+    """
+    Execute the given code in the provided sandbox.
+
+    :param sbx: Sandbox instance to run the code.
+    :param code: Code to execute.
+    :return: Tuple of aggregated output string and the raw execution object.
+    """
+    execution = sbx.run_code(code, on_stdout=lambda data: print("stdout:", data))
+    output = ""
+    if len(execution.logs.stdout) > 0:
+        output += "\n".join(execution.logs.stdout)
+    if len(execution.logs.stderr) > 0:
+        output += "\n".join(execution.logs.stderr)
+    if execution.error is not None:
+        output += execution.error.traceback
+    return output, execution
+
+
+def parse_exec_result_llm(execution: Any) -> str:
+    """
+    Parse the execution results and return a single concatenated output string.
+
+    :param execution: Execution object from the sandbox.
+    :return: Concatenated string of output messages.
+    """
+    output = ""
+    if len(execution.logs.stdout) > 0:
+        output += "\n".join(execution.logs.stdout)
+    if len(execution.logs.stderr) > 0:
+        output += "\n".join(execution.logs.stderr)
+    if execution.error is not None:
+        output += execution.error.traceback
+    return output
+
+
+def update_notebook_display(notebook_data):
+    notebook = nbformat.from_dict(notebook_data)
+    notebook_body, _ = html_exporter.from_notebook_node(notebook)
+    notebook_body = notebook_body.replace(bad_html_bad, "")
+    return notebook_body
+
+
+## --- Interactive Notebook Generation ---
+
+
+def run_interactive_notebook(
+    client: InferenceClient,
+    model: str,
+    tokenizer: Any,
+    messages: List[Dict[str, Any]],
+    sbx: Sandbox,
+    max_new_tokens: int = 512,
+) -> Any:
+    """
+    Generator function that iteratively builds and updates the Jupyter Notebook.
+
+    :param client: Hugging Face InferenceClient for text generation.
+    :param model: Model identifier.
+    :param tokenizer: Tokenizer corresponding to the model.
+    :param messages: List of conversation messages.
+    :param sbx: Sandbox instance for executing code.
+    :param max_new_tokens: Maximum tokens to generate per turn.
+    :yield: Tuple containing updated notebook HTML, notebook data, and messages.
+    """
+    notebook_data, code_cell_counter = create_base_notebook(messages)
+    turns = 0
+
+    # code_cell_counter = 0
+    while turns <= MAX_TURNS:
+        turns += 1
+        input_tokens = tokenizer.apply_chat_template(
+            messages,
+            chat_template=llama_template,
+            builtin_tools=["code_interpreter"],
+            add_generation_prompt=True,
+        )
+        model_input = tokenizer.decode(input_tokens)
+
+        print(f"Model input:\n{model_input}\n{'='*80}")
+
+        response_stream = client.text_generation(
+            model=model,
+            prompt=model_input,
+            details=True,
+            stream=True,
+            do_sample=True,
+            repetition_penalty=1.1,
+            temperature=0.8,
+            max_new_tokens=max_new_tokens,
+        )
+
+        assistant_response = ""
+        tokens = []
+
+        code_cell = False
+        for i, chunk in enumerate(response_stream):
+            if not chunk.token.special:
+                content = chunk.token.text
+            else:
+                content = ""
+            tokens.append(chunk.token.text)
+            assistant_response += content
+
+            if len(tokens) == 1:
+                create_cell = True
+                code_cell = "<|python_tag|>" in tokens[0]
+                if code_cell:
+                    code_cell_counter += 1
+            else:
+                create_cell = False
+
+            ## Update notebook cells in real-time
+            if create_cell:
+                if "<|python_tag|>" in tokens[0]:
+                    notebook_data["cells"].append(
+                        {
+                            "cell_type": "code",
+                            "execution_count": None,
+                            "metadata": {},
+                            "source": assistant_response,
+                            "outputs": [],
+                        }
+                    )
+                else:
+                    notebook_data["cells"].append(
+                        {
+                            "cell_type": "markdown",
+                            "metadata": {},
+                            "source": assistant_response,
+                        }
+                    )
+            else:
+                notebook_data["cells"][-1]["source"] = assistant_response
+            if i % 16 == 0:
+                yield update_notebook_display(notebook_data), notebook_data, messages
+        yield update_notebook_display(notebook_data), notebook_data, messages
+
+        ## If a code cell was generated, execute the code
+        if code_cell:
+            notebook_data["cells"][-1]["execution_count"] = code_cell_counter
+
+            exec_result, execution = execute_code(sbx, assistant_response)
+            messages.append(
+                {
+                    "role": "assistant",
+                    "content": assistant_response,
+                    "tool_calls": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "code_interpreter",
+                                "arguments": {"code": assistant_response},
+                            },
+                        }
+                    ],
+                }
+            )
+            messages.append(
+                {
+                    "role": "ipython",
+                    "content": parse_exec_result_llm(execution),
+                    "nbformat": parse_exec_result_nb(execution),
+                }
+            )
+
+            ## Update the last code cell with execution results
+            notebook_data["cells"][-1]["outputs"] = parse_exec_result_nb(execution)
+            update_notebook_display(notebook_data)
+        else:
+            messages.append({"role": "assistant", "content": assistant_response})
+            if tokens[-1] == "<|eot_id|>":
+                break
+
+    yield update_notebook_display(notebook_data), notebook_data, messages

runbooks/__init__.py

@@ -1,7 +1,75 @@
 """
-runbooks
+CloudOps Runbooks - Enterprise CloudOps Automation Toolkit
 
-Provides utility functions for math operations, AWS S3 interactions, and data management.
+Provides comprehensive AWS automation capabilities including:
+- Cloud Foundations Assessment Tool (CFAT)
+- Multi-account resource inventory
+- Organization management
+- Control Tower automation
+- Identity and access management
+- Centralized logging setup
 """
 
-# __version__ = "0.2.5"
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+# Keep package version in sync with distribution metadata
+try:
+    __version__ = _pkg_version("runbooks")
+except Exception:
+    # Fallback if metadata is unavailable during editable installs
+    __version__ = "0.6.1"
+
+# Core module exports
+from runbooks.config import RunbooksConfig, load_config, save_config
+from runbooks.utils import ensure_directory, setup_logging, validate_aws_profile
+
+# Cloud Foundations exports - using direct structure
+try:
+    from runbooks.cfat.runner import AssessmentRunner
+    from runbooks.inventory.core.collector import InventoryCollector
+    from runbooks.organizations.manager import OUManager
+
+    __all__ = [
+        "__version__",
+        "setup_logging",
+        "load_config",
+        "save_config",
+        "RunbooksConfig",
+        "AssessmentRunner",
+        "InventoryCollector",
+        "OUManager",
+        "ensure_directory",
+        "validate_aws_profile",
+    ]
+except ImportError as e:
+    # Graceful degradation if dependencies aren't available
+    __all__ = [
+        "__version__",
+        "setup_logging",
+        "load_config",
+        "save_config",
+        "RunbooksConfig",
+        "ensure_directory",
+        "validate_aws_profile",
+    ]
+
+# FinOps exports
+from runbooks.finops import get_cost_data, get_trend, run_dashboard
+
+__all__ = [
+    "__version__",
+    "setup_logging",
+    "load_config",
+    "save_config",
+    "RunbooksConfig",
+    "AssessmentRunner",
+    "InventoryCollector",
+    "OUManager",
+    "ensure_directory",
+    "validate_aws_profile",
+    # FinOps
+    "run_dashboard",
+    "get_cost_data",
+    "get_trend",
+]

runbooks/__main__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+"""
+CloudOps Runbooks CLI Entry Point
+
+This module enables the package to be executed as:
+    python -m runbooks
+    uv run python -m runbooks
+"""
+
+from runbooks.main import main
+
+if __name__ == "__main__":
+    main()

runbooks/aws/ec2_describe_instances.py

@@ -125,7 +125,7 @@ def display_instances_json(instances: List[Dict[str, str]]) -> None:
     Args:
         instances (List[Dict[str, str]]): List of EC2 instance details.
     """
-    print(json.dumps(instances, indent=4)) if instances else print("No instances found.")
+    (print(json.dumps(instances, indent=4)) if instances else print("No instances found."))
 
 
 # ==============================

runbooks/aws/ec2_run_instances.py

@@ -135,7 +135,10 @@ def launch_ec2_instances(
 
         ## ✅ Apply Tags
         if tags:
-            ec2_client.create_tags(Resources=instance_ids, Tags=[{"Key": k, "Value": v} for k, v in tags.items()])
+            ec2_client.create_tags(
+                Resources=instance_ids,
+                Tags=[{"Key": k, "Value": v} for k, v in tags.items()],
+            )
             logger.info(f"Applied tags: {tags}")
 
         return instance_ids
@@ -182,7 +185,10 @@ def lambda_handler(event, context):
         )
 
         ## ✅ Return Success Response
-        return {"statusCode": 200, "body": json.dumps({"message": "Instances launched", "InstanceIDs": instance_ids})}
+        return {
+            "statusCode": 200,
+            "body": json.dumps({"message": "Instances launched", "InstanceIDs": instance_ids}),
+        }
     except Exception as e:
         logger.error(f"Lambda Handler Error: {e}")
         return {"statusCode": 500, "body": json.dumps({"error": str(e)})}

runbooks/aws/ec2_start_stop_instances.py

@@ -27,14 +27,19 @@ Usage (Lambda):
     Trigger event with: {"action": "start"} or {"action": "stop"}
 """
 
-import argparse  ## For CLI mode support
+import argparse  # # For CLI mode support
 import json
 import os
 import sys
 from typing import Dict, List
 
 import boto3
-from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError, PartialCredentialsError
+from botocore.exceptions import (
+    BotoCoreError,
+    ClientError,
+    NoCredentialsError,
+    PartialCredentialsError,
+)
 
 from runbooks.utils.logger import configure_logger
 
@@ -161,7 +166,10 @@ def lambda_handler(event, context):
         instance_ids = fetch_instances(ec2_client, tag_key="AutoStart", tag_value="True")
         perform_action(ec2_client, instance_ids, action)
 
-        return {"statusCode": 200, "body": json.dumps(f"Action '{action}' completed successfully.")}
+        return {
+            "statusCode": 200,
+            "body": json.dumps(f"Action '{action}' completed successfully."),
+        }
 
     except Exception as e:
         logger.error(f"Error: {e}")
@@ -173,7 +181,12 @@ def main():
     CLI Entry Point for Python Usage.
     """
     parser = argparse.ArgumentParser(description="EC2 Scheduler Script")
-    parser.add_argument("--action", choices=["start", "stop"], required=True, help="Action to perform (start/stop).")
+    parser.add_argument(
+        "--action",
+        choices=["start", "stop"],
+        required=True,
+        help="Action to perform (start/stop).",
+    )
     args = parser.parse_args()
 
     try:

runbooks/aws/ec2_unused_volumes.py

@@ -126,7 +126,11 @@ def send_sns_notification(unused_volumes: List[Dict[str, str]]) -> None:
         ## ✅ Publish to SNS
         logger.info(f"Sending notification to SNS topic: {SNS_TOPIC_ARN}...")
         logger.info(f"📤 Sending SNS notification to SNS topic: {SNS_TOPIC_ARN}")
-        sns_client.publish(TopicArn=SNS_TOPIC_ARN, Subject="Unused EBS Volumes Report", Message=email_body)
+        sns_client.publish(
+            TopicArn=SNS_TOPIC_ARN,
+            Subject="Unused EBS Volumes Report",
+            Message=email_body,
+        )
         logger.info("✅ SNS notification sent successfully.")
 
     except ClientError as e:

runbooks/aws/s3_create_bucket.py

@@ -14,7 +14,7 @@ from typing import Optional
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
 
-from runbooks.utils.logger import configure_logger  ## Import reusable logger
+from runbooks.utils.logger import configure_logger  # # Import reusable logger
 
 ## Initialize Logger
 logger = configure_logger("list_s3_buckets")
@@ -87,7 +87,9 @@ def create_s3_bucket(bucket_name: str, region: str) -> Optional[str]:
             )
         else:
             response = s3_client.create_bucket(
-                Bucket=bucket_name, ACL="private", CreateBucketConfiguration={"LocationConstraint": region}
+                Bucket=bucket_name,
+                ACL="private",
+                CreateBucketConfiguration={"LocationConstraint": region},
             )
 
         logger.info(f"✅ Bucket '{bucket_name}' created successfully at {response['Location']}.")

runbooks/aws/s3_list_objects.py

@@ -129,7 +129,12 @@ def main():
     parser = argparse.ArgumentParser(description="List objects in an AWS S3 bucket.")
     parser.add_argument("--bucket", required=True, help="The name of the S3 bucket.")
     parser.add_argument("--prefix", default=None, help="Filter objects by prefix.")
-    parser.add_argument("--max-keys", type=int, default=1000, help="Max number of keys to fetch (default: 1000).")
+    parser.add_argument(
+        "--max-keys",
+        type=int,
+        default=1000,
+        help="Max number of keys to fetch (default: 1000).",
+    )
 
     args = parser.parse_args()
 

runbooks/aws/tagging_lambda_handler.py

@@ -48,7 +48,15 @@ LOCAL_FILE_PATH = "/tmp/tags.json"  ## Local Temp file path
 # ==============================
 # VALIDATION CONFIGURATIONS
 # ==============================
-REQUIRED_TAGS = ["Account Name", "Functional Area", "WBS Code", "Business Unit", "Managed by", "CostGroup", "TechOwner"]
+REQUIRED_TAGS = [
+    "Account Name",
+    "Functional Area",
+    "WBS Code",
+    "Business Unit",
+    "Managed by",
+    "CostGroup",
+    "TechOwner",
+]
 
 TAG_VALUE_REGEX = r"^[a-zA-Z0-9\s\-_@]+$"  ## Allowed characters for tag values
 
@@ -166,7 +174,10 @@ def lambda_handler(event, context):
         apply_tags_to_instance(instance_id, tags)
 
         ## ✅ Success Response
-        return {"statusCode": 200, "body": json.dumps(f"Tags successfully applied to instance {instance_id}")}
+        return {
+            "statusCode": 200,
+            "body": json.dumps(f"Tags successfully applied to instance {instance_id}"),
+        }
     except Exception as e:
         logger.error(f"Error during tagging process: {e}")
         return {"statusCode": 500, "body": json.dumps(f"Error: {str(e)}")}

runbooks/aws/tags.json

@@ -0,0 +1,12 @@
+[
+    {"Key": "Environment", "Value": "Production"},
+    {"Key": "Project", "Value": "JHCApp"},
+    {"Key": "CostCenter", "Value": "JHC007"},
+    {"Key": "Account Name", "Value": "OceanSoft Platform"},
+    {"Key": "Functional Area", "Value": "Shared Services"},
+    {"Key": "WBS Code", "Value": "OS12345"},
+    {"Key": "Business Unit", "Value": "Platform"},
+    {"Key": "Managed by", "Value": "DevOps Team"},
+    {"Key": "CostGroup", "Value": "CloudOps"},
+    {"Key": "TechOwner", "Value": "tech@oceansoft.io"}
+]