k8s-helper-cli 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

k8s_helper/core.py

@@ -1679,3 +1679,948 @@ class K8sClient:
         
         print(f"❌ Timeout waiting for deployment '{name}' to be ready")
         return False
+
+    # ======================
+    # MONITORING OPERATIONS
+    # ======================
+    def setup_monitoring(self, namespace: str = "monitoring", 
+                        grafana_service_type: str = "NodePort",
+                        import_dashboard: bool = True,
+                        wait_for_ready: bool = True) -> Dict[str, Any]:
+        """Setup complete monitoring stack with Prometheus and Grafana
+        
+        Args:
+            namespace: Namespace to deploy monitoring stack
+            grafana_service_type: Service type for Grafana (NodePort, LoadBalancer, ClusterIP)
+            import_dashboard: Whether to import default Kubernetes dashboard
+            wait_for_ready: Whether to wait for deployments to be ready
+            
+        Returns:
+            Dictionary with deployment info, URLs, and credentials
+        """
+        result = {
+            'namespace': namespace,
+            'prometheus': {},
+            'grafana': {},
+            'success': False,
+            'error': None
+        }
+        
+        try:
+            # Create monitoring namespace
+            self._create_monitoring_namespace(namespace)
+            
+            # Deploy Prometheus
+            prometheus_result = self._deploy_prometheus(namespace)
+            result['prometheus'] = prometheus_result
+            
+            # Deploy Grafana
+            grafana_result = self._deploy_grafana(namespace, grafana_service_type)
+            result['grafana'] = grafana_result
+            
+            if wait_for_ready:
+                # Wait for deployments to be ready
+                if self._wait_for_monitoring_ready(namespace):
+                    # Configure Grafana data source and dashboard
+                    if self._configure_grafana(namespace, import_dashboard):
+                        result['success'] = True
+                    else:
+                        result['error'] = "Failed to configure Grafana"
+                else:
+                    result['error'] = "Monitoring deployments failed to become ready"
+            else:
+                result['success'] = True
+            
+            return result
+            
+        except Exception as e:
+            result['error'] = str(e)
+            return result
+
+    def _create_monitoring_namespace(self, namespace: str) -> bool:
+        """Create monitoring namespace if it doesn't exist"""
+        try:
+            self.core_v1.read_namespace(name=namespace)
+            print(f"✅ Namespace '{namespace}' already exists")
+            return True
+        except ApiException as e:
+            if e.status == 404:
+                # Create namespace
+                namespace_obj = client.V1Namespace(
+                    metadata=client.V1ObjectMeta(name=namespace)
+                )
+                self.core_v1.create_namespace(body=namespace_obj)
+                print(f"✅ Created namespace '{namespace}'")
+                return True
+            else:
+                raise e
+
+    def _deploy_prometheus(self, namespace: str) -> Dict[str, Any]:
+        """Deploy Prometheus to the cluster"""
+        result = {'deployed': False, 'service_name': 'prometheus-service'}
+        
+        # Prometheus ConfigMap
+        prometheus_config = """
+global:
+  scrape_interval: 15s
+scrape_configs:
+  - job_name: 'kubernetes-apiservers'
+    kubernetes_sd_configs:
+    - role: endpoints
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+      action: keep
+      regex: default;kubernetes;https
+
+  - job_name: 'kubernetes-nodes'
+    kubernetes_sd_configs:
+    - role: node
+    scheme: https
+    tls_config:
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    relabel_configs:
+    - action: labelmap
+      regex: __meta_kubernetes_node_label_(.+)
+
+  - job_name: 'kubernetes-pods'
+    kubernetes_sd_configs:
+    - role: pod
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      action: keep
+      regex: true
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      action: replace
+      target_label: __metrics_path__
+      regex: (.+)
+    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      action: replace
+      regex: ([^:]+)(?::\\d+)?;(\\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - source_labels: [__meta_kubernetes_namespace]
+      action: replace
+      target_label: kubernetes_namespace
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: kubernetes_pod_name
+"""
+        
+        # Create ConfigMap
+        configmap = client.V1ConfigMap(
+            metadata=client.V1ObjectMeta(name="prometheus-config", namespace=namespace),
+            data={"prometheus.yml": prometheus_config}
+        )
+        
+        try:
+            self.core_v1.create_namespaced_config_map(namespace=namespace, body=configmap)
+        except ApiException as e:
+            if e.status == 409:  # Already exists
+                self.core_v1.replace_namespaced_config_map(
+                    name="prometheus-config", namespace=namespace, body=configmap
+                )
+        
+        # Create ServiceAccount and RBAC
+        self._create_prometheus_rbac(namespace)
+        
+        # Prometheus Deployment
+        deployment = client.V1Deployment(
+            metadata=client.V1ObjectMeta(name="prometheus", namespace=namespace),
+            spec=client.V1DeploymentSpec(
+                replicas=1,
+                selector=client.V1LabelSelector(match_labels={"app": "prometheus"}),
+                template=client.V1PodTemplateSpec(
+                    metadata=client.V1ObjectMeta(labels={"app": "prometheus"}),
+                    spec=client.V1PodSpec(
+                        service_account_name="prometheus",
+                        containers=[
+                            client.V1Container(
+                                name="prometheus",
+                                image="prom/prometheus:latest",
+                                ports=[client.V1ContainerPort(container_port=9090)],
+                                args=[
+                                    "--config.file=/etc/prometheus/prometheus.yml",
+                                    "--storage.tsdb.path=/prometheus/",
+                                    "--web.console.libraries=/etc/prometheus/console_libraries",
+                                    "--web.console.templates=/etc/prometheus/consoles",
+                                    "--web.enable-lifecycle"
+                                ],
+                                volume_mounts=[
+                                    client.V1VolumeMount(
+                                        name="prometheus-config",
+                                        mount_path="/etc/prometheus/"
+                                    ),
+                                    client.V1VolumeMount(
+                                        name="prometheus-storage",
+                                        mount_path="/prometheus/"
+                                    )
+                                ]
+                            )
+                        ],
+                        volumes=[
+                            client.V1Volume(
+                                name="prometheus-config",
+                                config_map=client.V1ConfigMapVolumeSource(
+                                    name="prometheus-config"
+                                )
+                            ),
+                            client.V1Volume(
+                                name="prometheus-storage",
+                                empty_dir=client.V1EmptyDirVolumeSource()
+                            )
+                        ]
+                    )
+                )
+            )
+        )
+        
+        try:
+            self.apps_v1.create_namespaced_deployment(namespace=namespace, body=deployment)
+        except ApiException as e:
+            if e.status == 409:  # Already exists
+                self.apps_v1.replace_namespaced_deployment(
+                    name="prometheus", namespace=namespace, body=deployment
+                )
+        
+        # Prometheus Service
+        service = client.V1Service(
+            metadata=client.V1ObjectMeta(name="prometheus-service", namespace=namespace),
+            spec=client.V1ServiceSpec(
+                selector={"app": "prometheus"},
+                ports=[client.V1ServicePort(port=9090, target_port=9090)],
+                type="ClusterIP"
+            )
+        )
+        
+        try:
+            self.core_v1.create_namespaced_service(namespace=namespace, body=service)
+        except ApiException as e:
+            if e.status == 409:  # Already exists
+                self.core_v1.replace_namespaced_service(
+                    name="prometheus-service", namespace=namespace, body=service
+                )
+        
+        result['deployed'] = True
+        return result
+
+    def _create_prometheus_rbac(self, namespace: str) -> None:
+        """Create RBAC resources for Prometheus"""
+        # ServiceAccount
+        service_account = client.V1ServiceAccount(
+            metadata=client.V1ObjectMeta(name="prometheus", namespace=namespace)
+        )
+        
+        try:
+            self.core_v1.create_namespaced_service_account(namespace=namespace, body=service_account)
+        except ApiException as e:
+            if e.status != 409:  # Ignore if already exists
+                raise e
+        
+        # ClusterRole
+        rbac_v1 = client.RbacAuthorizationV1Api()
+        cluster_role = client.V1ClusterRole(
+            metadata=client.V1ObjectMeta(name="prometheus"),
+            rules=[
+                client.V1PolicyRule(
+                    api_groups=[""],
+                    resources=["nodes", "nodes/proxy", "services", "endpoints", "pods"],
+                    verbs=["get", "list", "watch"]
+                ),
+                client.V1PolicyRule(
+                    api_groups=["extensions"],
+                    resources=["ingresses"],
+                    verbs=["get", "list", "watch"]
+                ),
+                client.V1PolicyRule(
+                    non_resource_urls=["/metrics"],
+                    verbs=["get"]
+                )
+            ]
+        )
+        
+        try:
+            rbac_v1.create_cluster_role(body=cluster_role)
+        except ApiException as e:
+            if e.status != 409:  # Ignore if already exists
+                raise e
+        
+        # ClusterRoleBinding
+        cluster_role_binding = client.V1ClusterRoleBinding(
+            metadata=client.V1ObjectMeta(name="prometheus"),
+            subjects=[
+                client.V1Subject(
+                    kind="ServiceAccount",
+                    name="prometheus",
+                    namespace=namespace
+                )
+            ],
+            role_ref=client.V1RoleRef(
+                kind="ClusterRole",
+                name="prometheus",
+                api_group="rbac.authorization.k8s.io"
+            )
+        )
+        
+        try:
+            rbac_v1.create_cluster_role_binding(body=cluster_role_binding)
+        except ApiException as e:
+            if e.status != 409:  # Ignore if already exists
+                raise e
+
+    def _deploy_grafana(self, namespace: str, service_type: str = "NodePort") -> Dict[str, Any]:
+        """Deploy Grafana to the cluster"""
+        result = {
+            'deployed': False,
+            'service_name': 'grafana-service',
+            'service_type': service_type,
+            'admin_user': 'admin',
+            'admin_password': 'admin123'
+        }
+        
+        # Grafana Deployment
+        deployment = client.V1Deployment(
+            metadata=client.V1ObjectMeta(name="grafana", namespace=namespace),
+            spec=client.V1DeploymentSpec(
+                replicas=1,
+                selector=client.V1LabelSelector(match_labels={"app": "grafana"}),
+                template=client.V1PodTemplateSpec(
+                    metadata=client.V1ObjectMeta(labels={"app": "grafana"}),
+                    spec=client.V1PodSpec(
+                        containers=[
+                            client.V1Container(
+                                name="grafana",
+                                image="grafana/grafana:latest",
+                                ports=[client.V1ContainerPort(container_port=3000)],
+                                env=[
+                                    client.V1EnvVar(name="GF_SECURITY_ADMIN_USER", value="admin"),
+                                    client.V1EnvVar(name="GF_SECURITY_ADMIN_PASSWORD", value="admin123"),
+                                    client.V1EnvVar(name="GF_INSTALL_PLUGINS", value="grafana-kubernetes-app")
+                                ],
+                                volume_mounts=[
+                                    client.V1VolumeMount(
+                                        name="grafana-storage",
+                                        mount_path="/var/lib/grafana"
+                                    )
+                                ]
+                            )
+                        ],
+                        volumes=[
+                            client.V1Volume(
+                                name="grafana-storage",
+                                empty_dir=client.V1EmptyDirVolumeSource()
+                            )
+                        ]
+                    )
+                )
+            )
+        )
+        
+        try:
+            self.apps_v1.create_namespaced_deployment(namespace=namespace, body=deployment)
+        except ApiException as e:
+            if e.status == 409:  # Already exists
+                self.apps_v1.replace_namespaced_deployment(
+                    name="grafana", namespace=namespace, body=deployment
+                )
+        
+        # Grafana Service
+        service_ports = [client.V1ServicePort(port=3000, target_port=3000)]
+        if service_type == "NodePort":
+            service_ports[0].node_port = 30300
+        
+        service = client.V1Service(
+            metadata=client.V1ObjectMeta(name="grafana-service", namespace=namespace),
+            spec=client.V1ServiceSpec(
+                selector={"app": "grafana"},
+                ports=service_ports,
+                type=service_type
+            )
+        )
+        
+        try:
+            self.core_v1.create_namespaced_service(namespace=namespace, body=service)
+            result['deployed'] = True
+        except ApiException as e:
+            if e.status == 409:  # Already exists
+                self.core_v1.replace_namespaced_service(
+                    name="grafana-service", namespace=namespace, body=service
+                )
+                result['deployed'] = True
+        
+        return result
+
+    def _wait_for_monitoring_ready(self, namespace: str, timeout: int = 300) -> bool:
+        """Wait for Prometheus and Grafana to be ready"""
+        deployments = ["prometheus", "grafana"]
+        start_time = time.time()
+        
+        for deployment_name in deployments:
+            print(f"⏳ Waiting for {deployment_name} to be ready...")
+            deployment_ready = False
+            
+            while time.time() - start_time < timeout and not deployment_ready:
+                try:
+                    deployment = self.apps_v1.read_namespaced_deployment(
+                        name=deployment_name, namespace=namespace
+                    )
+                    
+                    if (deployment.status.ready_replicas == deployment.spec.replicas and 
+                        deployment.status.ready_replicas > 0):
+                        print(f"✅ {deployment_name} is ready")
+                        deployment_ready = True
+                    else:
+                        ready = deployment.status.ready_replicas or 0
+                        total = deployment.spec.replicas
+                        print(f"⏳ {deployment_name}: {ready}/{total} replicas ready...")
+                        time.sleep(5)
+                        
+                except ApiException as e:
+                    print(f"❌ Error checking {deployment_name} status: {e}")
+                    return False
+            
+            if not deployment_ready:
+                print(f"❌ Timeout waiting for {deployment_name} to be ready")
+                return False
+        
+        return True
+
+    def _configure_grafana(self, namespace: str, import_dashboard: bool = True) -> bool:
+        """Configure Grafana with Prometheus data source and dashboard"""
+        try:
+            # Wait a bit for Grafana to fully start
+            time.sleep(10)
+            
+            import requests
+            import json
+            
+            # Get Grafana service URL
+            grafana_url = self._get_grafana_url(namespace)
+            if not grafana_url:
+                print("❌ Could not determine Grafana URL")
+                return False
+            
+            print(f"🔧 Configuring Grafana at {grafana_url}")
+            
+            # Add Prometheus data source
+            datasource_payload = {
+                "name": "Prometheus",
+                "type": "prometheus",
+                "url": f"http://prometheus-service.{namespace}.svc.cluster.local:9090",
+                "access": "proxy",
+                "isDefault": True
+            }
+            
+            auth = ('admin', 'admin123')
+            
+            # Add data source
+            response = requests.post(
+                f"{grafana_url}/api/datasources",
+                json=datasource_payload,
+                auth=auth,
+                timeout=30
+            )
+            
+            if response.status_code in [200, 409]:  # Success or already exists
+                print("✅ Prometheus data source configured")
+            else:
+                print(f"⚠️  Warning: Could not add Prometheus data source: {response.text}")
+            
+            # Import default dashboard if requested
+            if import_dashboard:
+                self._import_kubernetes_dashboard(grafana_url, auth)
+            
+            return True
+            
+        except Exception as e:
+            print(f"⚠️  Warning: Could not configure Grafana automatically: {e}")
+            print("💡 You can manually add Prometheus as a data source in Grafana")
+            return False
+
+    def _get_grafana_url(self, namespace: str) -> Optional[str]:
+        """Get Grafana service URL"""
+        try:
+            service = self.core_v1.read_namespaced_service(
+                name="grafana-service", namespace=namespace
+            )
+            
+            if service.spec.type == "NodePort":
+                # Try to get node IP
+                nodes = self.core_v1.list_node()
+                if nodes.items:
+                    # Get external IP or internal IP
+                    node_ip = None
+                    for node in nodes.items:
+                        for address in node.status.addresses:
+                            if address.type == "ExternalIP":
+                                node_ip = address.address
+                                break
+                        if not node_ip:
+                            for address in node.status.addresses:
+                                if address.type == "InternalIP":
+                                    node_ip = address.address
+                                    break
+                        if node_ip:
+                            break
+                    
+                    if node_ip:
+                        node_port = None
+                        for port in service.spec.ports:
+                            if port.node_port:
+                                node_port = port.node_port
+                                break
+                        
+                        if node_port:
+                            return f"http://{node_ip}:{node_port}"
+            
+            elif service.spec.type == "LoadBalancer":
+                if service.status.load_balancer.ingress:
+                    ingress = service.status.load_balancer.ingress[0]
+                    host = ingress.ip or ingress.hostname
+                    if host:
+                        return f"http://{host}:3000"
+            
+            # Fallback to port-forward approach
+            return None
+            
+        except Exception as e:
+            print(f"Warning: Could not determine Grafana URL: {e}")
+            return None
+
+    def _import_kubernetes_dashboard(self, grafana_url: str, auth: tuple) -> None:
+        """Import a default Kubernetes dashboard"""
+        try:
+            import requests
+            
+            # Simple Kubernetes cluster dashboard JSON
+            dashboard_json = {
+                "dashboard": {
+                    "id": None,
+                    "title": "Kubernetes Cluster Overview",
+                    "tags": ["kubernetes"],
+                    "timezone": "browser",
+                    "panels": [
+                        {
+                            "id": 1,
+                            "title": "Cluster CPU Usage",
+                            "type": "stat",
+                            "targets": [
+                                {
+                                    "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+                                    "refId": "A"
+                                }
+                            ],
+                            "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
+                        },
+                        {
+                            "id": 2,
+                            "title": "Cluster Memory Usage",
+                            "type": "stat",
+                            "targets": [
+                                {
+                                    "expr": "100 * (1 - ((avg_over_time(node_memory_MemFree_bytes[10m]) + avg_over_time(node_memory_Cached_bytes[10m]) + avg_over_time(node_memory_Buffers_bytes[10m])) / avg_over_time(node_memory_MemTotal_bytes[10m])))",
+                                    "refId": "A"
+                                }
+                            ],
+                            "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
+                        },
+                        {
+                            "id": 3,
+                            "title": "Pod Count",
+                            "type": "stat",
+                            "targets": [
+                                {
+                                    "expr": "sum(kube_pod_info)",
+                                    "refId": "A"
+                                }
+                            ],
+                            "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
+                        }
+                    ],
+                    "time": {"from": "now-1h", "to": "now"},
+                    "refresh": "30s"
+                },
+                "overwrite": True
+            }
+            
+            response = requests.post(
+                f"{grafana_url}/api/dashboards/db",
+                json=dashboard_json,
+                auth=auth,
+                timeout=30
+            )
+            
+            if response.status_code == 200:
+                print("✅ Kubernetes dashboard imported")
+            else:
+                print(f"⚠️  Warning: Could not import dashboard: {response.text}")
+                
+        except Exception as e:
+            print(f"⚠️  Warning: Could not import dashboard: {e}")
+
+    def get_monitoring_info(self, namespace: str = "monitoring") -> Dict[str, Any]:
+        """Get monitoring stack information and URLs"""
+        result = {
+            'namespace': namespace,
+            'prometheus': {'running': False, 'url': None},
+            'grafana': {'running': False, 'url': None, 'credentials': None},
+            'services': []
+        }
+        
+        try:
+            # Check if deployments are running
+            deployments = self.apps_v1.list_namespaced_deployment(namespace=namespace)
+            
+            for deployment in deployments.items:
+                if deployment.metadata.name == "prometheus":
+                    result['prometheus']['running'] = (
+                        deployment.status.ready_replicas == deployment.spec.replicas and
+                        deployment.status.ready_replicas > 0
+                    )
+                elif deployment.metadata.name == "grafana":
+                    result['grafana']['running'] = (
+                        deployment.status.ready_replicas == deployment.spec.replicas and
+                        deployment.status.ready_replicas > 0
+                    )
+            
+            # Get service information
+            services = self.core_v1.list_namespaced_service(namespace=namespace)
+            
+            for service in services.items:
+                service_info = {
+                    'name': service.metadata.name,
+                    'type': service.spec.type,
+                    'cluster_ip': service.spec.cluster_ip,
+                    'ports': []
+                }
+                
+                for port in service.spec.ports:
+                    port_info = {
+                        'port': port.port,
+                        'target_port': port.target_port,
+                        'protocol': port.protocol
+                    }
+                    if hasattr(port, 'node_port') and port.node_port:
+                        port_info['node_port'] = port.node_port
+                    service_info['ports'].append(port_info)
+                
+                result['services'].append(service_info)
+                
+                # Set URLs
+                if service.metadata.name == "grafana-service":
+                    result['grafana']['credentials'] = {
+                        'username': 'admin',
+                        'password': 'admin123'
+                    }
+                    
+                    if service.spec.type == "NodePort":
+                        # Get node IP and port
+                        url_info = self.get_service_url("grafana-service", namespace)
+                        if url_info and url_info.get('external_url'):
+                            result['grafana']['url'] = url_info['external_url']
+                
+                elif service.metadata.name == "prometheus-service":
+                    if service.spec.type == "NodePort":
+                        url_info = self.get_service_url("prometheus-service", namespace)
+                        if url_info and url_info.get('external_url'):
+                            result['prometheus']['url'] = url_info['external_url']
+            
+            return result
+            
+        except ApiException as e:
+            result['error'] = str(e)
+            return result
+
+    def add_prometheus_target(self, job_name: str, targets: List[str], 
+                            namespace: str = "monitoring", 
+                            metrics_path: str = "/metrics",
+                            scrape_interval: str = "15s") -> bool:
+        """
+        Add a new scrape target to Prometheus configuration
+        
+        Args:
+            job_name: Name for the job in Prometheus config
+            targets: List of target addresses (e.g., ["service:port", "1.2.3.4:9090"])
+            namespace: Monitoring namespace
+            metrics_path: Path to metrics endpoint
+            scrape_interval: How often to scrape the target
+        
+        Returns:
+            bool: True if target was added successfully
+        """
+        try:
+            # Get current ConfigMap
+            configmap = self.core_v1.read_namespaced_config_map(
+                name="prometheus-config", 
+                namespace=namespace
+            )
+            
+            current_config = configmap.data.get("prometheus.yml", "")
+            
+            # Parse YAML to add new target
+            import yaml
+            config_data = yaml.safe_load(current_config)
+            
+            # Check if job already exists
+            existing_jobs = [job['job_name'] for job in config_data.get('scrape_configs', [])]
+            if job_name in existing_jobs:
+                print(f"❌ Job '{job_name}' already exists. Use update or remove it first.")
+                return False
+            
+            # Create new job config
+            new_job = {
+                'job_name': job_name,
+                'scrape_interval': scrape_interval,
+                'metrics_path': metrics_path,
+                'static_configs': [
+                    {'targets': targets}
+                ]
+            }
+            
+            # Add to scrape configs
+            if 'scrape_configs' not in config_data:
+                config_data['scrape_configs'] = []
+            
+            config_data['scrape_configs'].append(new_job)
+            
+            # Convert back to YAML
+            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
+            
+            # Update ConfigMap
+            configmap.data["prometheus.yml"] = updated_config
+            self.core_v1.replace_namespaced_config_map(
+                name="prometheus-config",
+                namespace=namespace,
+                body=configmap
+            )
+            
+            # Restart Prometheus deployment to pick up new config
+            self._restart_prometheus_deployment(namespace)
+            
+            print(f"✅ Added Prometheus target job '{job_name}' with targets: {targets}")
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error adding Prometheus target: {e}")
+            return False
+
+    def remove_prometheus_target(self, job_name: str, namespace: str = "monitoring") -> bool:
+        """
+        Remove a scrape target from Prometheus configuration
+        
+        Args:
+            job_name: Name of the job to remove
+            namespace: Monitoring namespace
+        
+        Returns:
+            bool: True if target was removed successfully
+        """
+        try:
+            # Get current ConfigMap
+            configmap = self.core_v1.read_namespaced_config_map(
+                name="prometheus-config", 
+                namespace=namespace
+            )
+            
+            current_config = configmap.data.get("prometheus.yml", "")
+            
+            # Parse YAML to remove target
+            import yaml
+            config_data = yaml.safe_load(current_config)
+            
+            # Find and remove the job
+            original_jobs = config_data.get('scrape_configs', [])
+            updated_jobs = [job for job in original_jobs if job.get('job_name') != job_name]
+            
+            if len(original_jobs) == len(updated_jobs):
+                print(f"❌ Job '{job_name}' not found in Prometheus configuration")
+                return False
+            
+            config_data['scrape_configs'] = updated_jobs
+            
+            # Convert back to YAML
+            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
+            
+            # Update ConfigMap
+            configmap.data["prometheus.yml"] = updated_config
+            self.core_v1.replace_namespaced_config_map(
+                name="prometheus-config",
+                namespace=namespace,
+                body=configmap
+            )
+            
+            # Restart Prometheus deployment to pick up new config
+            self._restart_prometheus_deployment(namespace)
+            
+            print(f"✅ Removed Prometheus target job '{job_name}'")
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error removing Prometheus target: {e}")
+            return False
+
+    def list_prometheus_targets(self, namespace: str = "monitoring") -> Dict[str, Any]:
+        """
+        List all configured Prometheus targets
+        
+        Args:
+            namespace: Monitoring namespace
+        
+        Returns:
+            Dict containing target information
+        """
+        try:
+            # Get current ConfigMap
+            configmap = self.core_v1.read_namespaced_config_map(
+                name="prometheus-config", 
+                namespace=namespace
+            )
+            
+            current_config = configmap.data.get("prometheus.yml", "")
+            
+            # Parse YAML to get targets
+            import yaml
+            config_data = yaml.safe_load(current_config)
+            
+            targets = []
+            for job in config_data.get('scrape_configs', []):
+                job_info = {
+                    'job_name': job.get('job_name', 'unknown'),
+                    'scrape_interval': job.get('scrape_interval', 'default'),
+                    'metrics_path': job.get('metrics_path', '/metrics'),
+                    'targets': []
+                }
+                
+                # Get static targets
+                if 'static_configs' in job:
+                    for static_config in job['static_configs']:
+                        job_info['targets'].extend(static_config.get('targets', []))
+                
+                # Mark kubernetes discovery jobs
+                if 'kubernetes_sd_configs' in job:
+                    job_info['type'] = 'kubernetes_discovery'
+                    job_info['targets'] = ['<kubernetes_discovery>']
+                else:
+                    job_info['type'] = 'static'
+                
+                targets.append(job_info)
+            
+            return {
+                'namespace': namespace,
+                'targets': targets,
+                'total_jobs': len(targets)
+            }
+            
+        except Exception as e:
+            return {'error': str(e)}
+
+    def update_prometheus_target(self, job_name: str, targets: List[str],
+                               namespace: str = "monitoring",
+                               metrics_path: str = None,
+                               scrape_interval: str = None) -> bool:
+        """
+        Update an existing Prometheus target
+        
+        Args:
+            job_name: Name of the job to update
+            targets: New list of target addresses
+            namespace: Monitoring namespace
+            metrics_path: Optional new metrics path
+            scrape_interval: Optional new scrape interval
+        
+        Returns:
+            bool: True if target was updated successfully
+        """
+        try:
+            # Get current ConfigMap
+            configmap = self.core_v1.read_namespaced_config_map(
+                name="prometheus-config", 
+                namespace=namespace
+            )
+            
+            current_config = configmap.data.get("prometheus.yml", "")
+            
+            # Parse YAML to update target
+            import yaml
+            config_data = yaml.safe_load(current_config)
+            
+            # Find and update the job
+            job_found = False
+            for job in config_data.get('scrape_configs', []):
+                if job.get('job_name') == job_name:
+                    job_found = True
+                    
+                    # Update targets
+                    if 'static_configs' not in job:
+                        job['static_configs'] = [{}]
+                    
+                    job['static_configs'][0]['targets'] = targets
+                    
+                    # Update optional fields
+                    if metrics_path:
+                        job['metrics_path'] = metrics_path
+                    if scrape_interval:
+                        job['scrape_interval'] = scrape_interval
+                    
+                    break
+            
+            if not job_found:
+                print(f"❌ Job '{job_name}' not found in Prometheus configuration")
+                return False
+            
+            # Convert back to YAML
+            updated_config = yaml.dump(config_data, default_flow_style=False, sort_keys=False)
+            
+            # Update ConfigMap
+            configmap.data["prometheus.yml"] = updated_config
+            self.core_v1.replace_namespaced_config_map(
+                name="prometheus-config",
+                namespace=namespace,
+                body=configmap
+            )
+            
+            # Restart Prometheus deployment to pick up new config
+            self._restart_prometheus_deployment(namespace)
+            
+            print(f"✅ Updated Prometheus target job '{job_name}' with targets: {targets}")
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error updating Prometheus target: {e}")
+            return False
+
+    def _restart_prometheus_deployment(self, namespace: str) -> bool:
+        """Restart Prometheus deployment to reload configuration"""
+        try:
+            # Get current deployment
+            deployment = self.apps_v1.read_namespaced_deployment(
+                name="prometheus", 
+                namespace=namespace
+            )
+            
+            # Update deployment to trigger restart
+            if deployment.spec.template.metadata.annotations is None:
+                deployment.spec.template.metadata.annotations = {}
+            
+            import time
+            deployment.spec.template.metadata.annotations['kubectl.kubernetes.io/restartedAt'] = str(int(time.time()))
+            
+            # Apply the update
+            self.apps_v1.patch_namespaced_deployment(
+                name="prometheus",
+                namespace=namespace,
+                body=deployment
+            )
+            
+            print("🔄 Restarting Prometheus deployment to reload configuration...")
+            return True
+            
+        except Exception as e:
+            print(f"⚠️  Could not restart Prometheus deployment: {e}")
+            return False