Delete app.py

app.py

@@ -1,1973 +0,0 @@
-"""
-Enterprise Agentic Reliability Framework - Main Application (FIXED VERSION)
-Multi-Agent AI System for Production Reliability Monitoring
-
-CRITICAL FIXES APPLIED:
-- Removed event loop creation (uses Gradio native async)
-- Fixed FAISS thread safety with single-writer pattern
-- ProcessPoolExecutor for CPU-intensive encoding
-- Atomic saves with fsync
-- Dependency injection
-- Rate limiting
-- Comprehensive input validation
-- Circuit breakers for agent resilience
-"""
-
-import os
-import json
-import numpy as np
-import gradio as gr
-import requests
-import pandas as pd
-import datetime
-import threading
-import logging
-import asyncio
-import tempfile
-from typing import List, Dict, Any, Optional, Tuple
-from collections import deque, OrderedDict
-from dataclasses import dataclass, asdict
-from enum import Enum
-from concurrent.futures import ProcessPoolExecutor
-from queue import Queue
-from circuitbreaker import circuit
-import atomicwrites
-
-# Import our modules
-from models import (
-    ReliabilityEvent, EventSeverity, AnomalyResult,
-    HealingAction, ForecastResult, PolicyCondition
-)
-from claude_adapter import get_claude_adapter
-from healing_policies import PolicyEngine, DEFAULT_HEALING_POLICIES
-
-# === Logging Configuration ===
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger(__name__)
-# Initialize Claude adapter for AI reasoning
-claude_adapter = get_claude_adapter()
-
-
-# === CONSTANTS (FIXED: Extracted all magic numbers) ===
-class Constants:
-    """Centralized constants to eliminate magic numbers"""
-    
-    # Thresholds
-    LATENCY_WARNING = 150.0
-    LATENCY_CRITICAL = 300.0
-    LATENCY_EXTREME = 500.0
-    
-    ERROR_RATE_WARNING = 0.05
-    ERROR_RATE_HIGH = 0.15
-    ERROR_RATE_CRITICAL = 0.3
-    
-    CPU_WARNING = 0.8
-    CPU_CRITICAL = 0.9
-    
-    MEMORY_WARNING = 0.8
-    MEMORY_CRITICAL = 0.9
-    
-    # Forecasting
-    SLOPE_THRESHOLD_INCREASING = 5.0
-    SLOPE_THRESHOLD_DECREASING = -2.0
-    
-    FORECAST_MIN_DATA_POINTS = 5
-    FORECAST_LOOKAHEAD_MINUTES = 15
-    
-    # Performance
-    HISTORY_WINDOW = 50
-    MAX_EVENTS_STORED = 1000
-    AGENT_TIMEOUT_SECONDS = 5
-    CACHE_EXPIRY_MINUTES = 15
-    
-    # FAISS
-    FAISS_BATCH_SIZE = 10
-    FAISS_SAVE_INTERVAL_SECONDS = 30
-    VECTOR_DIM = 384
-    
-    # Business metrics
-    BASE_REVENUE_PER_MINUTE = 100.0
-    BASE_USERS = 1000
-    
-    # Rate limiting
-    MAX_REQUESTS_PER_MINUTE = 60
-    MAX_REQUESTS_PER_HOUR = 500
-
-
-# === Configuration ===
-class Config:
-    """Centralized configuration for the reliability framework"""
-    HF_TOKEN: str = os.getenv("HF_TOKEN", "").strip()
-    HF_API_URL: str = "https://router.huggingface.co/hf-inference/v1/completions"
-    
-    INDEX_FILE: str = os.getenv("INDEX_FILE", "data/incident_vectors.index")
-    TEXTS_FILE: str = os.getenv("TEXTS_FILE", "data/incident_texts.json")
-    DATA_DIR: str = os.getenv("DATA_DIR", "data")
-    
-    # Create data directory if it doesn't exist
-    os.makedirs(DATA_DIR, exist_ok=True)
-
-
-config = Config()
-HEADERS = {"Authorization": f"Bearer {config.HF_TOKEN}"} if config.HF_TOKEN else {}
-
-
-# === Input Validation (FIXED: Comprehensive validation) ===
-def validate_component_id(component_id: str) -> Tuple[bool, str]:
-    """Validate component ID format"""
-    if not isinstance(component_id, str):
-        return False, "Component ID must be a string"
-    
-    if not (1 <= len(component_id) <= 255):
-        return False, "Component ID must be 1-255 characters"
-    
-    import re
-    if not re.match(r"^[a-z0-9-]+$", component_id):
-        return False, "Component ID must contain only lowercase letters, numbers, and hyphens"
-    
-    return True, ""
-
-
-def validate_inputs(
-    latency: Any,
-    error_rate: Any,
-    throughput: Any,
-    cpu_util: Any,
-    memory_util: Any
-) -> Tuple[bool, str]:
-    """
-    Comprehensive input validation with type checking
-    
-    FIXED: Added proper type validation before conversion
-    """
-    try:
-        # Type conversion with error handling
-        try:
-            latency_f = float(latency)
-        except (ValueError, TypeError):
-            return False, "❌ Invalid latency: must be a number"
-        
-        try:
-            error_rate_f = float(error_rate)
-        except (ValueError, TypeError):
-            return False, "❌ Invalid error rate: must be a number"
-        
-        try:
-            throughput_f = float(throughput) if throughput else 1000.0
-        except (ValueError, TypeError):
-            return False, "❌ Invalid throughput: must be a number"
-        
-        # CPU and memory are optional
-        cpu_util_f = None
-        if cpu_util:
-            try:
-                cpu_util_f = float(cpu_util)
-            except (ValueError, TypeError):
-                return False, "❌ Invalid CPU utilization: must be a number"
-        
-        memory_util_f = None
-        if memory_util:
-            try:
-                memory_util_f = float(memory_util)
-            except (ValueError, TypeError):
-                return False, "❌ Invalid memory utilization: must be a number"
-        
-        # Range validation
-        if not (0 <= latency_f <= 10000):
-            return False, "❌ Invalid latency: must be between 0-10000ms"
-        
-        if not (0 <= error_rate_f <= 1):
-            return False, "❌ Invalid error rate: must be between 0-1"
-        
-        if throughput_f < 0:
-            return False, "❌ Invalid throughput: must be positive"
-        
-        if cpu_util_f is not None and not (0 <= cpu_util_f <= 1):
-            return False, "❌ Invalid CPU utilization: must be between 0-1"
-        
-        if memory_util_f is not None and not (0 <= memory_util_f <= 1):
-            return False, "❌ Invalid memory utilization: must be between 0-1"
-        
-        return True, ""
-        
-    except Exception as e:
-        logger.error(f"Validation error: {e}", exc_info=True)
-        return False, f"❌ Validation error: {str(e)}"
-
-
-# === Thread-Safe Data Structures ===
-class ThreadSafeEventStore:
-    """Thread-safe storage for reliability events"""
-    
-    def __init__(self, max_size: int = Constants.MAX_EVENTS_STORED):
-        self._events = deque(maxlen=max_size)
-        self._lock = threading.RLock()
-        logger.info(f"Initialized ThreadSafeEventStore with max_size={max_size}")
-    
-    def add(self, event: ReliabilityEvent) -> None:
-        """Add event to store"""
-        with self._lock:
-            self._events.append(event)
-            logger.debug(f"Added event for {event.component}: {event.severity.value}")
-    
-    def get_recent(self, n: int = 15) -> List[ReliabilityEvent]:
-        """Get n most recent events"""
-        with self._lock:
-            return list(self._events)[-n:] if self._events else []
-    
-    def get_all(self) -> List[ReliabilityEvent]:
-        """Get all events"""
-        with self._lock:
-            return list(self._events)
-    
-    def count(self) -> int:
-        """Get total event count"""
-        with self._lock:
-            return len(self._events)
-
-
-# === FAISS Integration (FIXED: Single-writer pattern for thread safety) ===
-class ProductionFAISSIndex:
-    """
-    Production-safe FAISS index with single-writer pattern
-    
-    CRITICAL FIX: FAISS is NOT thread-safe for concurrent writes
-    Solution: Queue-based single writer thread + atomic saves
-    """
-    
-    def __init__(self, index, texts: List[str]):
-        self.index = index
-        self.texts = texts
-        self._lock = threading.RLock()
-        
-        # FIXED: Initialize shutdown event BEFORE starting thread
-        self._shutdown = threading.Event()
-        
-        # Single writer thread (no concurrent write conflicts)
-        self._write_queue: Queue = Queue()
-        self._writer_thread = threading.Thread(
-            target=self._writer_loop,
-            daemon=True,
-            name="FAISSWriter"
-        )
-        self._writer_thread.start()  # ← Only start ONCE, AFTER _shutdown exists
-        
-        # ProcessPool for encoding (avoids GIL + memory leaks)
-        self._encoder_pool = ProcessPoolExecutor(max_workers=2)
-        
-        logger.info(
-            f"Initialized ProductionFAISSIndex with {len(texts)} vectors, "
-            f"single-writer pattern"
-        )
-    
-    def add_async(self, vector: np.ndarray, text: str) -> None:
-        """
-        Add vector and text asynchronously (thread-safe)
-        
-        FIXED: Queue-based design - no concurrent FAISS writes
-        """
-        self._write_queue.put((vector, text))
-        logger.debug(f"Queued vector for indexing: {text[:50]}...")
-    
-    def _writer_loop(self) -> None:
-        """
-        Single writer thread - processes queue in batches
-        
-        This ensures only ONE thread ever writes to FAISS index
-        """
-        batch = []
-        last_save = datetime.datetime.now()
-        save_interval = datetime.timedelta(
-            seconds=Constants.FAISS_SAVE_INTERVAL_SECONDS
-        )
-        
-        while not self._shutdown.is_set():
-            try:
-                # Collect batch (non-blocking with timeout)
-                import queue
-                try:
-                    item = self._write_queue.get(timeout=1.0)
-                    batch.append(item)
-                except queue.Empty:
-                    pass
-                
-                # Process batch when ready
-                if len(batch) >= Constants.FAISS_BATCH_SIZE or \
-                   (batch and datetime.datetime.now() - last_save > save_interval):
-                    
-                    self._flush_batch(batch)
-                    batch = []
-                    
-                    # Periodic save
-                    if datetime.datetime.now() - last_save > save_interval:
-                        self._save_atomic()
-                        last_save = datetime.datetime.now()
-                        
-            except Exception as e:
-                logger.error(f"Writer loop error: {e}", exc_info=True)
-    
-    def _flush_batch(self, batch: List[Tuple[np.ndarray, str]]) -> None:
-        """
-        Flush batch to FAISS index
-        
-        SAFE: Only called from single writer thread
-        """
-        if not batch:
-            return
-        
-        try:
-            vectors = np.vstack([v for v, _ in batch])
-            texts = [t for _, t in batch]
-            
-            # SAFE: Single writer - no concurrent access
-            self.index.add(vectors)
-            
-            with self._lock:  # Only lock for text list modification
-                self.texts.extend(texts)
-            
-            logger.info(f"Flushed batch of {len(batch)} vectors to FAISS index")
-            
-        except Exception as e:
-            logger.error(f"Error flushing batch: {e}", exc_info=True)
-    
-    def _save_atomic(self) -> None:
-        """
-        Atomic save with fsync for durability
-        
-        FIXED: Prevents corruption on crash
-        """
-        try:
-            import faiss
-            
-            # Write to temporary file first
-            with tempfile.NamedTemporaryFile(
-                mode='wb',
-                delete=False,
-                dir=os.path.dirname(config.INDEX_FILE),
-                prefix='index_',
-                suffix='.tmp'
-            ) as tmp:
-                temp_path = tmp.name
-            
-            # Write index
-            faiss.write_index(self.index, temp_path)
-            
-            # Fsync for durability
-            with open(temp_path, 'r+b') as f:
-                f.flush()
-                os.fsync(f.fileno())
-            
-            # Atomic rename
-            os.replace(temp_path, config.INDEX_FILE)
-            
-            # Save texts with atomic write
-            with self._lock:
-                texts_copy = self.texts.copy()
-            
-            with atomicwrites.atomic_write(
-                config.TEXTS_FILE,
-                mode='w',
-                overwrite=True
-            ) as f:
-                json.dump(texts_copy, f)
-            
-            logger.info(
-                f"Atomically saved FAISS index with {len(texts_copy)} vectors"
-            )
-            
-        except Exception as e:
-            logger.error(f"Error saving index: {e}", exc_info=True)
-    
-    def get_count(self) -> int:
-        """Get total count of vectors"""
-        with self._lock:
-            return len(self.texts) + self._write_queue.qsize()
-    
-    def force_save(self) -> None:
-        """Force immediate save of pending vectors"""
-        logger.info("Forcing FAISS index save...")
-        
-        # Wait for queue to drain (with timeout)
-        timeout = 10.0
-        start = datetime.datetime.now()
-        
-        while not self._write_queue.empty():
-            if (datetime.datetime.now() - start).total_seconds() > timeout:
-                logger.warning("Force save timeout - queue not empty")
-                break
-            import time
-            time.sleep(0.1)
-        
-        self._save_atomic()
-    
-    def shutdown(self) -> None:
-        """Graceful shutdown"""
-        logger.info("Shutting down FAISS index...")
-        self._shutdown.set()
-        self.force_save()
-        self._writer_thread.join(timeout=5.0)
-        self._encoder_pool.shutdown(wait=True)
-
-
-# === FAISS & Embeddings Setup ===
-try:
-    from sentence_transformers import SentenceTransformer
-    import faiss
-    
-    logger.info("Loading SentenceTransformer model...")
-    model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-    logger.info("SentenceTransformer model loaded successfully")
-    
-    if os.path.exists(config.INDEX_FILE):
-        logger.info(f"Loading existing FAISS index from {config.INDEX_FILE}")
-        index = faiss.read_index(config.INDEX_FILE)
-        
-        if index.d != Constants.VECTOR_DIM:
-            logger.warning(
-                f"Index dimension mismatch: {index.d} != {Constants.VECTOR_DIM}. "
-                f"Creating new index."
-            )
-            index = faiss.IndexFlatL2(Constants.VECTOR_DIM)
-            incident_texts = []
-        else:
-            with open(config.TEXTS_FILE, "r") as f:
-                incident_texts = json.load(f)
-            logger.info(f"Loaded {len(incident_texts)} incident texts")
-    else:
-        logger.info("Creating new FAISS index")
-        index = faiss.IndexFlatL2(Constants.VECTOR_DIM)
-        incident_texts = []
-    
-    thread_safe_index = ProductionFAISSIndex(index, incident_texts)
-    
-except ImportError as e:
-    logger.warning(f"FAISS or SentenceTransformers not available: {e}")
-    index = None
-    incident_texts = []
-    model = None
-    thread_safe_index = None
-except Exception as e:
-    logger.error(f"Error initializing FAISS: {e}", exc_info=True)
-    index = None
-    incident_texts = []
-    model = None
-    thread_safe_index = None
-
-    # === Predictive Models ===
-class SimplePredictiveEngine:
-    """
-    Lightweight forecasting engine with proper constant usage
-    
-    FIXED: All magic numbers extracted to Constants
-    """
-    
-    def __init__(self, history_window: int = Constants.HISTORY_WINDOW):
-        self.history_window = history_window
-        self.service_history: Dict[str, deque] = {}
-        self.prediction_cache: Dict[str, Tuple[ForecastResult, datetime.datetime]] = {}
-        self.max_cache_age = datetime.timedelta(minutes=Constants.CACHE_EXPIRY_MINUTES)
-        self._lock = threading.RLock()
-        logger.info(f"Initialized SimplePredictiveEngine with history_window={history_window}")
-    
-    def add_telemetry(self, service: str, event_data: Dict) -> None:
-        """Add telemetry data to service history"""
-        with self._lock:
-            if service not in self.service_history:
-                self.service_history[service] = deque(maxlen=self.history_window)
-            
-            telemetry_point = {
-                'timestamp': datetime.datetime.now(datetime.timezone.utc),
-                'latency': event_data.get('latency_p99', 0),
-                'error_rate': event_data.get('error_rate', 0),
-                'throughput': event_data.get('throughput', 0),
-                'cpu_util': event_data.get('cpu_util'),
-                'memory_util': event_data.get('memory_util')
-            }
-            
-            self.service_history[service].append(telemetry_point)
-            self._clean_cache()
-    
-    def _clean_cache(self) -> None:
-        """Remove expired entries from prediction cache"""
-        now = datetime.datetime.now(datetime.timezone.utc)
-        expired = [k for k, (_, ts) in self.prediction_cache.items() 
-                   if now - ts > self.max_cache_age]
-        for k in expired:
-            del self.prediction_cache[k]
-        
-        if expired:
-            logger.debug(f"Cleaned {len(expired)} expired cache entries")
-    
-    def forecast_service_health(
-        self,
-        service: str,
-        lookahead_minutes: int = Constants.FORECAST_LOOKAHEAD_MINUTES
-    ) -> List[ForecastResult]:
-        """Forecast service health metrics"""
-        with self._lock:
-            if service not in self.service_history or \
-               len(self.service_history[service]) < Constants.FORECAST_MIN_DATA_POINTS:
-                return []
-            
-            history = list(self.service_history[service])
-        
-        forecasts = []
-        
-        # Forecast latency
-        latency_forecast = self._forecast_latency(history, lookahead_minutes)
-        if latency_forecast:
-            forecasts.append(latency_forecast)
-        
-        # Forecast error rate
-        error_forecast = self._forecast_error_rate(history, lookahead_minutes)
-        if error_forecast:
-            forecasts.append(error_forecast)
-        
-        # Forecast resource utilization
-        resource_forecasts = self._forecast_resources(history, lookahead_minutes)
-        forecasts.extend(resource_forecasts)
-        
-        # Cache results
-        with self._lock:
-            for forecast in forecasts:
-                cache_key = f"{service}_{forecast.metric}"
-                self.prediction_cache[cache_key] = (forecast, datetime.datetime.now(datetime.timezone.utc))
-        
-        return forecasts
-    
-    def _forecast_latency(
-        self,
-        history: List,
-        lookahead_minutes: int
-    ) -> Optional[ForecastResult]:
-        """Forecast latency using linear regression"""
-        try:
-            latencies = [point['latency'] for point in history[-20:]]
-            
-            if len(latencies) < Constants.FORECAST_MIN_DATA_POINTS:
-                return None
-            
-            # Linear trend
-            x = np.arange(len(latencies))
-            slope, intercept = np.polyfit(x, latencies, 1)
-            
-            # Predict next value
-            next_x = len(latencies)
-            predicted_latency = slope * next_x + intercept
-            
-            # Calculate confidence
-            residuals = latencies - (slope * x + intercept)
-            confidence = max(0, 1 - (np.std(residuals) / max(1, np.mean(latencies))))
-            
-            # Determine trend and risk
-            if slope > Constants.SLOPE_THRESHOLD_INCREASING:
-                trend = "increasing"
-                risk = "critical" if predicted_latency > Constants.LATENCY_EXTREME else "high"
-            elif slope < Constants.SLOPE_THRESHOLD_DECREASING:
-                trend = "decreasing"
-                risk = "low"
-            else:
-                trend = "stable"
-                risk = "low" if predicted_latency < Constants.LATENCY_WARNING else "medium"
-            
-            # Calculate time to reach critical threshold
-            time_to_critical = None
-            if slope > 0 and predicted_latency < Constants.LATENCY_EXTREME:
-                denominator = predicted_latency - latencies[-1]
-                if abs(denominator) > 0.1:
-                    minutes_to_critical = lookahead_minutes * \
-                        (Constants.LATENCY_EXTREME - predicted_latency) / denominator
-                    if minutes_to_critical > 0:
-                        time_to_critical = minutes_to_critical
-            
-            return ForecastResult(
-                metric="latency",
-                predicted_value=predicted_latency,
-                confidence=confidence,
-                trend=trend,
-                time_to_threshold=time_to_critical,
-                risk_level=risk
-            )
-            
-        except Exception as e:
-            logger.error(f"Latency forecast error: {e}", exc_info=True)
-            return None
-    
-    def _forecast_error_rate(
-        self,
-        history: List,
-        lookahead_minutes: int
-    ) -> Optional[ForecastResult]:
-        """Forecast error rate using exponential smoothing"""
-        try:
-            error_rates = [point['error_rate'] for point in history[-15:]]
-            
-            if len(error_rates) < Constants.FORECAST_MIN_DATA_POINTS:
-                return None
-            
-            # Exponential smoothing
-            alpha = 0.3
-            forecast = error_rates[0]
-            for rate in error_rates[1:]:
-                forecast = alpha * rate + (1 - alpha) * forecast
-            
-            predicted_rate = forecast
-            
-            # Trend analysis
-            recent_trend = np.mean(error_rates[-3:]) - np.mean(error_rates[-6:-3])
-            
-            if recent_trend > 0.02:
-                trend = "increasing"
-                risk = "critical" if predicted_rate > Constants.ERROR_RATE_CRITICAL else "high"
-            elif recent_trend < -0.01:
-                trend = "decreasing"
-                risk = "low"
-            else:
-                trend = "stable"
-                risk = "low" if predicted_rate < Constants.ERROR_RATE_WARNING else "medium"
-            
-            # Confidence based on volatility
-            confidence = max(0, 1 - (np.std(error_rates) / max(0.01, np.mean(error_rates))))
-            
-            return ForecastResult(
-                metric="error_rate",
-                predicted_value=predicted_rate,
-                confidence=confidence,
-                trend=trend,
-                risk_level=risk
-            )
-            
-        except Exception as e:
-            logger.error(f"Error rate forecast error: {e}", exc_info=True)
-            return None
-    
-    def _forecast_resources(
-        self,
-        history: List,
-        lookahead_minutes: int
-    ) -> List[ForecastResult]:
-        """Forecast CPU and memory utilization"""
-        forecasts = []
-        
-        # CPU forecast
-        cpu_values = [point['cpu_util'] for point in history if point.get('cpu_util') is not None]
-        if len(cpu_values) >= Constants.FORECAST_MIN_DATA_POINTS:
-            try:
-                predicted_cpu = np.mean(cpu_values[-5:])
-                trend = "increasing" if cpu_values[-1] > np.mean(cpu_values[-10:-5]) else "stable"
-                
-                risk = "low"
-                if predicted_cpu > Constants.CPU_CRITICAL:
-                    risk = "critical"
-                elif predicted_cpu > Constants.CPU_WARNING:
-                    risk = "high"
-                elif predicted_cpu > 0.7:
-                    risk = "medium"
-                
-                forecasts.append(ForecastResult(
-                    metric="cpu_util",
-                    predicted_value=predicted_cpu,
-                    confidence=0.7,
-                    trend=trend,
-                    risk_level=risk
-                ))
-            except Exception as e:
-                logger.error(f"CPU forecast error: {e}", exc_info=True)
-        
-        # Memory forecast
-        memory_values = [point['memory_util'] for point in history if point.get('memory_util') is not None]
-        if len(memory_values) >= Constants.FORECAST_MIN_DATA_POINTS:
-            try:
-                predicted_memory = np.mean(memory_values[-5:])
-                trend = "increasing" if memory_values[-1] > np.mean(memory_values[-10:-5]) else "stable"
-                
-                risk = "low"
-                if predicted_memory > Constants.MEMORY_CRITICAL:
-                    risk = "critical"
-                elif predicted_memory > Constants.MEMORY_WARNING:
-                    risk = "high"
-                elif predicted_memory > 0.7:
-                    risk = "medium"
-                
-                forecasts.append(ForecastResult(
-                    metric="memory_util",
-                    predicted_value=predicted_memory,
-                    confidence=0.7,
-                    trend=trend,
-                    risk_level=risk
-                ))
-            except Exception as e:
-                logger.error(f"Memory forecast error: {e}", exc_info=True)
-        
-        return forecasts
-    
-    def get_predictive_insights(self, service: str) -> Dict[str, Any]:
-        """Generate actionable insights from forecasts"""
-        forecasts = self.forecast_service_health(service)
-        
-        critical_risks = [f for f in forecasts if f.risk_level in ["high", "critical"]]
-        warnings = []
-        recommendations = []
-        
-        for forecast in critical_risks:
-            if forecast.metric == "latency" and forecast.risk_level in ["high", "critical"]:
-                warnings.append(f"📈 Latency expected to reach {forecast.predicted_value:.0f}ms")
-                if forecast.time_to_threshold:
-                    minutes = int(forecast.time_to_threshold)
-                    recommendations.append(f"⏰ Critical latency (~{Constants.LATENCY_EXTREME}ms) in ~{minutes} minutes")
-                recommendations.append("🔧 Consider scaling or optimizing dependencies")
-            
-            elif forecast.metric == "error_rate" and forecast.risk_level in ["high", "critical"]:
-                warnings.append(f"🚨 Errors expected to reach {forecast.predicted_value*100:.1f}%")
-                recommendations.append("🐛 Investigate recent deployments or dependency issues")
-            
-            elif forecast.metric == "cpu_util" and forecast.risk_level in ["high", "critical"]:
-                warnings.append(f"🔥 CPU expected at {forecast.predicted_value*100:.1f}%")
-                recommendations.append("⚡ Consider scaling compute resources")
-            
-            elif forecast.metric == "memory_util" and forecast.risk_level in ["high", "critical"]:
-                warnings.append(f"💾 Memory expected at {forecast.predicted_value*100:.1f}%")
-                recommendations.append("🧹 Check for memory leaks or optimize usage")
-        
-        return {
-            'service': service,
-            'forecasts': [
-                {
-                    'metric': f.metric,
-                    'predicted_value': f.predicted_value,
-                    'confidence': f.confidence,
-                    'trend': f.trend,
-                    'risk_level': f.risk_level,
-                    'time_to_threshold': f.time_to_threshold
-                }
-                for f in forecasts
-            ],
-            'warnings': warnings[:3],
-            'recommendations': list(dict.fromkeys(recommendations))[:3],
-            'critical_risk_count': len(critical_risks),
-            'forecast_timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat()
-        }
-
-
-class BusinessImpactCalculator:
-    """Calculate business impact of anomalies"""
-    
-    def __init__(self, revenue_per_request: float = 0.01):
-        self.revenue_per_request = revenue_per_request
-        logger.info(f"Initialized BusinessImpactCalculator")
-    
-    def calculate_impact(
-        self,
-        event: ReliabilityEvent,
-        duration_minutes: int = 5
-    ) -> Dict[str, Any]:
-        """Calculate business impact for a reliability event"""
-        base_revenue_per_minute = Constants.BASE_REVENUE_PER_MINUTE
-        
-        impact_multiplier = 1.0
-        
-        # Impact factors
-        if event.latency_p99 > Constants.LATENCY_CRITICAL:
-            impact_multiplier += 0.5
-        if event.error_rate > 0.1:
-            impact_multiplier += 0.8
-        if event.cpu_util and event.cpu_util > Constants.CPU_CRITICAL:
-            impact_multiplier += 0.3
-        
-        revenue_loss = base_revenue_per_minute * impact_multiplier * (duration_minutes / 60)
-        
-        base_users_affected = Constants.BASE_USERS
-        user_impact_multiplier = (event.error_rate * 10) + \
-            (max(0, event.latency_p99 - 100) / 500)
-        affected_users = int(base_users_affected * user_impact_multiplier)
-        
-        # Severity classification
-        if revenue_loss > 500 or affected_users > 5000:
-            severity = "CRITICAL"
-        elif revenue_loss > 100 or affected_users > 1000:
-            severity = "HIGH"
-        elif revenue_loss > 50 or affected_users > 500:
-            severity = "MEDIUM"
-        else:
-            severity = "LOW"
-        
-        logger.info(
-            f"Business impact: ${revenue_loss:.2f} revenue loss, "
-            f"{affected_users} users, {severity} severity"
-        )
-        
-        return {
-            'revenue_loss_estimate': round(revenue_loss, 2),
-            'affected_users_estimate': affected_users,
-            'severity_level': severity,
-            'throughput_reduction_pct': round(min(100, user_impact_multiplier * 100), 1)
-        }
-
-
-class AdvancedAnomalyDetector:
-    """Enhanced anomaly detection with adaptive thresholds"""
-    
-    def __init__(self):
-        self.historical_data = deque(maxlen=100)
-        self.adaptive_thresholds = {
-            'latency_p99': Constants.LATENCY_WARNING,
-            'error_rate': Constants.ERROR_RATE_WARNING
-        }
-        self._lock = threading.RLock()
-        logger.info("Initialized AdvancedAnomalyDetector")
-    
-    def detect_anomaly(self, event: ReliabilityEvent) -> bool:
-        """Detect if event is anomalous using adaptive thresholds"""
-        with self._lock:
-            latency_anomaly = event.latency_p99 > self.adaptive_thresholds['latency_p99']
-            error_anomaly = event.error_rate > self.adaptive_thresholds['error_rate']
-            
-            resource_anomaly = False
-            if event.cpu_util and event.cpu_util > Constants.CPU_CRITICAL:
-                resource_anomaly = True
-            if event.memory_util and event.memory_util > Constants.MEMORY_CRITICAL:
-                resource_anomaly = True
-            
-            self._update_thresholds(event)
-            
-            is_anomaly = latency_anomaly or error_anomaly or resource_anomaly
-            
-            if is_anomaly:
-                logger.info(
-                    f"Anomaly detected for {event.component}: "
-                    f"latency={latency_anomaly}, error={error_anomaly}, "
-                    f"resource={resource_anomaly}"
-                )
-            
-            return is_anomaly
-    
-    def _update_thresholds(self, event: ReliabilityEvent) -> None:
-        """Update adaptive thresholds based on historical data"""
-        self.historical_data.append(event)
-        
-        if len(self.historical_data) > 10:
-            recent_latencies = [e.latency_p99 for e in list(self.historical_data)[-20:]]
-            new_threshold = np.percentile(recent_latencies, 90)
-            self.adaptive_thresholds['latency_p99'] = new_threshold
-            logger.debug(f"Updated adaptive latency threshold to {new_threshold:.2f}ms")
-
-            # === Multi-Agent System ===
-class AgentSpecialization(Enum):
-    """Agent specialization types"""
-    DETECTIVE = "anomaly_detection"
-    DIAGNOSTICIAN = "root_cause_analysis"
-    PREDICTIVE = "predictive_analytics"
-
-
-class BaseAgent:
-    """Base class for all specialized agents"""
-    
-    def __init__(self, specialization: AgentSpecialization):
-        self.specialization = specialization
-        self.performance_metrics = {
-            'processed_events': 0,
-            'successful_analyses': 0,
-            'average_confidence': 0.0
-        }
-    
-    async def analyze(self, event: ReliabilityEvent) -> Dict[str, Any]:
-        """Base analysis method to be implemented by specialized agents"""
-        raise NotImplementedError
-
-
-class AnomalyDetectionAgent(BaseAgent):
-    """Specialized agent for anomaly detection and pattern recognition"""
-    
-    def __init__(self):
-        super().__init__(AgentSpecialization.DETECTIVE)
-        logger.info("Initialized AnomalyDetectionAgent")
-    
-    async def analyze(self, event: ReliabilityEvent) -> Dict[str, Any]:
-        """Perform comprehensive anomaly analysis"""
-        try:
-            anomaly_score = self._calculate_anomaly_score(event)
-            
-            return {
-                'specialization': self.specialization.value,
-                'confidence': anomaly_score,
-                'findings': {
-                    'anomaly_score': anomaly_score,
-                    'severity_tier': self._classify_severity(anomaly_score),
-                    'primary_metrics_affected': self._identify_affected_metrics(event)
-                },
-                'recommendations': self._generate_detection_recommendations(event, anomaly_score)
-            }
-        except Exception as e:
-            logger.error(f"AnomalyDetectionAgent error: {e}", exc_info=True)
-            return {
-                'specialization': self.specialization.value,
-                'confidence': 0.0,
-                'findings': {},
-                'recommendations': [f"Analysis error: {str(e)}"]
-            }
-    
-    def _calculate_anomaly_score(self, event: ReliabilityEvent) -> float:
-        """Calculate comprehensive anomaly score (0-1)"""
-        scores = []
-        
-        # Latency anomaly (weighted 40%)
-        if event.latency_p99 > Constants.LATENCY_WARNING:
-            latency_score = min(1.0, (event.latency_p99 - Constants.LATENCY_WARNING) / 500)
-            scores.append(0.4 * latency_score)
-        
-        # Error rate anomaly (weighted 30%)
-        if event.error_rate > Constants.ERROR_RATE_WARNING:
-            error_score = min(1.0, event.error_rate / 0.3)
-            scores.append(0.3 * error_score)
-        
-        # Resource anomaly (weighted 30%)
-        resource_score = 0
-        if event.cpu_util and event.cpu_util > Constants.CPU_WARNING:
-            resource_score += 0.15 * min(1.0, (event.cpu_util - Constants.CPU_WARNING) / 0.2)
-        if event.memory_util and event.memory_util > Constants.MEMORY_WARNING:
-            resource_score += 0.15 * min(1.0, (event.memory_util - Constants.MEMORY_WARNING) / 0.2)
-        scores.append(resource_score)
-        
-        return min(1.0, sum(scores))
-    
-    def _classify_severity(self, anomaly_score: float) -> str:
-        """Classify severity tier based on anomaly score"""
-        if anomaly_score > 0.8:
-            return "CRITICAL"
-        elif anomaly_score > 0.6:
-            return "HIGH"
-        elif anomaly_score > 0.4:
-            return "MEDIUM"
-        else:
-            return "LOW"
-    
-    def _identify_affected_metrics(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
-        """Identify which metrics are outside normal ranges"""
-        affected = []
-        
-        # Latency checks
-        if event.latency_p99 > Constants.LATENCY_EXTREME:
-            affected.append({
-                "metric": "latency",
-                "value": event.latency_p99,
-                "severity": "CRITICAL",
-                "threshold": Constants.LATENCY_WARNING
-            })
-        elif event.latency_p99 > Constants.LATENCY_CRITICAL:
-            affected.append({
-                "metric": "latency",
-                "value": event.latency_p99,
-                "severity": "HIGH",
-                "threshold": Constants.LATENCY_WARNING
-            })
-        elif event.latency_p99 > Constants.LATENCY_WARNING:
-            affected.append({
-                "metric": "latency",
-                "value": event.latency_p99,
-                "severity": "MEDIUM",
-                "threshold": Constants.LATENCY_WARNING
-            })
-        
-        # Error rate checks
-        if event.error_rate > Constants.ERROR_RATE_CRITICAL:
-            affected.append({
-                "metric": "error_rate",
-                "value": event.error_rate,
-                "severity": "CRITICAL",
-                "threshold": Constants.ERROR_RATE_WARNING
-            })
-        elif event.error_rate > Constants.ERROR_RATE_HIGH:
-            affected.append({
-                "metric": "error_rate",
-                "value": event.error_rate,
-                "severity": "HIGH",
-                "threshold": Constants.ERROR_RATE_WARNING
-            })
-        elif event.error_rate > Constants.ERROR_RATE_WARNING:
-            affected.append({
-                "metric": "error_rate",
-                "value": event.error_rate,
-                "severity": "MEDIUM",
-                "threshold": Constants.ERROR_RATE_WARNING
-            })
-        
-        # CPU checks
-        if event.cpu_util and event.cpu_util > Constants.CPU_CRITICAL:
-            affected.append({
-                "metric": "cpu",
-                "value": event.cpu_util,
-                "severity": "CRITICAL",
-                "threshold": Constants.CPU_WARNING
-            })
-        elif event.cpu_util and event.cpu_util > Constants.CPU_WARNING:
-            affected.append({
-                "metric": "cpu",
-                "value": event.cpu_util,
-                "severity": "HIGH",
-                "threshold": Constants.CPU_WARNING
-            })
-        
-        # Memory checks
-        if event.memory_util and event.memory_util > Constants.MEMORY_CRITICAL:
-            affected.append({
-                "metric": "memory",
-                "value": event.memory_util,
-                "severity": "CRITICAL",
-                "threshold": Constants.MEMORY_WARNING
-            })
-        elif event.memory_util and event.memory_util > Constants.MEMORY_WARNING:
-            affected.append({
-                "metric": "memory",
-                "value": event.memory_util,
-                "severity": "HIGH",
-                "threshold": Constants.MEMORY_WARNING
-            })
-        
-        return affected
-    
-    def _generate_detection_recommendations(
-        self,
-        event: ReliabilityEvent,
-        anomaly_score: float
-    ) -> List[str]:
-        """Generate actionable recommendations"""
-        recommendations = []
-        affected_metrics = self._identify_affected_metrics(event)
-        
-        for metric in affected_metrics:
-            metric_name = metric["metric"]
-            severity = metric["severity"]
-            value = metric["value"]
-            threshold = metric["threshold"]
-            
-            if metric_name == "latency":
-                if severity == "CRITICAL":
-                    recommendations.append(
-                        f"🚨 CRITICAL: Latency {value:.0f}ms (>{threshold}ms) - "
-                        f"Check database & external dependencies"
-                    )
-                elif severity == "HIGH":
-                    recommendations.append(
-                        f"⚠️ HIGH: Latency {value:.0f}ms (>{threshold}ms) - "
-                        f"Investigate service performance"
-                    )
-                else:
-                    recommendations.append(
-                        f"📈 Latency elevated: {value:.0f}ms (>{threshold}ms) - Monitor trend"
-                    )
-            
-            elif metric_name == "error_rate":
-                if severity == "CRITICAL":
-                    recommendations.append(
-                        f"🚨 CRITICAL: Error rate {value*100:.1f}% (>{threshold*100:.1f}%) - "
-                        f"Check recent deployments"
-                    )
-                elif severity == "HIGH":
-                    recommendations.append(
-                        f"⚠️ HIGH: Error rate {value*100:.1f}% (>{threshold*100:.1f}%) - "
-                        f"Review application logs"
-                    )
-                else:
-                    recommendations.append(
-                        f"📈 Errors increasing: {value*100:.1f}% (>{threshold*100:.1f}%)"
-                    )
-            
-            elif metric_name == "cpu":
-                recommendations.append(
-                    f"🔥 CPU {severity}: {value*100:.1f}% utilization - Consider scaling"
-                )
-            
-            elif metric_name == "memory":
-                recommendations.append(
-                    f"💾 Memory {severity}: {value*100:.1f}% utilization - Check for memory leaks"
-                )
-        
-        # Overall severity recommendations
-        if anomaly_score > 0.8:
-            recommendations.append("🎯 IMMEDIATE ACTION REQUIRED: Multiple critical metrics affected")
-        elif anomaly_score > 0.6:
-            recommendations.append("🎯 INVESTIGATE: Significant performance degradation detected")
-        elif anomaly_score > 0.4:
-            recommendations.append("📊 MONITOR: Early warning signs detected")
-        
-        return recommendations[:4]
-
-
-class RootCauseAgent(BaseAgent):
-    """Specialized agent for root cause analysis"""
-    
-    def __init__(self):
-        super().__init__(AgentSpecialization.DIAGNOSTICIAN)
-        logger.info("Initialized RootCauseAgent")
-    
-    async def analyze(self, event: ReliabilityEvent) -> Dict[str, Any]:
-        """Perform root cause analysis"""
-        try:
-            causes = self._analyze_potential_causes(event)
-            
-            return {
-                'specialization': self.specialization.value,
-                'confidence': 0.7,
-                'findings': {
-                    'likely_root_causes': causes,
-                    'evidence_patterns': self._identify_evidence(event),
-                    'investigation_priority': self._prioritize_investigation(causes)
-                },
-                'recommendations': [
-                    f"Check {cause['cause']} for issues" for cause in causes[:2]
-                ]
-            }
-        except Exception as e:
-            logger.error(f"RootCauseAgent error: {e}", exc_info=True)
-            return {
-                'specialization': self.specialization.value,
-                'confidence': 0.0,
-                'findings': {},
-                'recommendations': [f"Analysis error: {str(e)}"]
-            }
-    
-    def _analyze_potential_causes(self, event: ReliabilityEvent) -> List[Dict[str, Any]]:
-        """Analyze potential root causes based on event patterns"""
-        causes = []
-        
-        # Pattern 1: Database/External Dependency Failure
-        if event.latency_p99 > Constants.LATENCY_EXTREME and event.error_rate > 0.2:
-            causes.append({
-                "cause": "Database/External Dependency Failure",
-                "confidence": 0.85,
-                "evidence": f"Extreme latency ({event.latency_p99:.0f}ms) with high errors ({event.error_rate*100:.1f}%)",
-                "investigation": "Check database connection pool, external API health"
-            })
-        
-        # Pattern 2: Resource Exhaustion
-        if (event.cpu_util and event.cpu_util > Constants.CPU_CRITICAL and
-            event.memory_util and event.memory_util > Constants.MEMORY_CRITICAL):
-            causes.append({
-                "cause": "Resource Exhaustion",
-                "confidence": 0.90,
-                "evidence": f"CPU ({event.cpu_util*100:.1f}%) and Memory ({event.memory_util*100:.1f}%) critically high",
-                "investigation": "Check for memory leaks, infinite loops, insufficient resources"
-            })
-        
-        # Pattern 3: Application Bug / Configuration Issue
-        if event.error_rate > Constants.ERROR_RATE_CRITICAL and event.latency_p99 < 200:
-            causes.append({
-                "cause": "Application Bug / Configuration Issue",
-                "confidence": 0.75,
-                "evidence": f"High error rate ({event.error_rate*100:.1f}%) without latency impact",
-                "investigation": "Review recent deployments, configuration changes, application logs"
-            })
-        
-        # Pattern 4: Gradual Performance Degradation
-        if (200 <= event.latency_p99 <= 400 and
-            Constants.ERROR_RATE_WARNING <= event.error_rate <= Constants.ERROR_RATE_HIGH):
-            causes.append({
-                "cause": "Gradual Performance Degradation",
-                "confidence": 0.65,
-                "evidence": f"Moderate latency ({event.latency_p99:.0f}ms) and errors ({event.error_rate*100:.1f}%)",
-                "investigation": "Check resource trends, dependency performance, capacity planning"
-            })
-        
-        # Default: Unknown pattern
-        if not causes:
-            causes.append({
-                "cause": "Unknown - Requires Investigation",
-                "confidence": 0.3,
-                "evidence": "Pattern does not match known failure modes",
-                "investigation": "Complete system review needed"
-            })
-        
-        return causes
-    
-    def _identify_evidence(self, event: ReliabilityEvent) -> List[str]:
-        """Identify evidence patterns in the event data"""
-        evidence = []
-        
-        if event.latency_p99 > event.error_rate * 1000:
-            evidence.append("latency_disproportionate_to_errors")
-        
-        if (event.cpu_util and event.cpu_util > Constants.CPU_WARNING and
-            event.memory_util and event.memory_util > Constants.MEMORY_WARNING):
-            evidence.append("correlated_resource_exhaustion")
-        
-        if event.error_rate > Constants.ERROR_RATE_HIGH and event.latency_p99 < Constants.LATENCY_CRITICAL:
-            evidence.append("errors_without_latency_impact")
-        
-        return evidence
-    
-    def _prioritize_investigation(self, causes: List[Dict[str, Any]]) -> str:
-        """Determine investigation priority"""
-        for cause in causes:
-            if "Database" in cause["cause"] or "Resource Exhaustion" in cause["cause"]:
-                return "HIGH"
-        return "MEDIUM"
-
-
-class PredictiveAgent(BaseAgent):
-    """Specialized agent for predictive analytics"""
-    
-    def __init__(self, engine: SimplePredictiveEngine):
-        super().__init__(AgentSpecialization.PREDICTIVE)
-        self.engine = engine
-        logger.info("Initialized PredictiveAgent")
-    
-    async def analyze(self, event: ReliabilityEvent) -> Dict[str, Any]:
-        """Perform predictive analysis for future risks"""
-        try:
-            event_data = {
-                'latency_p99': event.latency_p99,
-                'error_rate': event.error_rate,
-                'throughput': event.throughput,
-                'cpu_util': event.cpu_util,
-                'memory_util': event.memory_util
-            }
-            self.engine.add_telemetry(event.component, event_data)
-            
-            insights = self.engine.get_predictive_insights(event.component)
-            
-            return {
-                'specialization': self.specialization.value,
-                'confidence': 0.8 if insights['critical_risk_count'] > 0 else 0.5,
-                'findings': insights,
-                'recommendations': insights['recommendations']
-            }
-        except Exception as e:
-            logger.error(f"PredictiveAgent error: {e}", exc_info=True)
-            return {
-                'specialization': self.specialization.value,
-                'confidence': 0.0,
-                'findings': {},
-                'recommendations': [f"Analysis error: {str(e)}"]
-            }
-
-
-# FIXED: Add circuit breaker for agent resilience
-@circuit(failure_threshold=3, recovery_timeout=30, name="agent_circuit_breaker")
-async def call_agent_with_protection(agent: BaseAgent, event: ReliabilityEvent) -> Dict[str, Any]:
-    """
-    Call agent with circuit breaker protection
-    
-    FIXED: Prevents cascading failures from misbehaving agents
-    """
-    try:
-        result = await asyncio.wait_for(
-            agent.analyze(event),
-            timeout=Constants.AGENT_TIMEOUT_SECONDS
-        )
-        return result
-    except asyncio.TimeoutError:
-        logger.warning(f"Agent {agent.specialization.value} timed out")
-        raise
-    except Exception as e:
-        logger.error(f"Agent {agent.specialization.value} error: {e}", exc_info=True)
-        raise
-
-
-class OrchestrationManager:
-    """Orchestrates multiple specialized agents for comprehensive analysis"""
-    
-    def __init__(
-        self,
-        detective: Optional[AnomalyDetectionAgent] = None,
-        diagnostician: Optional[RootCauseAgent] = None,
-        predictive: Optional[PredictiveAgent] = None
-    ):
-        """
-        Initialize orchestration manager
-        
-        FIXED: Dependency injection for testability
-        """
-        self.agents = {
-            AgentSpecialization.DETECTIVE: detective or AnomalyDetectionAgent(),
-            AgentSpecialization.DIAGNOSTICIAN: diagnostician or RootCauseAgent(),
-            AgentSpecialization.PREDICTIVE: predictive or PredictiveAgent(SimplePredictiveEngine()),
-        }
-        logger.info(f"Initialized OrchestrationManager with {len(self.agents)} agents")
-    
-    async def orchestrate_analysis(self, event: ReliabilityEvent) -> Dict[str, Any]:
-        """
-        Coordinate multiple agents for comprehensive analysis
-        
-        FIXED: Improved timeout handling with circuit breakers
-        """
-        # Create tasks for all agents
-        agent_tasks = []
-        agent_specs = []
-        
-        for spec, agent in self.agents.items():
-            agent_tasks.append(call_agent_with_protection(agent, event))
-            agent_specs.append(spec)
-        
-        # FIXED: Parallel execution with global timeout
-        agent_results = {}
-        
-        try:
-            # Run all agents in parallel with global timeout
-            results = await asyncio.wait_for(
-                asyncio.gather(*agent_tasks, return_exceptions=True),
-                timeout=Constants.AGENT_TIMEOUT_SECONDS + 1
-            )
-            
-            # Process results
-            for spec, result in zip(agent_specs, results):
-                if isinstance(result, Exception):
-                    logger.error(f"Agent {spec.value} failed: {result}")
-                    continue
-                
-                agent_results[spec.value] = result
-                logger.debug(f"Agent {spec.value} completed successfully")
-                
-        except asyncio.TimeoutError:
-            logger.warning("Agent orchestration timed out")
-        except Exception as e:
-            logger.error(f"Agent orchestration error: {e}", exc_info=True)
-        
-        return self._synthesize_agent_findings(event, agent_results)
-    
-    def _synthesize_agent_findings(
-        self,
-        event: ReliabilityEvent,
-        agent_results: Dict
-    ) -> Dict[str, Any]:
-        """Combine insights from all specialized agents"""
-        detective_result = agent_results.get(AgentSpecialization.DETECTIVE.value)
-        diagnostician_result = agent_results.get(AgentSpecialization.DIAGNOSTICIAN.value)
-        predictive_result = agent_results.get(AgentSpecialization.PREDICTIVE.value)
-        
-        if not detective_result:
-            logger.warning("No detective agent results available")
-            return {'error': 'No agent results available'}
-        
-        synthesis = {
-            'incident_summary': {
-                'severity': detective_result['findings'].get('severity_tier', 'UNKNOWN'),
-                'anomaly_confidence': detective_result['confidence'],
-                'primary_metrics_affected': [
-                    metric["metric"] for metric in
-                    detective_result['findings'].get('primary_metrics_affected', [])
-                ]
-            },
-            'root_cause_insights': diagnostician_result['findings'] if diagnostician_result else {},
-            'predictive_insights': predictive_result['findings'] if predictive_result else {},
-            'recommended_actions': self._prioritize_actions(
-                detective_result.get('recommendations', []),
-                diagnostician_result.get('recommendations', []) if diagnostician_result else [],
-                predictive_result.get('recommendations', []) if predictive_result else []
-            ),
-            'agent_metadata': {
-                'participating_agents': list(agent_results.keys()),
-                'analysis_timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat()
-            }
-        }
-        
-        return synthesis
-    
-    def _prioritize_actions(
-        self,
-        detection_actions: List[str],
-        diagnosis_actions: List[str],
-        predictive_actions: List[str]
-    ) -> List[str]:
-        """Combine and prioritize actions from multiple agents"""
-        all_actions = detection_actions + diagnosis_actions + predictive_actions
-        seen = set()
-        unique_actions = []
-        for action in all_actions:
-            if action not in seen:
-                seen.add(action)
-                unique_actions.append(action)
-        return unique_actions[:5]
-
-        # === Enhanced Reliability Engine ===
-class EnhancedReliabilityEngine:
-    """
-    Main engine for processing reliability events
-    
-    FIXED: Dependency injection for all components
-    """
-    
-    def __init__(
-        self,
-        orchestrator: Optional[OrchestrationManager] = None,
-        policy_engine: Optional[PolicyEngine] = None,
-        event_store: Optional[ThreadSafeEventStore] = None,
-        anomaly_detector: Optional[AdvancedAnomalyDetector] = None,
-        business_calculator: Optional[BusinessImpactCalculator] = None
-    ):
-        """
-        Initialize reliability engine with dependency injection
-        
-        FIXED: All dependencies injected for testability
-        """
-        self.orchestrator = orchestrator or OrchestrationManager()
-        self.policy_engine = policy_engine or PolicyEngine()
-        self.event_store = event_store or ThreadSafeEventStore()
-        self.anomaly_detector = anomaly_detector or AdvancedAnomalyDetector()
-        self.business_calculator = business_calculator or BusinessImpactCalculator()
-        
-        self.performance_metrics = {
-            'total_incidents_processed': 0,
-            'multi_agent_analyses': 0,
-            'anomalies_detected': 0
-        }
-        self._lock = threading.RLock()
-        logger.info("Initialized EnhancedReliabilityEngine")
-    
-    async def process_event_enhanced(
-        self,
-        component: str,
-        latency: float,
-        error_rate: float,
-        throughput: float = 1000,
-        cpu_util: Optional[float] = None,
-        memory_util: Optional[float] = None
-    ) -> Dict[str, Any]:
-        """
-        Process a reliability event through the complete analysis pipeline
-        
-        FIXED: Proper async/await throughout
-        """
-        logger.info(
-            f"Processing event for {component}: latency={latency}ms, "
-            f"error_rate={error_rate*100:.1f}%"
-        )
-        
-        # Validate component ID
-        is_valid, error_msg = validate_component_id(component)
-        if not is_valid:
-            return {'error': error_msg, 'status': 'INVALID'}
-        
-        # Create event
-        try:
-            event = ReliabilityEvent(
-                component=component,
-                latency_p99=latency,
-                error_rate=error_rate,
-                throughput=throughput,
-                cpu_util=cpu_util,
-                memory_util=memory_util,
-                upstream_deps=["auth-service", "database"] if component == "api-service" else []
-            )
-        except Exception as e:
-            logger.error(f"Event creation error: {e}", exc_info=True)
-            return {'error': f'Invalid event data: {str(e)}', 'status': 'INVALID'}
-        
-        # Multi-agent analysis
-        agent_analysis = await self.orchestrator.orchestrate_analysis(event)
-        
-        # Anomaly detection
-        is_anomaly = self.anomaly_detector.detect_anomaly(event)
-        
-        # Determine severity based on agent confidence
-        agent_confidence = 0.0
-        if agent_analysis and 'incident_summary' in agent_analysis:
-            agent_confidence = agent_analysis.get('incident_summary', {}).get('anomaly_confidence', 0)
-        else:
-            agent_confidence = 0.8 if is_anomaly else 0.1
-        
-        # Set event severity
-        if agent_confidence > 0.8:
-            severity = EventSeverity.CRITICAL
-        elif agent_confidence > 0.6:
-            severity = EventSeverity.HIGH
-        elif agent_confidence > 0.4:
-            severity = EventSeverity.MEDIUM
-        else:
-            severity = EventSeverity.LOW
-        
-        # Create mutable copy with updated severity
-        event = event.model_copy(update={'severity': severity})
-        
-        # Evaluate healing policies
-        healing_actions = self.policy_engine.evaluate_policies(event)
-        
-        # Calculate business impact
-        business_impact = self.business_calculator.calculate_impact(event) if is_anomaly else None
-        
-        # Store in vector database for similarity detection
-        if thread_safe_index is not None and model is not None and is_anomaly:
-            try:
-                # FIXED: Non-blocking encoding with ProcessPoolExecutor
-                analysis_text = agent_analysis.get('recommended_actions', ['No analysis'])[0]
-                vector_text = f"{component} {latency} {error_rate} {analysis_text}"
-                
-                # Encode asynchronously
-                loop = asyncio.get_event_loop()
-                vec = await loop.run_in_executor(
-                    thread_safe_index._encoder_pool,
-                    model.encode,
-                    [vector_text]
-                )
-                
-                thread_safe_index.add_async(np.array(vec, dtype=np.float32), vector_text)
-            except Exception as e:
-                logger.error(f"Error storing vector: {e}", exc_info=True)
-        
-        # Build comprehensive result
-        result = {
-            "timestamp": event.timestamp.isoformat(),
-            "component": component,
-            "latency_p99": latency,
-            "error_rate": error_rate,
-            "throughput": throughput,
-            "status": "ANOMALY" if is_anomaly else "NORMAL",
-            "multi_agent_analysis": agent_analysis,
-            "healing_actions": [action.value for action in healing_actions],
-            "business_impact": business_impact,
-            "severity": event.severity.value,
-            "similar_incidents_count": thread_safe_index.get_count() if thread_safe_index and is_anomaly else 0,
-            "processing_metadata": {
-                "agents_used": agent_analysis.get('agent_metadata', {}).get('participating_agents', []),
-                "analysis_confidence": agent_analysis.get('incident_summary', {}).get('anomaly_confidence', 0)
-            }
-        }
-        
-        # Store event in history
-        self.event_store.add(event)
-        
-        # Update performance metrics
-        with self._lock:
-            self.performance_metrics['total_incidents_processed'] += 1
-            self.performance_metrics['multi_agent_analyses'] += 1
-            if is_anomaly:
-                self.performance_metrics['anomalies_detected'] += 1
-        
-        logger.info(f"Event processed: {result['status']} with {result['severity']} severity")
-        
-        # Enhance with Claude AI reasoning (optional layer)
-        try:
-            result = await self.enhance_with_claude(event, result)
-        except Exception as e:
-            logger.error(f"Failed to enhance with Claude: {e}")
-            # Continue without enhancement
-        
-        return result
-    
-    async def enhance_with_claude(
-        self, 
-        event: ReliabilityEvent, 
-        agent_results: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        """
-        Enhance agent results with Claude AI reasoning
-        
-        This is a NON-INVASIVE layer - all existing logic stays intact.
-        If Claude fails, original results are returned unchanged.
-        """
-        try:
-            # Build comprehensive context for Claude
-            context_parts = []
-            
-            # Add event summary
-            context_parts.append("INCIDENT SUMMARY:")
-            context_parts.append(f"Component: {event.component}")
-            context_parts.append(f"Timestamp: {event.timestamp.isoformat()}")
-            context_parts.append(f"Severity: {event.severity.value}")
-            context_parts.append("")
-            
-            # Add metrics
-            context_parts.append("METRICS:")
-            context_parts.append(f"• Latency P99: {event.latency_p99}ms")
-            context_parts.append(f"• Error Rate: {event.error_rate:.1%}")
-            context_parts.append(f"• Throughput: {event.throughput} req/s")
-            if event.cpu_util:
-                context_parts.append(f"• CPU: {event.cpu_util:.1%}")
-            if event.memory_util:
-                context_parts.append(f"• Memory: {event.memory_util:.1%}")
-            context_parts.append("")
-            
-            # Add agent findings
-            if agent_results:
-                context_parts.append("AGENT ANALYSIS:")
-                if 'multi_agent_analysis' in agent_results:
-                    analysis = agent_results['multi_agent_analysis']
-                    context_parts.append(json.dumps(analysis, indent=2))
-                elif 'incident_summary' in agent_results:
-                    context_parts.append(json.dumps(agent_results['incident_summary'], indent=2))
-            
-            context = "\n".join(context_parts)
-            
-            # Create prompt for Claude
-            prompt = f"""{context}
-
-TASK: Provide an executive summary synthesizing all agent analyses.
-Include:
-1. Concise incident description
-2. Most likely root cause
-3. Single best recovery action
-4. Estimated impact and recovery time
-
-Be specific and actionable."""
-
-            system_prompt = """You are a senior Site Reliability Engineer synthesizing 
-multiple AI agent analyses into clear, actionable guidance for incident response. 
-Focus on clarity, accuracy, and decisive recommendations."""
-            
-            # Get Claude's synthesis
-            logger.info("Requesting Claude synthesis of agent results")
-            claude_synthesis = claude_adapter.generate_completion(
-                prompt=prompt,
-                system_prompt=system_prompt
-            )
-            
-            # Add Claude's insights to results (non-destructive)
-            agent_results['claude_synthesis'] = {
-                'summary': claude_synthesis,
-                'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(),
-                'source': 'claude-opus-4'
-            }
-            
-            logger.info("✅ Claude synthesis added to results")
-            return agent_results
-            
-        except Exception as e:
-            logger.error(f"Claude enhancement failed: {e}", exc_info=True)
-            # Return original results unchanged - system still works!
-            return agent_results
-
-# === Initialize Engine (with dependency injection) ===
-enhanced_engine = EnhancedReliabilityEngine()
-
-
-# === Rate Limiting ===
-class RateLimiter:
-    """Simple rate limiter for request throttling"""
-    
-    def __init__(self, max_per_minute: int = Constants.MAX_REQUESTS_PER_MINUTE):
-        self.max_per_minute = max_per_minute
-        self.requests: deque = deque(maxlen=max_per_minute)
-        self._lock = threading.RLock()
-    
-    def is_allowed(self) -> Tuple[bool, str]:
-        """Check if request is allowed"""
-        with self._lock:
-            now = datetime.datetime.now(datetime.timezone.utc)
-            
-            # Remove requests older than 1 minute
-            one_minute_ago = now - datetime.timedelta(minutes=1)
-            while self.requests and self.requests[0] < one_minute_ago:
-                self.requests.popleft()
-            
-            # Check rate limit
-            if len(self.requests) >= self.max_per_minute:
-                return False, f"Rate limit exceeded: {self.max_per_minute} requests/minute"
-            
-            # Add current request
-            self.requests.append(now)
-            return True, ""
-
-
-rate_limiter = RateLimiter()
-
-
-# === Gradio UI ===
-def create_enhanced_ui():
-    """
-    Create the comprehensive Gradio UI for the reliability framework
-    
-    FIXED: Uses native async handlers (no event loop creation)
-    FIXED: Rate limiting on all endpoints
-    """
-    
-    with gr.Blocks(title="🧠 Agentic Reliability Framework", theme="soft") as demo:
-        gr.Markdown("""
-        # 🧠 Agentic Reliability Framework
-        **Multi-Agent AI System for Production Reliability**
-        
-        *Specialized AI agents working together to detect, diagnose, predict, and heal system issues*
-        
-        """)
-        
-        with gr.Row():
-            with gr.Column(scale=1):
-                gr.Markdown("### 📊 Telemetry Input")
-                component = gr.Dropdown(
-                    choices=["api-service", "auth-service", "payment-service", "database", "cache-service"],
-                    value="api-service",
-                    label="Component",
-                    info="Select the service being monitored"
-                )
-                latency = gr.Slider(
-                    minimum=10, maximum=1000, value=100, step=1,
-                    label="Latency P99 (ms)",
-                    info=f"Alert threshold: >{Constants.LATENCY_WARNING}ms (adaptive)"
-                )
-                error_rate = gr.Slider(
-                    minimum=0, maximum=0.5, value=0.02, step=0.001,
-                    label="Error Rate",
-                    info=f"Alert threshold: >{Constants.ERROR_RATE_WARNING}"
-                )
-                throughput = gr.Number(
-                    value=1000,
-                    label="Throughput (req/sec)",
-                    info="Current request rate"
-                )
-                cpu_util = gr.Slider(
-                    minimum=0, maximum=1, value=0.4, step=0.01,
-                    label="CPU Utilization",
-                    info="0.0 - 1.0 scale"
-                )
-                memory_util = gr.Slider(
-                    minimum=0, maximum=1, value=0.3, step=0.01,
-                    label="Memory Utilization",
-                    info="0.0 - 1.0 scale"
-                )
-                submit_btn = gr.Button("🚀 Submit Telemetry Event", variant="primary", size="lg")
-                
-            with gr.Column(scale=2):
-                gr.Markdown("### 🔍 Multi-Agent Analysis")
-                output_text = gr.Textbox(
-                    label="Agent Synthesis",
-                    placeholder="AI agents are analyzing...",
-                    lines=6
-                )
-                
-                with gr.Accordion("🤖 Agent Specialists Analysis", open=False):
-                    gr.Markdown("""
-                    **Specialized AI Agents:**
-                    - 🕵️ **Detective**: Anomaly detection & pattern recognition
-                    - 🔍 **Diagnostician**: Root cause analysis & investigation
-                    - 🔮 **Predictive**: Future risk forecasting & trend analysis
-                    """)
-                    
-                    agent_insights = gr.JSON(
-                        label="Detailed Agent Findings",
-                        value={}
-                    )
-                
-                with gr.Accordion("🔮 Predictive Analytics & Forecasting", open=False):
-                    gr.Markdown("""
-                    **Future Risk Forecasting:**
-                    - 📈 Latency trends and thresholds
-                    - 🚨 Error rate predictions
-                    - 🔥 Resource utilization forecasts
-                    - ⏰ Time-to-failure estimates
-                    """)
-                    
-                    predictive_insights = gr.JSON(
-                        label="Predictive Forecasts",
-                        value={}
-                    )
-
-                with gr.Accordion("🤖 Claude AI Synthesis", open=True):
-                    gr.Markdown("""
-                    **Claude Opus 4.5 Executive Summary:**
-                    - 📋 Incident synthesis from all agents
-                    - 🎯 Root cause identification
-                    - 💡 Recommended recovery actions
-                    - ⏰ Impact and recovery time estimates
-                    """)
-                    
-                    claude_output = gr.Markdown(
-                        value="*Claude AI synthesis will appear here after incident analysis*",
-                        label="AI Executive Summary"
-                    )
-                
-                gr.Markdown("### 📈 Recent Events (Last 15)")
-                events_table = gr.Dataframe(
-                    headers=["Timestamp", "Component", "Latency", "Error Rate", "Throughput", "Severity", "Analysis"],
-                    label="Event History",
-                    wrap=True,
-                )
-        
-        with gr.Accordion("ℹ️ Framework Capabilities", open=False):
-            gr.Markdown("""
-            - **🤖 Multi-Agent AI**: Specialized agents for detection, diagnosis, prediction, and healing
-            - **🔮 Predictive Analytics**: Forecast future risks and performance degradation
-            - **🔧 Policy-Based Healing**: Automated recovery actions based on severity and context
-            - **💰 Business Impact**: Revenue and user impact quantification
-            - **🎯 Adaptive Detection**: ML-powered thresholds that learn from your environment
-            - **📚 Vector Memory**: FAISS-based incident memory for similarity detection
-            - **⚡ Production Ready**: Circuit breakers, cooldowns, thread safety, and enterprise features
-            - **🔒 Security Patched**: All critical CVEs fixed (Gradio 5.50.0+, Requests 2.32.5+)
-            """)
-        
-        with gr.Accordion("🔧 Healing Policies", open=False):
-            policy_info = []
-            for policy in enhanced_engine.policy_engine.policies:
-                if policy.enabled:
-                    actions = ", ".join([action.value for action in policy.actions])
-                    policy_info.append(
-                        f"**{policy.name}** (Priority {policy.priority}): {actions}\n"
-                        f"  - Cooldown: {policy.cool_down_seconds}s\n"
-                        f"  - Max executions: {policy.max_executions_per_hour}/hour"
-                    )
-            
-            gr.Markdown("\n\n".join(policy_info))
-            
-        # FIXED: Native async handler (no event loop creation needed)
-        async def submit_event_enhanced_async(
-            component, latency, error_rate, throughput, cpu_util, memory_util
-        ):
-            """
-            Async event handler - uses Gradio's native async support
-            
-            CRITICAL FIX: No event loop creation - Gradio handles this
-            FIXED: Rate limiting added
-            FIXED: Comprehensive error handling
-            """
-            try:
-                # Rate limiting check
-                allowed, rate_msg = rate_limiter.is_allowed()
-                if not allowed:
-                    logger.warning(f"Rate limit exceeded")
-                    return rate_msg, {}, {}, "*Rate limit exceeded*", gr.Dataframe(value=[])
-                
-                # Type conversion
-                try:
-                    latency = float(latency)
-                    error_rate = float(error_rate)
-                    throughput = float(throughput) if throughput else 1000
-                    cpu_util = float(cpu_util) if cpu_util else None
-                    memory_util = float(memory_util) if memory_util else None
-                except (ValueError, TypeError) as e:
-                    error_msg = f"❌ Invalid input types: {str(e)}"
-                    logger.warning(error_msg)
-                    return error_msg, {}, {}, "*Invalid input type*", gr.Dataframe(value=[])
-                
-                # Input validation
-                is_valid, error_msg = validate_inputs(
-                    latency, error_rate, throughput, cpu_util, memory_util
-                )
-                if not is_valid:
-                    logger.warning(f"Invalid input: {error_msg}")
-                    return error_msg, {}, {}, "*Validation failed*", gr.Dataframe(value=[])
-                
-                # FIXED: Direct async call - no event loop creation needed
-                result = await enhanced_engine.process_event_enhanced(
-                    component, latency, error_rate, throughput, cpu_util, memory_util
-                )
-                
-                # Handle errors
-                if 'error' in result:
-                    return f"❌ {result['error']}", {}, {}, "*Error occurred*", gr.Dataframe(value=[])
-                
-                # Build table data (THREAD-SAFE)
-                table_data = []
-                for event in enhanced_engine.event_store.get_recent(15):
-                    table_data.append([
-                        event.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
-                        event.component,
-                        f"{event.latency_p99:.0f}ms",
-                        f"{event.error_rate:.3f}",
-                        f"{event.throughput:.0f}",
-                        event.severity.value.upper(),
-                        "Multi-agent analysis"
-                    ])
-                
-                # Format output message
-                status_emoji = "🚨" if result["status"] == "ANOMALY" else "✅"
-                output_msg = f"{status_emoji} **{result['status']}**\n"
-                
-                if "multi_agent_analysis" in result:
-                    analysis = result["multi_agent_analysis"]
-                    confidence = analysis.get('incident_summary', {}).get('anomaly_confidence', 0)
-                    output_msg += f"🎯 **Confidence**: {confidence*100:.1f}%\n"
-                    
-                    predictive_data = analysis.get('predictive_insights', {})
-                    if predictive_data.get('critical_risk_count', 0) > 0:
-                        output_msg += f"🔮 **PREDICTIVE**: {predictive_data['critical_risk_count']} critical risks forecast\n"
-                    
-                    if analysis.get('recommended_actions'):
-                        actions_preview = ', '.join(analysis['recommended_actions'][:2])
-                        output_msg += f"💡 **Top Insights**: {actions_preview}\n"
-                
-                if result.get("business_impact"):
-                    impact = result["business_impact"]
-                    output_msg += (
-                        f"💰 **Business Impact**: \${impact['revenue_loss_estimate']:.2f} | "
-                        f"👥 {impact['affected_users_estimate']} users | "
-                        f"🚨 {impact['severity_level']}\n"
-                    )
-                
-                if result.get("healing_actions") and result["healing_actions"] != ["no_action"]:
-                    actions = ", ".join(result["healing_actions"])
-                    output_msg += f"🔧 **Auto-Actions**: {actions}"
-                
-                agent_insights_data = result.get("multi_agent_analysis", {})
-                predictive_insights_data = agent_insights_data.get('predictive_insights', {})
-                
-                # Extract Claude synthesis for display
-                claude_synthesis = result.get('claude_synthesis', {})
-                claude_text = claude_synthesis.get('summary', '*No Claude synthesis available*')
-                
-                # Format Claude output beautifully
-                claude_display = f"""
-### 🤖 Claude Opus 4.5 Executive Analysis
-
-{claude_text}
-
----
-*Generated: {claude_synthesis.get('timestamp', 'N/A')}*  
-*Model: {claude_synthesis.get('source', 'claude-opus-4')}*
-"""
-                
-                return (
-                    output_msg,
-                    agent_insights_data,
-                    predictive_insights_data,
-                    claude_display,
-                    gr.Dataframe(
-                        headers=["Timestamp", "Component", "Latency", "Error Rate", "Throughput", "Severity", "Analysis"],
-                        value=table_data,
-                        wrap=True
-                    )
-                )
-                
-            except Exception as e:
-                error_msg = f"❌ Error processing event: {str(e)}"
-                logger.error(error_msg, exc_info=True)
-                return error_msg, {}, {}, "*Processing error*", gr.Dataframe(value=[])
-        
-        # FIXED: Use async handler directly
-        submit_btn.click(
-            fn=submit_event_enhanced_async,
-            inputs=[component, latency, error_rate, throughput, cpu_util, memory_util],
-            outputs=[output_text, agent_insights, predictive_insights, claude_output, events_table]
-        )
-    
-    return demo
-
-
-# === Main Entry Point ===
-if __name__ == "__main__":
-    logger.info("=" * 80)
-    logger.info("Starting Enterprise Agentic Reliability Framework (PATCHED VERSION)")
-    logger.info("=" * 80)
-    logger.info(f"Python version: {os.sys.version}")
-    logger.info(f"Total events in history: {enhanced_engine.event_store.count()}")
-    logger.info(f"Vector index size: {thread_safe_index.get_count() if thread_safe_index else 0}")
-    logger.info(f"Agents initialized: {len(enhanced_engine.orchestrator.agents)}")
-    logger.info(f"Policies loaded: {len(enhanced_engine.policy_engine.policies)}")
-    logger.info(f"Configuration: HF_TOKEN={'SET' if config.HF_TOKEN else 'NOT SET'}")
-    logger.info(f"Rate limit: {Constants.MAX_REQUESTS_PER_MINUTE} requests/minute")
-    logger.info("=" * 80)
-    
-    try:
-        demo = create_enhanced_ui()
-        
-        logger.info("Launching Gradio UI on 0.0.0.0:7860...")
-        demo.launch(
-            server_name="0.0.0.0",
-            server_port=7860,
-            share=False,
-            show_error=True
-        )
-    except KeyboardInterrupt:
-        logger.info("Received shutdown signal...")
-    except Exception as e:
-        logger.error(f"Application error: {e}", exc_info=True)
-    finally:
-        # Graceful shutdown
-        logger.info("Shutting down gracefully...")
-        
-        if thread_safe_index:
-            logger.info("Saving pending vectors before shutdown...")
-            thread_safe_index.shutdown()
-        
-        logger.info("=" * 80)
-        logger.info("Application shutdown complete")
-        logger.info("=" * 80)