Observability.

# metrics/naming.py

"""
Metric naming conventions and best practices.

Good metric names are:
- Descriptive: What does it measure?
- Consistent: Same format across services
- Unit-aware: Include the unit in the name
"""

from prometheus_client import Counter, Gauge, Histogram, Info
from typing import List

# =============================================================================
# NAMING CONVENTION
# =============================================================================
#
# Format: <namespace>_<subsystem>_<name>_<unit>
#
# Examples:
#   http_requests_total          (counter, no unit needed for counts)
#   http_request_duration_seconds (histogram with unit)
#   process_memory_bytes         (gauge with unit)
#   api_active_connections       (gauge, no unit for count)
#
# Labels should be:
#   - Low cardinality (not user_id, not request_id)
#   - Meaningful dimensions (method, status, endpoint)
#   - Consistent across metrics

# =============================================================================
# GOOD METRIC DEFINITIONS
# =============================================================================

# Counter for requests
http_requests_total = Counter(
    name='http_requests_total',
    documentation='Total number of HTTP requests',
    labelnames=['method', 'endpoint', 'status']
)

# Histogram for latency
http_request_duration_seconds = Histogram(
    name='http_request_duration_seconds',
    documentation='HTTP request latency in seconds',
    labelnames=['method', 'endpoint'],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
)

# Gauge for current state
db_connections_active = Gauge(
    name='db_connections_active',
    documentation='Number of active database connections',
    labelnames=['pool_name']
)

# Business metrics
orders_processed_total = Counter(
    name='orders_processed_total',
    documentation='Total orders processed',
    labelnames=['status', 'payment_method']
)

order_value_dollars = Histogram(
    name='order_value_dollars',
    documentation='Order value in dollars',
    labelnames=['currency'],
    buckets=[10, 25, 50, 100, 250, 500, 1000, 2500, 5000]
)


# =============================================================================
# BAD METRIC DEFINITIONS (DON'T DO THIS)
# =============================================================================

# ❌ BAD: High cardinality labels
# http_requests_total with labels: user_id, request_id, session_id
# This creates millions of time series!

# ❌ BAD: Inconsistent naming
# requestCount (camelCase)
# request-latency (dashes)
# Request_Duration_Ms (mixed case, abbreviated unit)

# ❌ BAD: Missing unit
# http_request_duration (seconds? milliseconds? who knows?)

# ❌ BAD: Too generic
# count (count of what?)
# duration (duration of what?)

# logging/structured.py

"""
Structured logging for observability.

Key principles:
1. JSON format for machine parsing
2. Consistent field names
3. Include context (trace_id, user_id, etc.)
4. Appropriate log levels
"""

import json
import logging
import sys
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from contextvars import ContextVar

# Context variables for request-scoped data
_request_context: ContextVar[Dict[str, Any]] = ContextVar(
    'request_context', 
    default={}
)


class StructuredFormatter(logging.Formatter):
    """
    Formats log records as JSON for structured logging.
    """
    
    STANDARD_FIELDS = [
        'timestamp',
        'level',
        'logger',
        'message',
        'trace_id',
        'span_id',
        'user_id',
        'tenant_id',
        'service',
        'environment',
    ]
    
    def __init__(self, service_name: str, environment: str):
        super().__init__()
        self.service_name = service_name
        self.environment = environment
    
    def format(self, record: logging.LogRecord) -> str:
        # Base fields
        log_entry = {
            'timestamp': datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
            'level': record.levelname,
            'logger': record.name,
            'message': record.getMessage(),
            'service': self.service_name,
            'environment': self.environment,
        }
        
        # Add context from request
        ctx = _request_context.get()
        if ctx:
            log_entry['trace_id'] = ctx.get('trace_id')
            log_entry['span_id'] = ctx.get('span_id')
            log_entry['user_id'] = ctx.get('user_id')
            log_entry['tenant_id'] = ctx.get('tenant_id')
            log_entry['request_id'] = ctx.get('request_id')
        
        # Add extra fields from the log call
        if hasattr(record, 'extra_fields'):
            log_entry.update(record.extra_fields)
        
        # Add exception info if present
        if record.exc_info:
            log_entry['exception'] = {
                'type': record.exc_info[0].__name__,
                'message': str(record.exc_info[1]),
                'traceback': self.formatException(record.exc_info)
            }
        
        # Add source location for errors
        if record.levelno >= logging.ERROR:
            log_entry['source'] = {
                'file': record.pathname,
                'line': record.lineno,
                'function': record.funcName
            }
        
        return json.dumps(log_entry, default=str)


class ContextLogger:
    """
    Logger that automatically includes context in all log messages.
    """
    
    def __init__(self, name: str):
        self.logger = logging.getLogger(name)
    
    def _log(self, level: int, message: str, **kwargs):
        """Log with extra fields."""
        # logger.log() applies the level check; stacklevel=3 makes the record
        # point at the caller of .info()/.error(), not at this wrapper
        self.logger.log(
            level,
            message,
            extra={'extra_fields': kwargs},
            stacklevel=3,
        )
    
    def debug(self, message: str, **kwargs):
        self._log(logging.DEBUG, message, **kwargs)
    
    def info(self, message: str, **kwargs):
        self._log(logging.INFO, message, **kwargs)
    
    def warning(self, message: str, **kwargs):
        self._log(logging.WARNING, message, **kwargs)
    
    def error(self, message: str, **kwargs):
        self._log(logging.ERROR, message, **kwargs)
    
    def critical(self, message: str, **kwargs):
        self._log(logging.CRITICAL, message, **kwargs)


def set_request_context(
    trace_id: str,
    span_id: str = None,
    user_id: str = None,
    tenant_id: str = None,
    request_id: str = None
):
    """Set context for the current request."""
    _request_context.set({
        'trace_id': trace_id,
        'span_id': span_id,
        'user_id': user_id,
        'tenant_id': tenant_id,
        'request_id': request_id
    })


# =============================================================================
# USAGE EXAMPLE
# =============================================================================

def setup_logging(service_name: str, environment: str):
    """Configure structured logging for the application."""
    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(StructuredFormatter(service_name, environment))
    
    root_logger = logging.getLogger()
    root_logger.addHandler(handler)
    root_logger.setLevel(logging.INFO)


# Example usage
logger = ContextLogger(__name__)

async def process_payment(payment_id: str, amount: float):
    """Process a payment with structured logging."""
    
    logger.info(
        "Processing payment",
        payment_id=payment_id,
        amount=amount,
        currency="USD"
    )
    
    try:
        result = await payment_gateway.charge(payment_id, amount)
        
        logger.info(
            "Payment successful",
            payment_id=payment_id,
            transaction_id=result.transaction_id,
            processing_time_ms=result.processing_time
        )
        
        return result
        
    except PaymentDeclinedException as e:
        logger.warning(
            "Payment declined",
            payment_id=payment_id,
            decline_reason=e.reason,
            decline_code=e.code
        )
        raise
        
    except Exception as e:
        logger.error(
            "Payment processing failed",
            payment_id=payment_id,
            error_type=type(e).__name__,
            error_message=str(e)
        )
        raise


# Output example (single line, formatted here for readability):
# {
#   "timestamp": "2024-01-15T10:30:00.000Z",
#   "level": "INFO",
#   "logger": "payment_service",
#   "message": "Processing payment",
#   "service": "payment-api",
#   "environment": "production",
#   "trace_id": "abc-123-def-456",
#   "user_id": "user-789",
#   "tenant_id": "tenant-001",
#   "payment_id": "pay-12345",
#   "amount": 99.99,
#   "currency": "USD"
# }

# tracing/opentelemetry_setup.py

"""
OpenTelemetry tracing setup and utilities.

OpenTelemetry is the industry standard for tracing.
It provides consistent APIs across languages.
"""

from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.trace import Status, StatusCode
from opentelemetry.propagate import set_global_textmap
from opentelemetry.propagators.b3 import B3MultiFormat
from contextlib import contextmanager
from typing import Optional, Dict, Any
import asyncio
import functools


def setup_tracing(
    service_name: str,
    otlp_endpoint: str = "localhost:4317",
    environment: str = "development"
):
    """
    Configure OpenTelemetry tracing for the application.
    """
    # Create tracer provider
    provider = TracerProvider(
        resource=Resource.create({
            "service.name": service_name,
            "deployment.environment": environment,
        })
    )
    
    # Export spans to collector (Jaeger, Zipkin, etc.)
    exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
    provider.add_span_processor(BatchSpanProcessor(exporter))
    
    # Set as global provider
    trace.set_tracer_provider(provider)
    
    # Set up context propagation (B3 format for compatibility)
    set_global_textmap(B3MultiFormat())
    
    # Auto-instrument common libraries
    FastAPIInstrumentor().instrument()
    HTTPXClientInstrumentor().instrument()
    SQLAlchemyInstrumentor().instrument()
    RedisInstrumentor().instrument()
    
    return trace.get_tracer(service_name)


class Tracing:
    """
    Tracing utilities for manual instrumentation.
    """
    
    def __init__(self, service_name: str):
        self.tracer = trace.get_tracer(service_name)
    
    @contextmanager
    def span(
        self,
        name: str,
        attributes: Optional[Dict[str, Any]] = None
    ):
        """
        Create a span within the current trace context.
        
        Usage:
            with tracing.span("process_payment", {"payment_id": "123"}):
                # ... do work ...
        """
        with self.tracer.start_as_current_span(name) as span:
            if attributes:
                for key, value in attributes.items():
                    span.set_attribute(key, value)
            
            try:
                yield span
            except Exception as e:
                span.set_status(Status(StatusCode.ERROR, str(e)))
                span.record_exception(e)
                raise
    
    def traced(
        self,
        name: Optional[str] = None,
        attributes: Optional[Dict[str, Any]] = None
    ):
        """
        Decorator to trace a function.
        
        Usage:
            @tracing.traced("process_order")
            async def process_order(order_id: str):
                ...
        """
        def decorator(func):
            span_name = name or func.__name__
            
            @functools.wraps(func)
            async def async_wrapper(*args, **kwargs):
                with self.span(span_name, attributes):
                    return await func(*args, **kwargs)
            
            @functools.wraps(func)
            def sync_wrapper(*args, **kwargs):
                with self.span(span_name, attributes):
                    return func(*args, **kwargs)
            
            if asyncio.iscoroutinefunction(func):
                return async_wrapper
            return sync_wrapper
        
        return decorator
    
    def add_event(self, name: str, attributes: Dict[str, Any] = None):
        """Add an event to the current span."""
        span = trace.get_current_span()
        span.add_event(name, attributes or {})
    
    def set_attribute(self, key: str, value: Any):
        """Set an attribute on the current span."""
        span = trace.get_current_span()
        span.set_attribute(key, value)
    
    def get_trace_id(self) -> Optional[str]:
        """Get the current trace ID."""
        ctx = trace.get_current_span().get_span_context()
        if ctx.is_valid:
            return format(ctx.trace_id, '032x')
        return None


# =============================================================================
# USAGE EXAMPLE
# =============================================================================

tracing = Tracing("payment-service")


@tracing.traced("process_payment")
async def process_payment(payment_id: str, amount: float):
    """Process a payment with full tracing."""
    
    # Add attributes to span
    tracing.set_attribute("payment.id", payment_id)
    tracing.set_attribute("payment.amount", amount)
    
    # Validate payment
    with tracing.span("validate_payment"):
        validation_result = await validate_payment(payment_id, amount)
        if not validation_result.valid:
            tracing.add_event("validation_failed", {
                "reason": validation_result.reason
            })
            raise ValidationError(validation_result.reason)
    
    # Check fraud
    with tracing.span("fraud_check", {"payment_id": payment_id}):
        fraud_score = await fraud_service.check(payment_id)
        tracing.set_attribute("fraud.score", fraud_score)
    
    # Process with payment gateway
    with tracing.span("payment_gateway_call"):
        tracing.add_event("calling_stripe")
        result = await stripe_client.charge(payment_id, amount)
        tracing.add_event("stripe_response", {
            "transaction_id": result.transaction_id
        })
    
    return result

# tracing/propagation.py

"""
Trace context propagation across service boundaries.

When Service A calls Service B, how does Service B know
to continue the same trace?

Answer: Propagate trace context in HTTP headers.
"""

from opentelemetry import trace
from opentelemetry.propagate import inject, extract
from typing import Dict
import httpx


class TracedHTTPClient:
    """
    HTTP client that propagates trace context.
    """
    
    def __init__(self):
        self.client = httpx.AsyncClient()
    
    async def request(
        self,
        method: str,
        url: str,
        **kwargs
    ) -> httpx.Response:
        """Make request with trace context."""
        
        # Get current trace context
        headers = kwargs.get('headers', {})
        
        # Inject trace context into headers
        inject(headers)
        
        kwargs['headers'] = headers
        
        # Make request (trace context now in headers)
        return await self.client.request(method, url, **kwargs)
    
    async def get(self, url: str, **kwargs) -> httpx.Response:
        return await self.request("GET", url, **kwargs)
    
    async def post(self, url: str, **kwargs) -> httpx.Response:
        return await self.request("POST", url, **kwargs)


# With the default W3C Trace Context propagator, the headers look like:
# {
#   "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
#   "tracestate": "congo=t61rcWkgMzE"
# }
#
# With the B3 propagator that setup_tracing() above installs, they look like:
# {
#   "X-B3-TraceId": "4bf92f3577b34da6a3ce929d0e0e4736",
#   "X-B3-SpanId": "00f067aa0ba902b7",
#   "X-B3-Sampled": "1"
# }


class TracedFastAPIMiddleware:
    """
    Middleware that extracts trace context from incoming requests.
    """
    
    async def __call__(self, request, call_next):
        # Extract trace context from headers
        context = extract(request.headers)
        
        # Continue trace or start new one
        tracer = trace.get_tracer(__name__)
        
        with tracer.start_as_current_span(
            f"{request.method} {request.url.path}",
            context=context,
            kind=trace.SpanKind.SERVER
        ) as span:
            # Add request attributes
            span.set_attribute("http.request.method", request.method)
            span.set_attribute("url.full", str(request.url))
            span.set_attribute("http.route", request.url.path)
            
            response = await call_next(request)
            
            # Add response attributes
            span.set_attribute("http.response.status_code", response.status_code)
            
            return response

# observability/fastapi_setup.py

"""
Complete observability setup for a FastAPI application.
"""

from fastapi import FastAPI, Request, Response
from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
from opentelemetry import trace
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
import time
import logging
import json
import uuid
from typing import Callable
from contextvars import ContextVar

# =============================================================================
# METRICS SETUP
# =============================================================================

# Request metrics (RED method)
REQUEST_COUNT = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status']
)

REQUEST_LATENCY = Histogram(
    'http_request_duration_seconds',
    'HTTP request latency',
    ['method', 'endpoint'],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
)

REQUESTS_IN_PROGRESS = Gauge(
    'http_requests_in_progress',
    'HTTP requests currently being processed',
    ['method', 'endpoint']
)

# Business metrics
ORDERS_CREATED = Counter(
    'orders_created_total',
    'Total orders created',
    ['status', 'payment_method']
)

ORDER_VALUE = Histogram(
    'order_value_dollars',
    'Order value in dollars',
    buckets=[10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000]
)


# =============================================================================
# LOGGING SETUP
# =============================================================================

_request_id: ContextVar[str] = ContextVar('request_id', default='')
_trace_id: ContextVar[str] = ContextVar('trace_id', default='')


class ObservabilityMiddleware:
    """
    Middleware that instruments requests with metrics, logs, and traces.
    """
    
    def __init__(self, app: FastAPI):
        self.app = app
    
    async def __call__(self, scope, receive, send):
        if scope['type'] != 'http':
            await self.app(scope, receive, send)
            return
        
        request = Request(scope, receive)
        method = request.method
        path = self._get_path_template(request)
        
        # Generate request ID
        request_id = request.headers.get('X-Request-ID', str(uuid.uuid4()))
        _request_id.set(request_id)
        
        # Get trace ID from OpenTelemetry
        ctx = trace.get_current_span().get_span_context()
        trace_id = format(ctx.trace_id, '032x') if ctx.is_valid else ''
        _trace_id.set(trace_id)
        
        # Track in-progress requests
        REQUESTS_IN_PROGRESS.labels(method=method, endpoint=path).inc()
        
        start_time = time.perf_counter()
        status_code = 500  # Default if exception occurs
        
        async def send_wrapper(message):
            nonlocal status_code
            if message['type'] == 'http.response.start':
                status_code = message['status']
            await send(message)
        
        try:
            # Log request
            logging.info(json.dumps({
                'event': 'request_started',
                'method': method,
                'path': str(request.url),
                'request_id': request_id,
                'trace_id': trace_id,
                'user_agent': request.headers.get('User-Agent', ''),
            }))
            
            await self.app(scope, receive, send_wrapper)
            
        finally:
            # Calculate duration
            duration = time.perf_counter() - start_time
            
            # Record metrics
            REQUEST_COUNT.labels(
                method=method,
                endpoint=path,
                status=str(status_code)
            ).inc()
            
            REQUEST_LATENCY.labels(
                method=method,
                endpoint=path
            ).observe(duration)
            
            REQUESTS_IN_PROGRESS.labels(method=method, endpoint=path).dec()
            
            # Log response
            logging.info(json.dumps({
                'event': 'request_completed',
                'method': method,
                'path': path,
                'status_code': status_code,
                'duration_ms': round(duration * 1000, 2),
                'request_id': request_id,
                'trace_id': trace_id,
            }))
    
    def _get_path_template(self, request: Request) -> str:
        """Get path template for metric labels (avoid high cardinality)."""
        # Use the route path template if available
        if hasattr(request, 'scope') and 'route' in request.scope:
            route = request.scope['route']
            if hasattr(route, 'path'):
                return route.path
        
        # Fallback: normalize the path
        path = request.url.path
        
        # Replace UUIDs with placeholder
        import re
        path = re.sub(
            r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
            '{id}',
            path
        )
        
        # Replace numeric IDs
        path = re.sub(r'/\d+', '/{id}', path)
        
        return path


def create_app() -> FastAPI:
    """Create FastAPI app with full observability."""
    app = FastAPI()
    
    # Add observability middleware
    app.add_middleware(ObservabilityMiddleware)
    
    # Instrument with OpenTelemetry
    FastAPIInstrumentor.instrument_app(app)
    
    # Metrics endpoint
    @app.get('/metrics')
    async def metrics():
        return Response(
            content=generate_latest(),
            media_type=CONTENT_TYPE_LATEST
        )
    
    # Health check (traced too, unless excluded: instrument_app(app, excluded_urls="health"))
    @app.get('/health')
    async def health():
        return {'status': 'healthy'}
    
    return app

# alerting/rules.py

"""
Alert rule definitions following best practices.

Good alerts are:
- Actionable: Someone should DO something
- Urgent: It matters NOW
- Clear: The problem is obvious
- Linked: Points to runbook
"""

from dataclasses import dataclass
from typing import List, Optional
from enum import Enum


class Severity(Enum):
    CRITICAL = "critical"  # Page immediately, any time
    WARNING = "warning"    # Page during business hours, or ticket
    INFO = "info"          # Log, review later


@dataclass
class AlertRule:
    name: str
    expression: str
    duration: str  # How long condition must be true
    severity: Severity
    summary: str
    description: str
    runbook_url: str
    labels: dict = None


# =============================================================================
# GOOD ALERT EXAMPLES
# =============================================================================

ALERT_RULES: List[AlertRule] = [
    
    # SLO-based alert: Error budget burning fast
    AlertRule(
        name="ErrorBudgetBurnRateCritical",
        expression="""
            (
              sum(rate(http_requests_total{status=~"5.."}[1h]))
              /
              sum(rate(http_requests_total[1h]))
            ) > 14.4 * 0.001
            and
            (
              sum(rate(http_requests_total{status=~"5.."}[5m]))
              /
              sum(rate(http_requests_total[5m]))
            ) > 14.4 * 0.001
        """,
        duration="5m",
        severity=Severity.CRITICAL,
        summary="Error budget burning at critical rate",
        description="Error rate is 14.4x the SLO threshold. "
                   "That burns 2% of the 30-day budget per hour (all of it in ~2 days).",
        runbook_url="https://wiki/runbooks/error-budget-burn",
    ),
    
    # Symptom-based alert: Users experiencing errors
    AlertRule(
        name="HighErrorRate",
        expression="""
            sum(rate(http_requests_total{status=~"5.."}[5m]))
            /
            sum(rate(http_requests_total[5m]))
            > 0.05
        """,
        duration="5m",
        severity=Severity.CRITICAL,
        summary="High error rate: {{ $value | humanizePercentage }}",
        description="More than 5% of requests are failing. "
                   "Users are being impacted.",
        runbook_url="https://wiki/runbooks/high-error-rate",
    ),
    
    # Latency alert
    AlertRule(
        name="HighLatency",
        expression="""
            histogram_quantile(0.99,
              sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
            ) > 1
        """,
        duration="5m",
        severity=Severity.WARNING,
        summary="p99 latency above 1 second",
        description="99th percentile latency is {{ $value }}s. "
                   "Users may experience slow responses.",
        runbook_url="https://wiki/runbooks/high-latency",
    ),
    
    # Saturation alert: Resource exhaustion warning
    AlertRule(
        name="DatabaseConnectionPoolNearExhaustion",
        expression="""
            db_connections_active / db_connections_max > 0.8
        """,
        duration="10m",
        severity=Severity.WARNING,
        summary="Database connection pool at {{ $value | humanizePercentage }}",
        description="Connection pool is nearly exhausted. "
                   "Requests may start failing if this continues.",
        runbook_url="https://wiki/runbooks/db-connection-pool",
    ),
    
    # Dependency alert
    AlertRule(
        name="PaymentProviderErrors",
        expression="""
            sum(rate(external_api_requests_total{
              service="payment-provider",
              status="error"
            }[5m]))
            /
            sum(rate(external_api_requests_total{
              service="payment-provider"
            }[5m]))
            > 0.1
        """,
        duration="5m",
        severity=Severity.CRITICAL,
        summary="Payment provider error rate high",
        description="10%+ of payment provider requests failing. "
                   "Payments may be impacted.",
        runbook_url="https://wiki/runbooks/payment-provider-errors",
    ),
]


# =============================================================================
# BAD ALERT EXAMPLES (DON'T DO THIS)
# =============================================================================

BAD_ALERTS = """

❌ BAD: CPU alert
   expression: cpu_usage > 80
   
   Why bad:
   - High CPU might be fine if latency is good
   - Not actionable (what do you do?)
   - Doesn't indicate user impact
   
   Better: Alert on latency or error rate

❌ BAD: Any error alert
   expression: error_count > 0
   
   Why bad:
   - Some errors are normal
   - Extremely noisy
   - Desensitizes on-call
   
   Better: Alert on error RATE exceeding threshold

❌ BAD: Missing runbook
   summary: "Something went wrong"
   runbook: (none)
   
   Why bad:
   - On-call can't help if they don't know what to do
   - Increases mean time to recovery
   
   Better: Always include runbook link

❌ BAD: Unclear scope
   summary: "Errors detected"
   
   Why bad:
   - Which service?
   - Which endpoint?
   - How bad?
   
   Better: "Payment API: 5% error rate affecting checkout"

"""

# Runbook: High Error Rate

## Alert Details
- **Alert name**: HighErrorRate
- **Severity**: Critical
- **Summary**: HTTP error rate exceeds 5%

## Quick Assessment (< 2 minutes)

1. **Check scope**: Is this one endpoint or all endpoints?
   ```
   Query: sum by (endpoint) (rate(http_requests_total{status=~"5.."}[5m]))
   ```

2. **Check timeline**: When did it start?
   - Look at dashboard for change point
   - Check recent deployments

3. **Check error types**: What's failing?
   ```
   Query: sum by (endpoint, status) (rate(http_requests_total{status=~"5.."}[5m]))
   ```

## Common Causes and Fixes

### Cause 1: Recent deployment broke something
**Symptoms**:
- Errors started at deployment time
- Specific endpoint failing

**Fix**:
```bash
# Rollback to previous version
kubectl rollout undo deployment/api-server
```

### Cause 2: Downstream service is down
**Symptoms**:
- Errors are timeouts or connection refused
- Logs show connection errors to specific service

**Fix**:
1. Check downstream service status
2. If down, check THEIR runbooks
3. Consider enabling fallback/circuit breaker

### Cause 3: Database is overloaded
**Symptoms**:
- Errors are timeouts
- Database CPU/connections high

**Fix**:
1. Check slow query log
2. Kill long-running queries if safe
3. Consider scaling read replicas

## Escalation
- If not resolved in 15 minutes: Page senior on-call
- If customer-reported: Loop in customer support
- If data loss suspected: Page database team

## Post-Incident
- [ ] Incident documented
- [ ] Timeline recorded
- [ ] Root cause identified
- [ ] Follow-up action items created

{
  "trace_id": "abc-123",
  "span": "database-query",
  "query": "SELECT * FROM products WHERE id IN (...)",
  "duration_ms": 2000,
  "rows_returned": 500
}