Incident Management.

# incident/alerting.py

"""
Alert design for effective on-call.

Good alerts are actionable and important.
Bad alerts cause fatigue and get ignored.
"""

from dataclasses import dataclass
from typing import List, Optional
from enum import Enum


class AlertSeverity(Enum):
    PAGE = "page"          # Wake someone up
    NOTIFY = "notify"      # Slack/email during business hours
    LOG = "log"            # Just log it


@dataclass
class AlertDefinition:
    """A well-designed alert."""
    name: str
    description: str
    severity: AlertSeverity
    
    # The query/condition
    condition: str
    
    # How long condition must be true before alerting
    duration: str
    
    # Runbook for responders
    runbook_url: str
    
    # Who to notify
    notify_channel: str
    
    # Tags for filtering
    tags: List[str]


# =============================================================================
# GOOD ALERT EXAMPLES
# =============================================================================

GOOD_ALERTS = [
    # This alert is ACTIONABLE and IMPORTANT
    AlertDefinition(
        name="CheckoutErrorRateHigh",
        description="Checkout error rate exceeds SLO, customers cannot complete purchases",
        severity=AlertSeverity.PAGE,
        condition='sum(rate(checkout_errors_total[5m])) / sum(rate(checkout_requests_total[5m])) > 0.01',
        duration="5m",
        runbook_url="https://wiki/runbooks/checkout-errors",
        notify_channel="payments-oncall",
        tags=["checkout", "revenue", "sev1"]
    ),
    
    # This alert has clear user impact
    AlertDefinition(
        name="APILatencyP99High",
        description="API p99 latency exceeds 1 second, users experiencing slow responses",
        severity=AlertSeverity.PAGE,
        condition='histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 1',
        duration="10m",
        runbook_url="https://wiki/runbooks/high-latency",
        notify_channel="api-oncall",
        tags=["latency", "user-experience"]
    ),
    
    # This is a WARNING, not a page
    AlertDefinition(
        name="DiskSpaceLow",
        description="Disk space below 20%, will need attention soon",
        severity=AlertSeverity.NOTIFY,
        condition='(node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.2',
        duration="30m",
        runbook_url="https://wiki/runbooks/disk-space",
        notify_channel="infrastructure",
        tags=["disk", "capacity"]
    ),
]


# =============================================================================
# BAD ALERT EXAMPLES (DON'T DO THIS)
# =============================================================================

BAD_ALERTS = """

❌ BAD: No runbook
   AlertDefinition(
       name="SomethingWrong",
       runbook_url=None  # Responder has no idea what to do
   )

❌ BAD: Too sensitive
   AlertDefinition(
       condition='error_count > 0',  # Any single error pages someone
       duration="0m"  # No buffer for transient issues
   )

❌ BAD: Not actionable
   AlertDefinition(
       name="CPUHigh",
       condition='cpu > 80%',  # So what? What should they do?
       # High CPU might be fine if latency is OK
   )

❌ BAD: Missing context
   AlertDefinition(
       name="Error",
       description="An error occurred"  # Which error? Where? Impact?
   )

❌ BAD: Pages for non-urgent issues
   AlertDefinition(
       name="SlowBackgroundJob",
       severity=AlertSeverity.PAGE,  # This can wait until morning
       description="Nightly report generation is slow"
   )

"""


# =============================================================================
# ALERT REVIEW CHECKLIST
# =============================================================================

ALERT_REVIEW = """
Before adding an alert, ask:

1. IS IT ACTIONABLE?
   □ Is there a runbook?
   □ Can the on-call actually fix it?
   □ Or do they just have to wait?

2. IS IT URGENT?
   □ Does it need to wake someone up?
   □ Or can it wait until morning?
   □ What's the actual user impact?

3. IS IT CLEAR?
   □ Does the name explain the problem?
   □ Does the description explain the impact?
   □ Can someone unfamiliar understand it?

4. IS THE THRESHOLD RIGHT?
   □ Will it fire for real problems?
   □ Will it fire for non-problems? (false positives)
   □ Is the duration appropriate?

5. DOES IT REDUCE NOISE?
   □ Can it be consolidated with similar alerts?
   □ Is it duplicating another alert?
   □ Will it cause alert fatigue?

If you can't answer YES to all of these,
the alert needs more work.
"""

# Runbook: High Error Rate on Payment Service

## Overview

**Alert**: PaymentServiceErrorRateHigh
**Severity**: SEV1 (pages immediately)
**User Impact**: Customers cannot complete purchases
**Revenue Impact**: ~$10,000/hour during peak

## Quick Assessment (< 2 min)

1. **Open dashboard**: https://grafana.internal/d/payments
2. **Check error rate graph**: Is it actually elevated or a spike that resolved?
3. **Check scope**: All endpoints or specific one?
   ```
   Query: sum by (endpoint) (rate(payment_errors_total[5m]))
   ```
4. **Check timeline**: When did it start? Any correlation with deployments?

If error rate is < 1% and stable, this may be a false alarm. Monitor for 5 min.

## Common Causes and Fixes

### Cause 1: Recent Deployment

**How to identify**:
- Check deployment history: https://deploy.internal/payments
- Did error rate spike after a deployment?

**How to fix**:
```bash
# Rollback to previous version
kubectl rollout undo deployment/payment-service -n production

# Verify rollback
kubectl rollout status deployment/payment-service -n production
```

**Expected result**: Error rate should drop within 2-3 minutes.

---

### Cause 2: Database Connection Pool Exhausted

**How to identify**:
- Check DB connection metrics in dashboard
- Look for "connection refused" or "timeout" in logs
  ```bash
  kubectl logs -l app=payment-service -n production | grep -i "connection"
  ```

**How to fix**:
1. Scale up payment service (reduces load per pod):
   ```bash
   kubectl scale deployment/payment-service --replicas=10 -n production
   ```

2. If that doesn't help, check if slow queries are holding connections:
   ```sql
   SELECT pid, now() - pg_stat_activity.query_start AS duration, query
   FROM pg_stat_activity
   WHERE state = 'active' AND now() - pg_stat_activity.query_start > interval '30 seconds';
   ```

3. Kill long-running queries if safe:
   ```sql
   SELECT pg_terminate_backend(pid) FROM pg_stat_activity
   WHERE state = 'active'
     AND now() - query_start > interval '2 minutes'
     AND pid <> pg_backend_pid();
   ```

---

### Cause 3: Payment Provider Outage

**How to identify**:
- Check payment provider status page: https://status.stripe.com
- Check external API latency in dashboard
- Look for timeout errors in logs

**How to fix**:
1. If provider has fallback, enable it:
   ```bash
   # Enable Adyen as fallback
   kubectl set env deployment/payment-service PAYMENT_FALLBACK_ENABLED=true
   ```

2. If no fallback, consider enabling maintenance mode:
   ```bash
   # Enable maintenance mode for checkout
   kubectl set env deployment/api-gateway CHECKOUT_MAINTENANCE=true
   ```

3. Update status page with provider attribution.

---

### Cause 4: Traffic Spike Overwhelming System

**How to identify**:
- Check request rate: is it much higher than normal?
- Check if auto-scaling is running but not keeping up

**How to fix**:
1. Manually scale up:
   ```bash
   kubectl scale deployment/payment-service --replicas=20 -n production
   ```

2. Enable rate limiting if needed:
   ```bash
   kubectl set env deployment/api-gateway RATE_LIMIT_ENABLED=true
   ```

## Escalation

**Escalate if**:
- Issue not resolved within 15 minutes
- Root cause unclear
- Database expertise needed
- Customer communication needed

**Escalate to**:
- Primary: @payments-team-lead (Slack)
- Secondary: @platform-lead
- SEV1 escalation: Page secondary on-call

**Information to provide**:
- Current error rate and duration
- Steps already taken
- Suspected cause (if any)
- Links to relevant logs/dashboards

## Communication

**Status page update** (if > 10 min duration):
```
Title: Payment Processing Degraded
Body: Some customers may experience errors when completing purchases. 
Our team is actively investigating. We will provide updates every 15 minutes.
```

**Internal notification**:
- Slack: #incidents
- Include: Error rate, start time, impact, current status

## Post-Incident

- [ ] Document timeline in incident channel
- [ ] Create postmortem if SEV1/SEV2
- [ ] Update runbook if new cause found
- [ ] Create follow-up tickets for improvements

# Postmortem: [Incident Title]

## Incident Summary

| Field | Value |
|-------|-------|
| **Date** | YYYY-MM-DD |
| **Duration** | X hours Y minutes |
| **Severity** | SEV1 / SEV2 / SEV3 |
| **Authors** | [Names] |
| **Status** | Draft / In Review / Complete |

### One-Line Summary
[One sentence describing what happened and the impact]

### Impact
- **User Impact**: [What users experienced]
- **Duration**: [How long they were impacted]
- **Scope**: [How many users affected]
- **Revenue Impact**: [$X estimated]
- **Data Impact**: [Any data loss/corruption]

---

## Timeline (All times in UTC)

| Time | Event |
|------|-------|
| 14:30 | Deployment of v2.3.1 to production |
| 14:35 | Error rate begins increasing |
| 14:42 | **DETECTED**: Alert fires for high error rate |
| 14:45 | **ENGAGED**: On-call acknowledges and begins investigation |
| 14:50 | Identified correlation with recent deployment |
| 14:55 | Rollback initiated |
| 15:00 | **MITIGATED**: Error rate back to normal |
| 15:15 | **RESOLVED**: Confirmed service fully restored |

**Key Metrics**:
- Time to Detect (TTD): 7 minutes (14:35 → 14:42)
- Time to Engage (TTE): 3 minutes (14:42 → 14:45)
- Time to Mitigate (TTM): 18 minutes from detection (14:42 → 15:00); rollback started at 13 minutes
- Total Incident Duration: 40 minutes (14:35 → 15:15)

---

## Root Cause Analysis

### What Happened
[Detailed technical explanation of what went wrong]

### Why It Happened (5 Whys)

1. **Why** did the service start returning errors?
   - Because database queries were timing out.

2. **Why** were queries timing out?
   - Because the new code introduced N+1 queries.

3. **Why** did the new code have N+1 queries?
   - Because the ORM change wasn't caught in code review.

4. **Why** wasn't it caught in code review?
   - Because we don't have automated detection for N+1 queries.

5. **Why** don't we have automated detection?
   - Because we haven't prioritized query analysis tooling.

**Root Cause**: Lack of automated N+1 query detection in our CI/CD pipeline.

### Contributing Factors
- Code review didn't catch the ORM change impact
- Load testing was skipped due to time pressure
- Staging environment doesn't have production-scale data

---

## What Went Well
- Alert fired quickly (7 min detection)
- On-call responded promptly
- Rollback procedure worked smoothly
- Communication to stakeholders was timely
- No data loss or corruption

## What Went Poorly
- N+1 query issue wasn't caught before production
- Took 5 minutes to link the errors to the deployment (the N+1 root cause was found later)
- There was no automated canary analysis, which would have caught this
- Staging data is not representative of production

---

## Action Items

| ID | Action | Owner | Priority | Due Date | Status |
|----|--------|-------|----------|----------|--------|
| 1 | Add N+1 query detection to CI | @alice | P1 | 2024-02-01 | TODO |
| 2 | Implement canary analysis for deployments | @bob | P1 | 2024-02-15 | TODO |
| 3 | Create production-like dataset for staging | @charlie | P2 | 2024-02-28 | TODO |
| 4 | Add ORM query patterns to code review checklist | @alice | P2 | 2024-02-01 | TODO |
| 5 | Document N+1 query patterns in eng wiki | @alice | P3 | 2024-02-07 | TODO |

---

## Lessons Learned

1. **Our deployment pipeline needs more automated checks**
   - We should catch performance regressions before production

2. **Rollback speed is critical**
   - 5-minute rollback time significantly reduced impact

3. **Staging != Production**
   - Data scale differences hide performance issues

---

## Appendix

### Relevant Logs
```
[Link to log search for incident timeframe]
```

### Dashboards
- [Link to metrics dashboard during incident]

### Related Incidents
- [Link to similar past incidents if any]

# incident/workflow.py

"""
Incident management workflow system.

Coordinates incident response from detection to resolution.
"""

from dataclasses import dataclass, field
from typing import List, Optional, Dict
from datetime import datetime, timezone
from enum import Enum
import uuid


class IncidentStatus(Enum):
    DETECTED = "detected"
    TRIAGING = "triaging"
    INVESTIGATING = "investigating"
    MITIGATING = "mitigating"
    MITIGATED = "mitigated"  # User impact stopped; the full fix may still be pending
    RESOLVED = "resolved"
    CLOSED = "closed"


class IncidentSeverity(Enum):
    SEV1 = 1  # Critical - all hands
    SEV2 = 2  # High - immediate attention
    SEV3 = 3  # Medium - business hours
    SEV4 = 4  # Low - normal queue


@dataclass
class IncidentRole:
    """A role assignment in an incident."""
    role: str  # "incident_commander", "technical_lead", "comms_lead"
    user_id: str
    assigned_at: datetime


@dataclass
class TimelineEvent:
    """An event in the incident timeline."""
    timestamp: datetime
    event_type: str  # "status_change", "action", "communication", "note"
    description: str
    user_id: str
    metadata: Dict = field(default_factory=dict)


@dataclass
class Incident:
    """An incident record."""
    id: str
    title: str
    description: str
    severity: IncidentSeverity
    status: IncidentStatus
    
    # Timing
    detected_at: datetime
    engaged_at: Optional[datetime] = None
    mitigated_at: Optional[datetime] = None
    resolved_at: Optional[datetime] = None
    
    # People
    roles: List[IncidentRole] = field(default_factory=list)
    
    # Timeline
    timeline: List[TimelineEvent] = field(default_factory=list)
    
    # Affected services
    affected_services: List[str] = field(default_factory=list)
    
    # Links
    slack_channel: Optional[str] = None
    video_call_link: Optional[str] = None
    postmortem_link: Optional[str] = None
    
    # Metadata
    tags: List[str] = field(default_factory=list)
    customer_impact: Optional[str] = None
    
    @property
    def time_to_detect(self) -> Optional[float]:
        """Minutes from actual issue start to detection."""
        # This would require knowing when issue actually started
        return None
    
    @property
    def time_to_engage(self) -> Optional[float]:
        """Minutes from detection to first responder."""
        if self.engaged_at:
            return (self.engaged_at - self.detected_at).total_seconds() / 60
        return None
    
    @property
    def time_to_mitigate(self) -> Optional[float]:
        """Minutes from detection to impact mitigation."""
        if self.mitigated_at:
            return (self.mitigated_at - self.detected_at).total_seconds() / 60
        return None
    
    @property
    def time_to_resolve(self) -> Optional[float]:
        """Minutes from detection to full resolution."""
        if self.resolved_at:
            return (self.resolved_at - self.detected_at).total_seconds() / 60
        return None


def _fmt_minutes(value: Optional[float]) -> str:
    """Format a duration in minutes; a milestone that never happened shows as n/a."""
    return "n/a" if value is None else f"{value:.0f} minutes"


class IncidentManager:
    """
    Manages incident lifecycle and coordination.
    """
    
    def __init__(
        self,
        slack_client,
        pagerduty_client,
        status_page_client,
        database
    ):
        self.slack = slack_client
        self.pagerduty = pagerduty_client
        self.status_page = status_page_client
        self.db = database
    
    async def create_incident(
        self,
        title: str,
        description: str,
        severity: IncidentSeverity,
        affected_services: List[str],
        triggered_by: str
    ) -> Incident:
        """
        Create a new incident and initiate response.
        """
        incident = Incident(
            id=str(uuid.uuid4()),
            title=title,
            description=description,
            severity=severity,
            status=IncidentStatus.DETECTED,
            detected_at=datetime.now(timezone.utc),
            affected_services=affected_services
        )
        
        # Add creation event
        incident.timeline.append(TimelineEvent(
            timestamp=datetime.now(timezone.utc),
            event_type="status_change",
            description=f"Incident created: {title}",
            user_id=triggered_by
        ))
        
        # Create incident channel
        channel = await self._create_incident_channel(incident)
        incident.slack_channel = channel
        
        # Page on-call if SEV1 or SEV2
        if severity in [IncidentSeverity.SEV1, IncidentSeverity.SEV2]:
            await self._page_on_call(incident)
        
        # Update status page if SEV1
        if severity == IncidentSeverity.SEV1:
            await self._create_status_page_incident(incident)
        
        # Save to database
        await self.db.save_incident(incident)
        
        return incident
    
    async def assign_role(
        self,
        incident_id: str,
        role: str,
        user_id: str,
        assigned_by: str
    ):
        """Assign a role to a user."""
        incident = await self.db.get_incident(incident_id)
        
        # Remove existing role assignment if any
        incident.roles = [r for r in incident.roles if r.role != role]
        
        # Add new assignment
        incident.roles.append(IncidentRole(
            role=role,
            user_id=user_id,
            assigned_at=datetime.now(timezone.utc)
        ))
        
        incident.timeline.append(TimelineEvent(
            timestamp=datetime.now(timezone.utc),
            event_type="action",
            description=f"Assigned {role} to {user_id}",
            user_id=assigned_by
        ))
        
        await self.db.save_incident(incident)
        await self._notify_channel(
            incident.slack_channel,
            f"🎯 {user_id} is now {role}"
        )
    
    async def update_status(
        self,
        incident_id: str,
        new_status: IncidentStatus,
        message: str,
        user_id: str
    ):
        """Update incident status."""
        incident = await self.db.get_incident(incident_id)
        old_status = incident.status
        incident.status = new_status
        
        # Update timing fields
        now = datetime.now(timezone.utc)
        if new_status == IncidentStatus.INVESTIGATING and not incident.engaged_at:
            incident.engaged_at = now
        elif new_status == IncidentStatus.MITIGATED and not incident.mitigated_at:
            # TTM ends when impact stops, not when mitigation work starts
            incident.mitigated_at = now
        elif new_status == IncidentStatus.RESOLVED and not incident.resolved_at:
            incident.resolved_at = now
            if not incident.mitigated_at:
                incident.mitigated_at = now  # Resolving also ends the impact
        
        # Add timeline event
        incident.timeline.append(TimelineEvent(
            timestamp=now,
            event_type="status_change",
            description=f"Status: {old_status.value} → {new_status.value}. {message}",
            user_id=user_id
        ))
        
        await self.db.save_incident(incident)
        
        # Notify channel
        status_emoji = {
            IncidentStatus.INVESTIGATING: "🔍",
            IncidentStatus.MITIGATING: "🔧",
            IncidentStatus.MITIGATED: "🩹",
            IncidentStatus.RESOLVED: "✅",
        }.get(new_status, "📋")
        
        await self._notify_channel(
            incident.slack_channel,
            f"{status_emoji} Status update: {new_status.value}\n> {message}"
        )
        
        # Update status page
        if incident.severity == IncidentSeverity.SEV1:
            await self._update_status_page(incident, message)
    
    async def add_note(
        self,
        incident_id: str,
        note: str,
        user_id: str
    ):
        """Add a note to the incident timeline."""
        incident = await self.db.get_incident(incident_id)
        
        incident.timeline.append(TimelineEvent(
            timestamp=datetime.now(timezone.utc),
            event_type="note",
            description=note,
            user_id=user_id
        ))
        
        await self.db.save_incident(incident)
        await self._notify_channel(
            incident.slack_channel,
            f"📝 Note from {user_id}: {note}"
        )
    
    async def resolve_incident(
        self,
        incident_id: str,
        resolution_summary: str,
        user_id: str
    ):
        """Resolve the incident."""
        await self.update_status(
            incident_id,
            IncidentStatus.RESOLVED,
            resolution_summary,
            user_id
        )
        
        incident = await self.db.get_incident(incident_id)
        
        # Generate summary
        summary = f"""
🎉 Incident Resolved

**Duration**: {_fmt_minutes(incident.time_to_resolve)}
**Time to Engage**: {_fmt_minutes(incident.time_to_engage)}
**Time to Mitigate**: {_fmt_minutes(incident.time_to_mitigate)}

**Resolution**: {resolution_summary}

Next steps:
1. Schedule postmortem (within 48 hours)
2. Complete any follow-up actions
3. Update documentation if needed
"""
        
        await self._notify_channel(incident.slack_channel, summary)
        
        # Close status page incident
        if incident.severity == IncidentSeverity.SEV1:
            await self._close_status_page_incident(incident)
    
    async def generate_postmortem_draft(
        self,
        incident_id: str
    ) -> str:
        """Generate a postmortem template from incident data."""
        incident = await self.db.get_incident(incident_id)
        
        # Format timeline
        timeline_table = "| Time | Event |\n|------|-------|\n"
        for event in incident.timeline:
            time_str = event.timestamp.strftime("%H:%M UTC")
            timeline_table += f"| {time_str} | {event.description} |\n"
        
        template = f"""
# Postmortem: {incident.title}

## Incident Summary

| Field | Value |
|-------|-------|
| **Date** | {incident.detected_at.strftime("%Y-%m-%d")} |
| **Duration** | {_fmt_minutes(incident.time_to_resolve)} |
| **Severity** | SEV{incident.severity.value} |
| **Status** | Draft |

### One-Line Summary
[TODO: One sentence describing what happened and the impact]

### Impact
- **User Impact**: [TODO]
- **Duration**: {_fmt_minutes(incident.time_to_resolve)}
- **Scope**: [TODO]

---

## Timeline (All times in UTC)

{timeline_table}

**Key Metrics**:
- Time to Engage (TTE): {_fmt_minutes(incident.time_to_engage)}
- Time to Mitigate (TTM): {_fmt_minutes(incident.time_to_mitigate)}
- Total Incident Duration: {_fmt_minutes(incident.time_to_resolve)}

---

## Root Cause Analysis

### What Happened
[TODO: Detailed technical explanation]

### Why It Happened (5 Whys)
[TODO: Apply 5 whys technique]

---

## Action Items

| ID | Action | Owner | Priority | Due Date | Status |
|----|--------|-------|----------|----------|--------|
| 1 | [TODO] | [TODO] | P1 | [TODO] | TODO |

---
"""
        return template