Timeout Hell.

import time
from dataclasses import dataclass
from typing import List
import statistics

@dataclass
class LatencyStats:
    p50: float
    p95: float
    p99: float
    p999: float
    max: float
    
    @classmethod
    def from_samples(cls, samples: List[float]) -> 'LatencyStats':
        sorted_samples = sorted(samples)
        n = len(sorted_samples)
        
        return cls(
            p50=sorted_samples[int(n * 0.50)],
            p95=sorted_samples[int(n * 0.95)],
            p99=sorted_samples[int(n * 0.99)],
            p999=sorted_samples[int(n * 0.999)] if n >= 1000 else sorted_samples[-1],
            max=sorted_samples[-1]
        )

# Example: Measuring a service
samples = [measure_call() for _ in range(10000)]
stats = LatencyStats.from_samples(samples)

print(f"P50: {stats.p50:.1f}ms")   # Typical response
print(f"P95: {stats.p95:.1f}ms")   # Most responses
print(f"P99: {stats.p99:.1f}ms")   # Almost all responses
print(f"P99.9: {stats.p999:.1f}ms") # Even outliers
print(f"Max: {stats.max:.1f}ms")   # Worst case seen

import requests

def call_service(url: str) -> dict:
    response = requests.get(url, timeout=5.0)  # Always 5 seconds
    return response.json()

from dataclasses import dataclass
from typing import Optional
import time

@dataclass
class TimeoutBudget:
    """Manages timeout budget across multiple calls."""
    
    total_budget_ms: float
    start_time: float = None
    
    def __post_init__(self):
        self.start_time = time.monotonic() * 1000  # Monotonic clock in ms
    
    def remaining(self) -> float:
        """How much time is left?"""
        elapsed = (time.monotonic() * 1000) - self.start_time
        return max(0, self.total_budget_ms - elapsed)
    
    def get_timeout(self, default: float, minimum: float = 100) -> float:
        """Get timeout for next call, respecting remaining budget."""
        remaining = self.remaining()
        
        if remaining <= minimum:
            raise TimeoutError("Budget exhausted")
        
        # Use default if we have plenty of time, otherwise use remaining
        return min(default, remaining - minimum)  # Keep 100ms buffer

# Usage
def process_payment(user_id: str, amount: float):
    budget = TimeoutBudget(total_budget_ms=4500)
    
    # Call 1: Fraud check (want 600ms, but respect budget)
    fraud_timeout = budget.get_timeout(default=600)
    fraud_result = fraud_check(user_id, amount, timeout=fraud_timeout)
    
    # Call 2: Bank API (want 3500ms, but respect remaining budget)
    bank_timeout = budget.get_timeout(default=3500)
    bank_result = bank_charge(user_id, amount, timeout=bank_timeout)
    
    # Call 3: Notification (want 300ms, but respect remaining budget)
    notify_timeout = budget.get_timeout(default=300)
    send_notification(user_id, bank_result, timeout=notify_timeout)
    
    return bank_result

from datetime import datetime, timedelta
from typing import Optional

class Deadline:
    """Represents an absolute deadline that propagates through calls."""
    
    def __init__(self, deadline: datetime):
        self.deadline = deadline
    
    @classmethod
    def from_timeout(cls, timeout_ms: float) -> 'Deadline':
        """Create deadline from relative timeout."""
        return cls(datetime.now() + timedelta(milliseconds=timeout_ms))
    
    def remaining_ms(self) -> float:
        """Milliseconds until deadline."""
        remaining = (self.deadline - datetime.now()).total_seconds() * 1000
        return max(0, remaining)
    
    def is_expired(self) -> bool:
        return datetime.now() >= self.deadline
    
    def to_header(self) -> str:
        """Convert to HTTP header for propagation."""
        return self.deadline.isoformat()
    
    @classmethod
    def from_header(cls, header: str) -> Optional['Deadline']:
        """Parse from HTTP header."""
        try:
            return cls(datetime.fromisoformat(header))
        except:
            return None

# In your API handler
def payment_handler(request):
    # Get deadline from incoming request (or create default)
    deadline_header = request.headers.get('X-Deadline')
    if deadline_header:
        deadline = Deadline.from_header(deadline_header)
    else:
        deadline = Deadline.from_timeout(5000)  # Default 5s
    
    # Check if already expired
    if deadline.is_expired():
        return error_response("Request deadline exceeded")
    
    # Use remaining time for downstream call
    remaining = deadline.remaining_ms()
    if remaining < 100:
        return error_response("Insufficient time remaining")
    
    # Call downstream with propagated deadline
    response = call_bank_api(
        timeout=remaining - 100,  # Keep 100ms buffer
        headers={'X-Deadline': deadline.to_header()}
    )
    
    return response

from collections import deque
import threading
import time

class AdaptiveTimeout:
    """
    Adjusts timeout based on recent response times.
    
    Uses the P99 of a sliding window of recent latencies.
    """
    
    def __init__(
        self,
        initial_timeout_ms: float = 1000,
        min_timeout_ms: float = 100,
        max_timeout_ms: float = 30000,
        safety_factor: float = 3.0,
        window_size: int = 100
    ):
        self.min_timeout = min_timeout_ms
        self.max_timeout = max_timeout_ms
        self.safety_factor = safety_factor
        
        # Track recent latencies
        self.latencies = deque(maxlen=window_size)
        self.lock = threading.Lock()
        
        # Initialize with default
        self.current_timeout = initial_timeout_ms
    
    def record_latency(self, latency_ms: float):
        """Record observed latency."""
        with self.lock:
            self.latencies.append(latency_ms)
            self._update_timeout()
    
    def record_timeout(self):
        """Record that a timeout occurred."""
        with self.lock:
            # Treat timeout as latency = current_timeout
            # This pushes the adaptive timeout higher
            self.latencies.append(self.current_timeout)
            self._update_timeout()
    
    def _update_timeout(self):
        """Recalculate timeout based on recent data."""
        if len(self.latencies) < 10:
            return  # Not enough data
        
        # Calculate P99 of recent latencies
        sorted_latencies = sorted(self.latencies)
        p99_index = int(len(sorted_latencies) * 0.99)
        p99 = sorted_latencies[p99_index]
        
        # New timeout = P99 × safety factor, clamped to bounds
        new_timeout = p99 * self.safety_factor
        self.current_timeout = max(self.min_timeout, 
                                    min(self.max_timeout, new_timeout))
    
    def get_timeout(self) -> float:
        """Get current timeout value."""
        return self.current_timeout

# Usage
fraud_check_timeout = AdaptiveTimeout(
    initial_timeout_ms=500,
    min_timeout_ms=100,
    max_timeout_ms=2000,
    safety_factor=2.5
)

def call_fraud_check(data: dict) -> dict:
    timeout = fraud_check_timeout.get_timeout()
    start = time.time()
    
    try:
        result = http_client.post('/fraud/check', json=data, timeout=timeout/1000)
        latency = (time.time() - start) * 1000
        fraud_check_timeout.record_latency(latency)
        return result.json()
    
    except TimeoutError:
        fraud_check_timeout.record_timeout()
        raise

# Bad: No timeout
response = requests.get(url)  # Could wait forever

# Good: Explicit timeout
response = requests.get(url, timeout=2.0)  # Give up after 2s

# Dedicated thread pool per dependency
fraud_check_pool = ThreadPoolExecutor(max_workers=20)
bank_api_pool = ThreadPoolExecutor(max_workers=50)
notification_pool = ThreadPoolExecutor(max_workers=10)

# If bank_api is slow, it only affects its 50 threads
# fraud_check and notification can still work

if active_requests > MAX_CONCURRENT:
    return Response(status=503, body="Service overloaded")

import requests

# Separate timeouts
response = requests.get(
    url,
    timeout=(3.0, 10.0)  # (connect_timeout, read_timeout)
)

# A single value (requests): used as BOTH the connect and the read timeout.
# It is NOT a cap on total request time; aiohttp (ClientTimeout total) offers
# a separate total timeout, while httpx has none (wrap the call instead).
response = requests.get(
    url,
    timeout=10.0
)

def process_payment_naive(user_id: str, amount: float) -> dict:
    """
    WRONG: No timeouts, no error handling.
    """
    # Step 1: Fraud check (could hang forever)
    fraud_result = requests.post(
        'http://fraud-service/check',
        json={'user_id': user_id, 'amount': amount}
    )
    
    if fraud_result.json()['is_fraudulent']:
        return {'status': 'rejected', 'reason': 'fraud'}
    
    # Step 2: Charge bank (could hang forever)
    bank_result = requests.post(
        'http://bank-api/charge',
        json={'user_id': user_id, 'amount': amount}
    )
    
    # Step 3: Send notification (could hang forever)
    requests.post(
        'http://notification-service/send',
        json={'user_id': user_id, 'message': 'Payment successful'}
    )
    
    return {'status': 'success', 'transaction_id': bank_result.json()['id']}

import requests
from requests.exceptions import Timeout, RequestException

def process_payment_v2(user_id: str, amount: float) -> dict:
    """
    Better: Has timeouts, but not well thought out.
    """
    try:
        # Step 1: Fraud check
        fraud_result = requests.post(
            'http://fraud-service/check',
            json={'user_id': user_id, 'amount': amount},
            timeout=5.0  # 5 second timeout
        )
        
        if fraud_result.json()['is_fraudulent']:
            return {'status': 'rejected', 'reason': 'fraud'}
        
        # Step 2: Charge bank
        bank_result = requests.post(
            'http://bank-api/charge',
            json={'user_id': user_id, 'amount': amount},
            timeout=5.0  # 5 second timeout
        )
        
        # Step 3: Send notification
        requests.post(
            'http://notification-service/send',
            json={'user_id': user_id, 'message': 'Payment successful'},
            timeout=5.0  # 5 second timeout
        )
        
        return {'status': 'success', 'transaction_id': bank_result.json()['id']}
    
    except Timeout:
        return {'status': 'error', 'reason': 'timeout'}
    except RequestException as e:
        return {'status': 'error', 'reason': str(e)}

import requests
from requests.exceptions import Timeout, RequestException
from dataclasses import dataclass
from typing import Optional
import time
import logging

logger = logging.getLogger(__name__)

@dataclass
class PaymentResult:
    status: str  # 'success', 'rejected', 'error'
    transaction_id: Optional[str] = None
    reason: Optional[str] = None

class PaymentService:
    """
    Payment service with proper timeout management.
    """
    
    # Service configuration
    TOTAL_BUDGET_MS = 4500  # Leave 500ms buffer from 5s SLA
    
    # Individual service timeouts (default, can be reduced by budget)
    FRAUD_TIMEOUT_MS = 600    # P99 is 200ms, so 3x
    BANK_TIMEOUT_MS = 3500    # P99 is 2000ms, so 1.75x (it's slow)
    NOTIFY_TIMEOUT_MS = 300   # P99 is 100ms, so 3x
    
    def __init__(self, fraud_url: str, bank_url: str, notify_url: str):
        self.fraud_url = fraud_url
        self.bank_url = bank_url
        self.notify_url = notify_url
    
    def process_payment(self, user_id: str, amount: float) -> PaymentResult:
        """
        Process a payment with proper timeout budget management.
        """
        start_time = time.monotonic()
        
        def remaining_budget() -> float:
            elapsed = (time.monotonic() - start_time) * 1000
            return max(0, self.TOTAL_BUDGET_MS - elapsed)
        
        def get_timeout(default_ms: float, min_ms: float = 100) -> float:
            """Get timeout respecting remaining budget."""
            remaining = remaining_budget()
            if remaining < min_ms:
                raise TimeoutError("Budget exhausted")
            return min(default_ms, remaining - min_ms) / 1000  # Convert to seconds
        
        # =====================================================================
        # Step 1: Fraud Check (REQUIRED)
        # =====================================================================
        try:
            timeout = get_timeout(self.FRAUD_TIMEOUT_MS)
            logger.info(f"Fraud check with timeout={timeout:.2f}s")
            
            fraud_response = requests.post(
                f'{self.fraud_url}/check',
                json={'user_id': user_id, 'amount': amount},
                timeout=(1.0, timeout)  # 1s connect, variable read
            )
            fraud_response.raise_for_status()
            fraud_result = fraud_response.json()
            
            if fraud_result.get('is_fraudulent'):
                return PaymentResult(
                    status='rejected',
                    reason='Transaction flagged as potentially fraudulent'
                )
                
        except Timeout:
            logger.warning(f"Fraud check timed out after {timeout:.2f}s")
            # DECISION: Fraud check timeout = reject (be safe)
            return PaymentResult(
                status='error',
                reason='Unable to verify transaction safety. Please try again.'
            )
        except RequestException as e:
            logger.error(f"Fraud check failed: {e}")
            return PaymentResult(status='error', reason='Verification service unavailable')
        
        # =====================================================================
        # Step 2: Bank Charge (REQUIRED)
        # =====================================================================
        try:
            timeout = get_timeout(self.BANK_TIMEOUT_MS)
            logger.info(f"Bank charge with timeout={timeout:.2f}s")
            
            bank_response = requests.post(
                f'{self.bank_url}/charge',
                json={'user_id': user_id, 'amount': amount},
                timeout=(2.0, timeout)  # 2s connect (external), variable read
            )
            bank_response.raise_for_status()
            bank_result = bank_response.json()
            transaction_id = bank_result['transaction_id']
            
        except Timeout:
            logger.warning(f"Bank API timed out after {timeout:.2f}s")
            # DECISION: Bank timeout = ambiguous! Did it charge or not?
            # We'll cover this in Day 2 (idempotency)
            return PaymentResult(
                status='error',
                reason='Payment processing timed out. Please check your statement before retrying.'
            )
        except RequestException as e:
            logger.error(f"Bank charge failed: {e}")
            return PaymentResult(status='error', reason='Payment processing failed')
        
        # =====================================================================
        # Step 3: Notification (OPTIONAL - fire and forget)
        # =====================================================================
        try:
            timeout = get_timeout(self.NOTIFY_TIMEOUT_MS, min_ms=50)
            logger.info(f"Notification with timeout={timeout:.2f}s")
            
            requests.post(
                f'{self.notify_url}/send',
                json={
                    'user_id': user_id,
                    'template': 'payment_success',
                    'data': {'amount': amount, 'transaction_id': transaction_id}
                },
                timeout=(0.5, timeout)
            )
        except Exception as e:
            # Notification failure should NOT fail the payment
            logger.warning(f"Notification failed (non-critical): {e}")
            # Could queue for retry, but payment still succeeded
        
        # =====================================================================
        # Success!
        # =====================================================================
        return PaymentResult(
            status='success',
            transaction_id=transaction_id
        )

def get_fraud_score(user_id: str) -> float:
    try:
        return fraud_service.check(user_id, timeout=0.5)
    except Timeout:
        # Return conservative default
        logger.warning(f"Fraud check timeout, using default for {user_id}")
        return 0.5  # Medium risk, will require additional verification

from threading import Semaphore

# Only allow 50 concurrent bank calls
bank_semaphore = Semaphore(50)

def call_bank_api(data: dict) -> dict:
    # Try to acquire permit (don't wait if all taken)
    acquired = bank_semaphore.acquire(blocking=False)
    
    if not acquired:
        # All permits taken = bank is backed up
        raise ServiceOverloadedError("Bank API at capacity")
    
    try:
        return requests.post(BANK_URL, json=data, timeout=3.5)
    finally:
        bank_semaphore.release()

from collections import deque
import time

class LoadShedder:
    """Reject requests when response time degrades."""
    
    def __init__(self, target_latency_ms: float = 100, window_size: int = 100):
        self.target_latency = target_latency_ms
        self.recent_latencies = deque(maxlen=window_size)
    
    def record(self, latency_ms: float):
        self.recent_latencies.append(latency_ms)
    
    def should_shed(self) -> bool:
        if len(self.recent_latencies) < 10:
            return False
        
        avg_latency = sum(self.recent_latencies) / len(self.recent_latencies)
        return avg_latency > self.target_latency * 3  # 3x target = shedding

shedder = LoadShedder(target_latency_ms=200)

@app.route('/payment')
def payment_handler():
    if shedder.should_shed():
        return Response(status=503, body="Service temporarily overloaded")
    
    start = time.time()
    result = process_payment(request.json)
    shedder.record((time.time() - start) * 1000)
    
    return result

import threading
import time
from collections import deque
from typing import Optional

class SmoothAdaptiveTimeout:
    """
    Adaptive timeout with exponential smoothing to prevent oscillation.
    """
    
    def __init__(
        self,
        initial_timeout_ms: float = 1000,
        min_timeout_ms: float = 100,
        max_timeout_ms: float = 30000,
        percentile: float = 0.99,
        safety_factor: float = 2.0,
        smoothing_factor: float = 0.1,  # How fast to adapt (0-1)
        window_size: int = 1000
    ):
        self.min_timeout = min_timeout_ms
        self.max_timeout = max_timeout_ms
        self.percentile = percentile
        self.safety_factor = safety_factor
        self.smoothing_factor = smoothing_factor
        
        self.samples = deque(maxlen=window_size)
        self.current_timeout = initial_timeout_ms
        self.lock = threading.Lock()
        
        # Track health
        self.timeout_count = 0
        self.success_count = 0
    
    def record_success(self, latency_ms: float):
        """Record successful call with observed latency."""
        with self.lock:
            self.samples.append(latency_ms)
            self.success_count += 1
            self._update_timeout()
    
    def record_timeout(self):
        """Record timeout occurrence."""
        with self.lock:
            # Record as if latency was at current timeout
            self.samples.append(self.current_timeout)
            self.timeout_count += 1
            # Increase timeout more aggressively on timeouts
            self.current_timeout = min(
                self.max_timeout,
                self.current_timeout * 1.5
            )
    
    def _update_timeout(self):
        """Update timeout based on recent samples."""
        if len(self.samples) < 20:
            return  # Need minimum data
        
        # Calculate target percentile
        sorted_samples = sorted(self.samples)
        idx = int(len(sorted_samples) * self.percentile)
        observed_percentile = sorted_samples[idx]
        
        # Target timeout
        target = observed_percentile * self.safety_factor
        target = max(self.min_timeout, min(self.max_timeout, target))
        
        # Smooth update (exponential moving average)
        self.current_timeout = (
            self.smoothing_factor * target +
            (1 - self.smoothing_factor) * self.current_timeout
        )
    
    def get_timeout(self) -> float:
        """Get current timeout in milliseconds."""
        return self.current_timeout
    
    def get_stats(self) -> dict:
        """Get health statistics."""
        total = self.timeout_count + self.success_count
        return {
            'current_timeout_ms': self.current_timeout,
            'success_rate': self.success_count / total if total > 0 else 1.0,
            'timeout_rate': self.timeout_count / total if total > 0 else 0.0,
            'sample_count': len(self.samples)
        }

# Prometheus alert
- alert: AdaptiveTimeoutTooHigh
  expr: adaptive_timeout_current_ms > 2000
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "Adaptive timeout has increased significantly"
    description: "Service {{ $labels.service }} timeout is {{ $value }}ms"

import asyncio
from typing import Optional

async def hedged_request(
    urls: list[str],
    hedge_delay_ms: float = 50,
    timeout_ms: float = 1000
) -> Optional[dict]:
    """
    Send request to first URL, hedge to second if slow.
    """
    
    async def fetch(url: str) -> dict:
        async with aiohttp.ClientSession() as session:
            async with session.get(url, timeout=timeout_ms/1000) as resp:
                return await resp.json()
    
    # Start first request
    primary_task = asyncio.create_task(fetch(urls[0]))
    
    # Wait for either response or hedge delay
    done, pending = await asyncio.wait(
        [primary_task],
        timeout=hedge_delay_ms / 1000
    )
    
    if done:
        # Primary responded quickly
        return primary_task.result()
    
    # Primary is slow, start hedge
    hedge_task = asyncio.create_task(fetch(urls[1]))
    
    # Wait for either to complete
    done, pending = await asyncio.wait(
        [primary_task, hedge_task],
        timeout=(timeout_ms - hedge_delay_ms) / 1000,
        return_when=asyncio.FIRST_COMPLETED
    )
    
    # Cancel the slower one
    for task in pending:
        task.cancel()
    
    if done:
        return done.pop().result()
    
    raise TimeoutError("Both requests timed out")

async def backup_request(
    primary_url: str,
    backup_url: str,
    primary_timeout_ms: float = 2000,
    backup_timeout_ms: float = 2000
) -> dict:
    """
    Try primary first, fall back to backup on failure.
    """
    
    try:
        return await fetch_with_timeout(primary_url, primary_timeout_ms)
    except (TimeoutError, RequestException) as e:
        logger.warning(f"Primary failed: {e}, trying backup")
        return await fetch_with_timeout(backup_url, backup_timeout_ms)

from datetime import datetime, timedelta
from contextvars import ContextVar

# Store deadline in context
request_deadline: ContextVar[datetime] = ContextVar('request_deadline')

class DeadlineMiddleware:
    """Middleware to set and propagate deadlines."""
    
    def __init__(self, app, default_timeout_ms: float = 5000):
        self.app = app
        self.default_timeout = timedelta(milliseconds=default_timeout_ms)
    
    async def __call__(self, request, call_next):
        # Check for incoming deadline header
        deadline_header = request.headers.get('X-Deadline')
        
        if deadline_header:
            deadline = datetime.fromisoformat(deadline_header)
        else:
            deadline = datetime.now() + self.default_timeout
        
        # Set in context for all downstream code
        token = request_deadline.set(deadline)
        
        try:
            return await call_next(request)
        finally:
            request_deadline.reset(token)

def get_remaining_timeout() -> float:
    """Get remaining time until deadline in seconds."""
    deadline = request_deadline.get()
    remaining = (deadline - datetime.now()).total_seconds()
    return max(0, remaining)

# Usage in any function
def call_downstream_service():
    timeout = get_remaining_timeout()
    if timeout < 0.1:
        raise DeadlineExceeded()
    
    return requests.get(url, timeout=timeout - 0.1)  # Keep 100ms buffer

"""
Production-ready payment service with proper timeout handling.
Demonstrates concepts from Day 1: Timeout Hell.
"""

import asyncio
import logging
import time
from dataclasses import dataclass, field
from typing import Optional, Dict, Any
from enum import Enum
import httpx
from prometheus_client import Counter, Histogram, Gauge

# =============================================================================
# Metrics
# =============================================================================

request_latency = Histogram(
    'payment_service_request_seconds',
    'Request latency by operation',
    ['operation'],
    buckets=[.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10]
)

timeout_counter = Counter(
    'payment_service_timeouts_total',
    'Timeout occurrences by service',
    ['service']
)

active_requests = Gauge(
    'payment_service_active_requests',
    'Currently active requests'
)

# =============================================================================
# Configuration
# =============================================================================

@dataclass
class ServiceConfig:
    url: str
    connect_timeout: float  # seconds
    read_timeout: float     # seconds
    required: bool          # Is this service required for success?

@dataclass 
class PaymentConfig:
    total_budget_ms: float = 4500
    
    fraud_service: ServiceConfig = field(default_factory=lambda: ServiceConfig(
        url='http://fraud-service',
        connect_timeout=1.0,
        read_timeout=0.6,
        required=True
    ))
    
    bank_api: ServiceConfig = field(default_factory=lambda: ServiceConfig(
        url='http://bank-api',
        connect_timeout=2.0,
        read_timeout=3.5,
        required=True
    ))
    
    notification_service: ServiceConfig = field(default_factory=lambda: ServiceConfig(
        url='http://notification-service',
        connect_timeout=0.5,
        read_timeout=0.3,
        required=False
    ))

# =============================================================================
# Timeout Budget
# =============================================================================

class TimeoutBudget:
    """Manages timeout budget across multiple service calls."""
    
    def __init__(self, total_ms: float):
        self.total_ms = total_ms
        self.start_time = time.monotonic()
    
    def remaining_ms(self) -> float:
        elapsed = (time.monotonic() - self.start_time) * 1000
        return max(0, self.total_ms - elapsed)
    
    def get_timeout(self, default_ms: float, min_ms: float = 100) -> float:
        """Get timeout respecting remaining budget."""
        remaining = self.remaining_ms()
        
        if remaining <= min_ms:
            raise BudgetExhaustedError(
                f"Budget exhausted. Remaining: {remaining:.0f}ms"
            )
        
        # Use lesser of default and remaining (minus buffer)
        return min(default_ms, remaining - min_ms) / 1000
    
    def is_exhausted(self) -> bool:
        return self.remaining_ms() < 100

class BudgetExhaustedError(Exception):
    pass

# =============================================================================
# Result Types
# =============================================================================

class PaymentStatus(Enum):
    SUCCESS = 'success'
    REJECTED = 'rejected'
    ERROR = 'error'

@dataclass
class PaymentResult:
    status: PaymentStatus
    transaction_id: Optional[str] = None
    error_message: Optional[str] = None
    metadata: Dict[str, Any] = field(default_factory=dict)

# =============================================================================
# Payment Service
# =============================================================================

class PaymentService:
    """
    Payment service demonstrating proper timeout management.
    """
    
    def __init__(self, config: PaymentConfig = None):
        self.config = config or PaymentConfig()
        self.logger = logging.getLogger('payment_service')
    
    async def process_payment(
        self,
        user_id: str,
        amount: float,
        idempotency_key: str  # We'll use this in Day 2
    ) -> PaymentResult:
        """
        Process a payment with proper timeout budget management.
        """
        active_requests.inc()
        start_time = time.monotonic()
        
        try:
            budget = TimeoutBudget(self.config.total_budget_ms)
            
            # Step 1: Fraud Check
            fraud_result = await self._check_fraud(budget, user_id, amount)
            if fraud_result.status != PaymentStatus.SUCCESS:
                return fraud_result
            
            # Step 2: Bank Charge
            bank_result = await self._charge_bank(budget, user_id, amount)
            if bank_result.status != PaymentStatus.SUCCESS:
                return bank_result
            
            transaction_id = bank_result.transaction_id
            
            # Step 3: Notification (fire and forget)
            asyncio.create_task(
                self._send_notification(user_id, amount, transaction_id)
            )
            
            return PaymentResult(
                status=PaymentStatus.SUCCESS,
                transaction_id=transaction_id
            )
        
        except BudgetExhaustedError as e:
            self.logger.error(f"Budget exhausted: {e}")
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message="Request took too long. Please try again."
            )
        
        finally:
            active_requests.dec()
            duration = time.monotonic() - start_time
            request_latency.labels(operation='process_payment').observe(duration)
    
    async def _check_fraud(
        self,
        budget: TimeoutBudget,
        user_id: str,
        amount: float
    ) -> PaymentResult:
        """Check transaction for fraud."""
        config = self.config.fraud_service
        
        try:
            timeout = budget.get_timeout(config.read_timeout * 1000)
            self.logger.info(f"Fraud check with timeout={timeout:.2f}s")
            
            async with httpx.AsyncClient() as client:
                response = await client.post(
                    f"{config.url}/check",
                    json={'user_id': user_id, 'amount': amount},
                    timeout=httpx.Timeout(
                        timeout,  # read/write/pool default
                        connect=config.connect_timeout
                    )
                )
                response.raise_for_status()
                data = response.json()
                
                if data.get('is_fraudulent'):
                    return PaymentResult(
                        status=PaymentStatus.REJECTED,
                        error_message='Transaction flagged as suspicious'
                    )
                
                return PaymentResult(status=PaymentStatus.SUCCESS)
        
        except httpx.TimeoutException:
            timeout_counter.labels(service='fraud').inc()
            self.logger.warning("Fraud check timed out")
            
            # Fraud timeout = reject (be safe)
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message='Unable to verify transaction safety'
            )
        
        except httpx.HTTPError as e:
            self.logger.error(f"Fraud check failed: {e}")
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message='Verification service unavailable'
            )
    
    async def _charge_bank(
        self,
        budget: TimeoutBudget,
        user_id: str,
        amount: float
    ) -> PaymentResult:
        """Charge the bank account."""
        config = self.config.bank_api
        
        try:
            timeout = budget.get_timeout(config.read_timeout * 1000)
            self.logger.info(f"Bank charge with timeout={timeout:.2f}s")
            
            async with httpx.AsyncClient() as client:
                response = await client.post(
                    f"{config.url}/charge",
                    json={'user_id': user_id, 'amount': amount},
                    timeout=httpx.Timeout(
                        timeout,  # read/write/pool default
                        connect=config.connect_timeout
                    )
                )
                response.raise_for_status()
                data = response.json()
                
                return PaymentResult(
                    status=PaymentStatus.SUCCESS,
                    transaction_id=data['transaction_id']
                )
        
        except httpx.TimeoutException:
            timeout_counter.labels(service='bank').inc()
            self.logger.warning("Bank API timed out")
            
            # Bank timeout = ambiguous! (Day 2 covers idempotency)
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message='Payment processing timed out. Check your statement before retrying.'
            )
        
        except httpx.HTTPStatusError as e:
            self.logger.error(f"Bank charge failed: {e}")
            
            if e.response.status_code == 402:
                return PaymentResult(
                    status=PaymentStatus.REJECTED,
                    error_message='Insufficient funds'
                )
            
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message='Payment processing failed'
            )
        
        except httpx.HTTPError as e:
            self.logger.error(f"Bank charge error: {e}")
            return PaymentResult(
                status=PaymentStatus.ERROR,
                error_message='Payment service unavailable'
            )
    
    async def _send_notification(
        self,
        user_id: str,
        amount: float,
        transaction_id: str
    ):
        """Send notification (non-critical)."""
        config = self.config.notification_service
        
        try:
            async with httpx.AsyncClient() as client:
                await client.post(
                    f"{config.url}/send",
                    json={
                        'user_id': user_id,
                        'template': 'payment_success',
                        'data': {
                            'amount': amount,
                            'transaction_id': transaction_id
                        }
                    },
                    timeout=httpx.Timeout(
                        config.read_timeout,  # read/write/pool default
                        connect=config.connect_timeout
                    )
                )
        except Exception as e:
            # Notification failure is NOT critical
            self.logger.warning(f"Notification failed (non-critical): {e}")
            # Could queue for retry here


# =============================================================================
# Example Usage
# =============================================================================

async def main():
    service = PaymentService()
    
    result = await service.process_payment(
        user_id='user_123',
        amount=99.99,
        idempotency_key='order_456_attempt_1'
    )
    
    print(f"Result: {result}")

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    asyncio.run(main())