The Silent Cache Poisoning Disaster: When CDN Becomes Your Enemy

Sarah_Support: Customer reporting they can see someone else's cart with $2,400 worth of items
Mike_Support: Multiple reports of users logged into wrong accounts - seeing different names/addresses
Jennifer_Support: URGENT: Customer can see partial credit card info (last 4 digits) for someone in Texas
David_Support: This is really bad - getting 20+ calls about privacy violations

# Check database integrity
SELECT user_id, session_id, cart_items FROM user_carts 
WHERE created_at > '2025-11-28 08:00:00';
# Result: All data correct, proper user associations

# Check authentication service
grep "session_hijack\|auth_failure" /var/log/auth-service/*.log
# Result: No suspicious activity, all sessions properly validated

# Check load balancer sticky sessions
curl -v https://api.shopfast.com/api/user/cart \
  -H "Cookie: session_id=user123_session"
# Result: Correctly routes to same backend server

# Check application code for session bleeding
git log --since="1 week ago" --grep="session\|cache\|user"
# Result: No recent changes to session management

# Test the same API from different locations
curl -H "Authorization: Bearer user_123_token" \
     -H "X-Forwarded-For: 192.168.1.1" \
     https://api.shopfast.com/api/user/cart

# Response: User 456's cart (WRONG USER!)

# Check CDN cache headers
curl -I https://api.shopfast.com/api/user/cart
# Result revealed the smoking gun:
HTTP/2 200 
cache-control: public, max-age=300
cf-cache-status: HIT  # ← DISASTER! Private data served from cache
x-cache-hits: 47      # ← 47 different users got the same cart!

# Emergency CDN cache purge - nuclear option
# Purge all potentially personalized endpoints

# CloudFlare API example
curl -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/purge_cache" \
     -H "Authorization: Bearer ${CF_TOKEN}" \
     -H "Content-Type: application/json" \
     -d '{
       "purge_everything": true
     }'

# More targeted purge by URL prefix ("files" takes exact URLs only, no wildcards)
curl -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/purge_cache" \
     -H "Authorization: Bearer ${CF_TOKEN}" \
     -H "Content-Type: application/json" \
     -d '{
       "prefixes": [
         "api.shopfast.com/api/user/",
         "api.shopfast.com/api/cart/",
         "api.shopfast.com/api/profile/",
         "api.shopfast.com/api/orders/"
       ]
     }'

# AWS CloudFront example
aws cloudfront create-invalidation \
    --distribution-id E1234567890123 \
    --paths "/api/user/*" "/api/cart/*" "/api/profile/*"

// Emergency CDN bypass headers - add to all personal endpoints
const bypassCDNMiddleware = (req, res, next) => {
  // Nuclear option: prevent ANY caching
  res.set({
    'Cache-Control': 'no-store, no-cache, must-revalidate, private',
    'Pragma': 'no-cache',
    'Expires': '0',
    'Surrogate-Control': 'no-store',
    'CDN-Cache-Control': 'no-store',  // CDN-targeted (Cloudflare and others honour it)
    'X-Accel-Expires': '0'        // Nginx specific
  });
  next();
};

// Apply to all potentially personal endpoints
const personalEndpoints = [
  '/api/user/*',
  '/api/cart/*', 
  '/api/profile/*',
  '/api/orders/*',
  '/api/wishlist/*',
  '/api/addresses/*',
  '/api/payment-methods/*'
];

personalEndpoints.forEach(path => {
  app.use(path, bypassCDNMiddleware);
});

// More sophisticated approach: make cache key include user context
app.use('/api', (req, res, next) => {
  const isPersonalEndpoint = req.path.match(/\/(user|cart|profile|orders)/);
  
  if (isPersonalEndpoint) {
    // Make CDN treat each user as separate cache entry
    res.set({
      'Vary': 'Authorization, Cookie',
      'Cache-Control': 'private, no-cache',
      'X-User-Context': req.user?.id || 'anonymous'
    });
  }
  next();
});

// Note: the CDN computes its cache key from the *request*, before the origin
// answers, so a response header (e.g. X-Cache-Key) cannot change it. Put the
// session cookie / Authorization header into the key in the CDN's own config,
// and never echo a bearer token back in a response header.

# CDN Configuration (WRONG - what they had)
cache_rules:
  - pattern: "/api/*"
    cache_ttl: 300  # 5 minutes
    cache_key: "url_path"  # ← DISASTER: ignores user context
    respect_origin_headers: false  # ← IGNORES backend cache headers!
    
# What it SHOULD have been:
cache_rules:
  - pattern: "/api/products/*"
    cache_ttl: 300
    cache_key: "url_path"
    
  - pattern: "/api/user/*"
    cache_ttl: 0  # NO CACHING
    bypass_cache: true
    
  - pattern: "/api/cart/*"  
    cache_ttl: 0  # NO CACHING
    bypass_cache: true

// Backend was sending correct headers (but CDN ignored them!)
app.get('/api/user/cart', authenticate, (req, res) => {
  res.set({
    'Cache-Control': 'private, no-cache',  // ← CDN ignored this!
    'Vary': 'Authorization'                // ← CDN ignored this too!
  });
  
  const cart = getUserCart(req.user.id);
  res.json(cart);
});

// CDN was using ONLY the URL path as cache key
// So these all got the same cached response:
// User A: GET /api/user/cart (Authorization: Bearer token_A)  
// User B: GET /api/user/cart (Authorization: Bearer token_B)  ← Same cache key!
// User C: GET /api/user/cart (Authorization: Bearer token_C)  ← Same cache key!

// Cache key should have been:
const cacheKey = `${url_path}:${authorization_header}:${user_id}`;

// Some edge locations were more affected because:
// 1. Different cache configurations per region
// 2. Some edges had "respect_origin_headers: true" (working correctly)
// 3. Others had "respect_origin_headers: false" (broken)

const edgeConfigs = {
  'us-east-1': { respect_origin_headers: true },   // Working
  'us-west-1': { respect_origin_headers: false },  // Broken  
  'eu-west-1': { respect_origin_headers: false },  // Broken
  'ap-southeast-1': { respect_origin_headers: true } // Working
};

// Comprehensive endpoint classification
const endpointTypes = {
  STATIC: {
    patterns: ['/assets/*', '/images/*', '/css/*', '/js/*'],
    cache_ttl: 86400,  // 24 hours
    cache_key: 'url_path',
    public: true
  },
  
  SEMI_STATIC: {
    patterns: ['/api/products/*', '/api/categories/*', '/api/search*'],
    cache_ttl: 300,    // 5 minutes  
    cache_key: 'url_path + query_params',
    public: true,
    vary_headers: ['Accept-Language', 'X-Currency']
  },
  
  USER_SPECIFIC: {
    patterns: ['/api/user/*', '/api/cart/*', '/api/orders/*'],
    cache_ttl: 0,      // NO CACHING
    bypass_cache: true,
    private: true
  },
  
  SESSION_DEPENDENT: {
    patterns: ['/api/recommendations/*', '/api/wishlist/*'],
    cache_ttl: 60,     // 1 minute max
    cache_key: 'url_path + user_id + session_id',
    private: true,
    vary_headers: ['Authorization', 'Cookie']
  }
};

// Middleware to enforce caching rules
function applyCachingStrategy(req, res, next) {
  const endpoint = classifyEndpoint(req.path);
  
  switch(endpoint.type) {
    case 'STATIC':
      res.set({
        'Cache-Control': `public, max-age=${endpoint.cache_ttl}`,
        'X-Cache-Type': 'static'
      });
      break;
      
    case 'SEMI_STATIC':
      res.set({
        'Cache-Control': `public, max-age=${endpoint.cache_ttl}`,
        'Vary': endpoint.vary_headers.join(', '),
        'X-Cache-Type': 'semi-static'
      });
      break;
      
    case 'USER_SPECIFIC':
      res.set({
        'Cache-Control': 'private, no-cache, no-store, must-revalidate',
        'Pragma': 'no-cache',
        'Expires': '0',
        'X-Cache-Type': 'user-specific'
      });
      break;
      
    case 'SESSION_DEPENDENT':
      res.set({
        'Cache-Control': `private, max-age=${endpoint.cache_ttl}`,
        'Vary': 'Authorization, Cookie',
        'X-User-Context': req.user?.id || 'anonymous',
        'X-Cache-Type': 'session-dependent'
      });
      break;
  }
  
  next();
}

class SecureCacheManager {
  constructor() {
    this.sensitivePatterns = [
      /\/api\/user\/.*$/,
      /\/api\/cart\/.*$/,
      /\/api\/orders\/.*$/,
      /\/api\/profile\/.*$/,
      /\/api\/payment-methods\/.*$/,
      /\/api\/addresses\/.*$/
    ];
  }
  
  // Layer 1: Application-level protection
  protectSensitiveEndpoint(req, res, next) {
    const isSensitive = this.sensitivePatterns.some(pattern => 
      pattern.test(req.path)
    );
    
    if (isSensitive) {
      // Multiple layers of protection
      res.set({
        'Cache-Control': 'private, no-cache, no-store, must-revalidate',
        'Pragma': 'no-cache',
        'Expires': '0',
        'Surrogate-Control': 'no-store',  // Varnish
        'X-Accel-Expires': '0',           // Nginx
        'CDN-Cache-Control': 'no-store',  // CDN-targeted (Cloudflare and others)
        'X-Security-Policy': 'no-cache-sensitive'
      });
      
      // Add user context to prevent cross-contamination
      if (req.user) {
        res.set('X-User-ID', req.user.id);
        res.set('Vary', 'Authorization, Cookie');  // Vary names *request* headers
      }
    }
    
    next();
  }
  
  // Layer 2: CDN-level validation
  validateCacheHeaders(req, res, next) {
    res.on('finish', () => {
      const isSensitive = this.sensitivePatterns.some(pattern => 
        pattern.test(req.path)
      );
      
      if (isSensitive) {
        const cacheControl = res.get('Cache-Control');
        
        // Alert if sensitive endpoint doesn't have proper headers
        if (!cacheControl?.includes('private') || 
            !cacheControl?.includes('no-cache')) {
          
          console.error('SECURITY ALERT: Sensitive endpoint without proper cache headers', {
            path: req.path,
            cacheControl,
            userAgent: req.get('User-Agent'),
            timestamp: new Date().toISOString()
          });
          
          // Send alert to security team
          this.sendSecurityAlert('cache_header_violation', {
            endpoint: req.path,
            headers: res.getHeaders()
          });
        }
      }
    });
    
    next();
  }
  
  // Layer 3: Runtime cache verification
  async verifyCacheIntegrity() {
    const testEndpoints = [
      '/api/user/profile',
      '/api/cart/items',
      '/api/orders/recent'
    ];
    
    for (const endpoint of testEndpoints) {
      // Test with different user tokens
      const responses = await Promise.all([
        this.testRequest(endpoint, 'user_token_1'),
        this.testRequest(endpoint, 'user_token_2'),
        this.testRequest(endpoint, 'user_token_3')
      ]);
      
      // Verify responses are different (not cached cross-user)
      const uniqueResponses = new Set(responses.map(r => JSON.stringify(r.data)));
      
      if (uniqueResponses.size !== responses.length) {
        console.error('CACHE POISONING DETECTED', {
          endpoint,
          uniqueResponses: uniqueResponses.size,
          totalRequests: responses.length,
          responses: responses.map(r => ({ 
            user: r.userId, 
            cacheStatus: r.headers['cf-cache-status'] 
          }))
        });
        
        // Emergency cache purge
        await this.emergencyCachePurge(endpoint);
      }
    }
  }
}

class SmartCacheKeyGenerator {
  generateCacheKey(req) {
    const baseKey = `${req.method}:${req.path}`;
    
    // Check if endpoint requires user context
    if (this.isUserSpecific(req.path)) {
      const userId = req.user?.id || 'anonymous';
      const sessionId = req.sessionID;
      return `${baseKey}:user:${userId}:session:${sessionId}`;
    }
    
    // Check if endpoint varies by location/currency
    if (this.isLocationSensitive(req.path)) {
      const country = req.headers['cf-ipcountry'] || 'unknown';
      const currency = req.headers['x-currency'] || 'USD';
      return `${baseKey}:geo:${country}:currency:${currency}`;
    }
    
    // Check if endpoint varies by language
    if (this.isLanguageSensitive(req.path)) {
      const language = req.headers['accept-language']?.split(',')[0] || 'en';
      return `${baseKey}:lang:${language}`;
    }
    
    // Default: just URL and query parameters
    const queryString = new URLSearchParams(req.query).toString();
    return queryString ? `${baseKey}:${queryString}` : baseKey;
  }
  
  isUserSpecific(path) {
    return /\/(user|cart|orders|profile|wishlist|addresses)\//.test(path);
  }
  
  isLocationSensitive(path) {
    return /\/(shipping|pricing|taxes|availability)\//.test(path);
  }
  
  isLanguageSensitive(path) {
    return /\/(products|search|categories)\//.test(path);
  }
}

// Multi-user cache poisoning test
describe('Cache Security Tests', () => {
  test('Personal endpoints should never cache across users', async () => {
    const personalEndpoints = [
      '/api/user/profile',
      '/api/cart/items', 
      '/api/orders/history'
    ];
    
    for (const endpoint of personalEndpoints) {
      // Create requests with different user tokens
      const user1Response = await request(app)
        .get(endpoint)
        .set('Authorization', 'Bearer user1_token')
        .expect(200);
        
      const user2Response = await request(app)
        .get(endpoint)
        .set('Authorization', 'Bearer user2_token')
        .expect(200);
      
      // Responses should be different (not cached)
      expect(user1Response.body).not.toEqual(user2Response.body);
      
      // Should have proper cache headers
      expect(user1Response.headers['cache-control']).toMatch(/private|no-cache/);
      expect(user2Response.headers['cache-control']).toMatch(/private|no-cache/);
      
      // Should not have cache hit headers
      expect(user1Response.headers['cf-cache-status']).not.toBe('HIT');
      expect(user2Response.headers['cf-cache-status']).not.toBe('HIT');
    }
  });
  
  test('CDN should respect origin cache headers', async () => {
    // Test that CDN doesn't override application cache settings
    const response = await request(cdnUrl)
      .get('/api/user/profile')
      .set('Authorization', 'Bearer test_token');
    
    expect(response.headers['cache-control']).toMatch(/private/);
    expect(response.headers['cf-cache-status']).toBe('BYPASS');
  });
  
  test('Static content should cache properly', async () => {
    const staticEndpoints = [
      '/assets/app.js',
      '/images/logo.png',
      '/api/products/123'
    ];
    
    for (const endpoint of staticEndpoints) {
      const response1 = await request(cdnUrl).get(endpoint);
      const response2 = await request(cdnUrl).get(endpoint);
      
      // Second request should be cached
      expect(response2.headers['cf-cache-status']).toBe('HIT');
      expect(response1.body).toEqual(response2.body);
    }
  });
});

class CacheSecurityMonitor {
  constructor() {
    this.alerts = new AlertManager();
    this.metrics = new MetricsCollector();
  }
  
  // Monitor for cache poisoning indicators
  async monitorCacheBehavior() {
    setInterval(async () => {
      await this.checkCrossPollination();
      await this.validateCacheHeaders();
      await this.monitorCacheHitRates();
    }, 30000); // Every 30 seconds
  }
  
  async checkCrossPollination() {
    // Test same endpoint with different user contexts
    const testResult = await this.multiUserCacheTest('/api/user/cart');
    
    if (testResult.possiblePoisoning) {
      await this.alerts.sendCritical('CACHE_POISONING_DETECTED', {
        endpoint: '/api/user/cart',
        evidence: testResult.evidence,
        affectedUsers: testResult.affectedUsers,
        action: 'IMMEDIATE_CACHE_PURGE_REQUIRED'
      });
    }
  }
  
  async validateCacheHeaders() {
    const sensitiveEndpoints = [
      '/api/user/profile',
      '/api/cart/items',
      '/api/orders/recent'
    ];
    
    for (const endpoint of sensitiveEndpoints) {
      const response = await fetch(`${CDN_URL}${endpoint}`, {
        headers: { 'Authorization': 'Bearer test_token' }
      });
      
      const cacheControl = response.headers.get('cache-control');
      const cfCacheStatus = response.headers.get('cf-cache-status');
      
      // Alert if sensitive endpoint is being cached
      if (!cacheControl?.includes('private') || cfCacheStatus === 'HIT') {
        await this.alerts.sendWarning('SENSITIVE_ENDPOINT_CACHED', {
          endpoint,
          cacheControl,
          cfCacheStatus,
          recommendation: 'Review CDN configuration'
        });
      }
    }
  }
  
  async monitorCacheHitRates() {
    const cacheMetrics = await this.metrics.getCacheMetrics();
    
    // Alert if user-specific endpoints have high cache hit rates
    const userEndpointHitRate = cacheMetrics.hitRates['/api/user/*'];
    if (userEndpointHitRate > 0.1) { // >10% is suspicious
      await this.alerts.sendWarning('SUSPICIOUS_USER_ENDPOINT_CACHE_HITS', {
        hitRate: userEndpointHitRate,
        expected: '0%',
        action: 'INVESTIGATE_CDN_CONFIGURATION'
      });
    }
  }
}

# CDN Configuration Schema with Validation
cdn_config_schema:
  cache_rules:
    type: array
    items:
      type: object
      properties:
        pattern:
          type: string
          description: "URL pattern to match"
        cache_behavior:
          type: string
          enum: ["cache", "bypass", "conditional"]
        cache_ttl:
          type: integer
          minimum: 0
          maximum: 86400
        cache_key_includes:
          type: array
          items:
            enum: ["url_path", "query_params", "user_id", "session_id", "auth_header"]
        security_classification:
          type: string
          enum: ["public", "semi-public", "private", "sensitive"]
          description: "Data sensitivity level"
      required: ["pattern", "cache_behavior", "security_classification"]
      
  # Security constraints
  constraints:
    - if:
        properties:
          security_classification:
            const: "private"
      then:
        properties:
          cache_behavior:
            const: "bypass"
        required: ["cache_behavior"]
        
    - if:
        properties:
          security_classification:
            const: "sensitive"  
      then:
        properties:
          cache_behavior:
            const: "bypass"
          cache_ttl:
            const: 0
        required: ["cache_behavior", "cache_ttl"]

# Example valid configuration
cache_rules:
  - pattern: "/api/products/*"
    cache_behavior: "cache"
    cache_ttl: 300
    cache_key_includes: ["url_path", "query_params"]
    security_classification: "public"
    
  - pattern: "/api/user/*"
    cache_behavior: "bypass"
    cache_ttl: 0
    security_classification: "private"
    
  - pattern: "/api/cart/*"
    cache_behavior: "bypass" 
    cache_ttl: 0
    security_classification: "sensitive"

// Automated pre-deployment checks
class CacheSecurityValidator {
  async validateDeployment(config) {
    const results = [];
    
    // Check 1: Ensure sensitive endpoints bypass cache
    results.push(await this.validateSensitiveEndpoints(config));
    
    // Check 2: Verify cache key generation includes user context
    results.push(await this.validateCacheKeys(config));
    
    // Check 3: Test multi-user scenarios
    results.push(await this.testMultiUserScenarios(config));
    
    // Check 4: Validate cache headers for all endpoint types
    results.push(await this.validateCacheHeaders(config));
    
    const failures = results.filter(r => !r.passed);
    
    if (failures.length > 0) {
      throw new Error(`Cache security validation failed: ${JSON.stringify(failures)}`);
    }
    
    return { passed: true, checks: results.length };
  }
  
  async validateSensitiveEndpoints(config) {
    const sensitivePatterns = [
      '/api/user/*',
      '/api/cart/*', 
      '/api/orders/*',
      '/api/profile/*',
      '/api/payment-methods/*'
    ];
    
    const violations = [];
    
    for (const pattern of sensitivePatterns) {
      const rule = config.cache_rules.find(r => 
        new RegExp(r.pattern.replace('*', '.*')).test(pattern)
      );
      
      if (!rule || rule.cache_behavior !== 'bypass') {
        violations.push({
          pattern,
          issue: 'Sensitive endpoint not configured to bypass cache',
          currentConfig: rule
        });
      }
    }
    
    return {
      passed: violations.length === 0,
      violations
    };
  }
  
  async testMultiUserScenarios(config) {
    // Simulate deployment and test with multiple users
    const testEndpoints = ['/api/user/profile', '/api/cart/items'];
    const violations = [];
    
    for (const endpoint of testEndpoints) {
      const user1Data = await this.simulateRequest(endpoint, 'user1_token');
      const user2Data = await this.simulateRequest(endpoint, 'user2_token');
      
      if (JSON.stringify(user1Data) === JSON.stringify(user2Data)) {
        violations.push({
          endpoint,
          issue: 'Same response returned for different users',
          evidence: { user1Data, user2Data }
        });
      }
    }
    
    return {
      passed: violations.length === 0,
      violations
    };
  }
}

const damageAssessment = {
  directCosts: {
    lostSales: 2400000,        // $2.4M in abandoned carts
    supportOvertime: 75000,     // Extra support staff
    engineeringTime: 150000,    // Emergency response team
    cdnOverage: 25000          // Cache purge and traffic costs
  },
  
  potentialCosts: {
    gdprFines: 50000000,       // GDPR cap: 4% of global turnover (or €20M if higher)
    lawsuits: 5000000,         // Estimated class action exposure
    brandDamage: 10000000,     // Estimated trust/reputation cost
    customerChurn: 8000000     // Lost lifetime value
  },
  
  mitigationCosts: {
    securityAudit: 200000,     // Third-party security review
    toolingUpgrade: 500000,    // Better monitoring/testing tools
    staffingIncrease: 1000000, // Additional security engineers
    insurancePremium: 300000   // Increased cyber insurance
  }
};

console.log('Total immediate impact:', 
  Object.values(damageAssessment.directCosts).reduce((a, b) => a + b, 0)
); // $2.65M

// Developers thought this was enough:
app.get('/api/user/cart', authenticate, (req, res) => {
  res.set('Cache-Control', 'private, no-cache');
  res.json(getUserCart(req.user.id));
});

// But CDN configuration can override ANY backend header:
cdn_config:
  override_origin_headers: true  # ← This kills security
  default_cache_ttl: 300        # ← This caches everything