Chaos Engineering: Building Resilient Distributed Systems

// Example E-commerce system
const system = {
  services: ['web', 'api', 'database', 'cache', 'payment-gateway'],
  metrics: ['response_time', 'error_rate', 'throughput'],
  steady_state: {
    response_time_ms: 200,
    error_rate_percent: 0.1,
    requests_per_second: 1000
  }
};

Hypothesis: The system will maintain 99.9% availability even if:
- One database replica fails
- Cache becomes unavailable
- API service experiences 50% latency increase

# Install Gremlin CLI
curl -O https://dl.gremlin.com/release/gremlin/darwin/gremlin
chmod +x gremlin
sudo mv gremlin /usr/local/bin/

# Authenticate
gremlin login

# Shutdown attack - terminate a container
gremlin attack container shutdown   --container-id $(docker ps -q | head -n 1)   --duration 60s

# CPU attack - stress CPU resources
gremlin attack cpu 100   --duration 60s   --cluster-targets kubelet,container

# Latency attack - add network delay
gremlin attack network latency   --interface eth0   --delay 1000   --duration 60s

# chaos-mesh-experiment.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-kill-experiment
  namespace: default
spec:
  action: pod-kill
  mode: one
  selector:
    namespaces:
      - default
    labelSelectors:
      app: my-app
  scheduler:
    cron: "@every 10m"

// Simulate database failure
const chaosExperiment = {
  name: 'database-failure-simulation',
  steady_state: {
    metric: 'successful_orders_per_minute',
    baseline: 50
  },
  hypothesis: 'System maintains >80% of baseline during DB failover',
  method: 'Terminate primary database replica',
  duration: '5 minutes',
  rollback: 'Promote standby replica to primary'
};

// Monitoring script
async function monitorDatabaseFailure() {
  const before = await getMetric('successful_orders_per_minute');

  // Inject failure
  await terminateDatabaseReplica('db-primary-01');

  // Wait for failover
  await sleep(30000);

  const during = await getMetric('successful_orders_per_minute');

  const recovery = (during / before) * 100;

  console.log(`Recovery: ${recovery}%`);

  if (recovery < 80) {
    console.error('FAIL: System didn't recover adequately');
  } else {
    console.log('PASS: System maintained acceptable performance');
  }

  // Cleanup: Restore database
  await restoreDatabase('db-primary-01');
}

// Test system resilience when cache fails
async function cacheFailureExperiment() {
  const config = {
    steady_state: {
      response_time_ms: 200,
      error_rate_percent: 0.1
    },
    hypothesis: 'Response time stays under 1s without cache'
  };

  // Monitor baseline
  const baseline = await measureMetrics();

  // Stop cache service
  await stopService('redis-01');

  // Measure impact
  const withoutCache = await measureMetrics();

  console.log(`Baseline: ${JSON.stringify(baseline)}`);
  console.log(`Without cache: ${JSON.stringify(withoutCache)}`);

  if (withoutCache.response_time_ms < 1000) {
    console.log('PASS: Acceptable performance without cache');
  } else {
    console.error('FAIL: System degrades too much without cache');
    console.error('Recommendation: Implement circuit breaker');
  }

  // Restart cache
  await startService('redis-01');
}

// Add latency to microservice calls
async function latencyExperiment() {
  const baseline = await measureAPIResponseTime();

  // Inject 500ms latency
  await injectLatency({
    target: 'api-service',
    delay: 500,
    jitter: 100,
    duration: '2 minutes'
  });

  // Monitor for timeouts
  const timeouts = await countTimeouts();
  const errorRate = await calculateErrorRate();

  if (errorRate > 0.05) {
    console.error('FAIL: Error rate too high with latency');
    console.error('Recommendation: Increase timeout thresholds');
  }

  if (timeouts > 0) {
    console.error('FAIL: Requests timing out');
    console.error('Recommendation: Implement retries with exponential backoff');
  }
}

class CircuitBreaker {
  constructor(service, threshold = 5, timeout = 60000) {
    this.service = service;
    this.threshold = threshold;
    this.timeout = timeout;
    this.failures = 0;
    this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
    this.lastFailureTime = null;
  }

  async call(method, ...args) {
    if (this.state === 'OPEN') {
      if (Date.now() - this.lastFailureTime > this.timeout) {
        this.state = 'HALF_OPEN';
      } else {
        throw new Error('Circuit breaker is OPEN');
      }
    }

    try {
      const result = await this.service[method](...args);
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  onSuccess() {
    this.failures = 0;
    if (this.state === 'HALF_OPEN') {
      this.state = 'CLOSED';
    }
  }

  onFailure() {
    this.failures++;
    this.lastFailureTime = Date.now();

    if (this.failures >= this.threshold) {
      this.state = 'OPEN';
      console.error('Circuit breaker opened due to failures');
    }
  }
}

// Usage
const circuitBreaker = new CircuitBreaker(apiService);

try {
  const result = await circuitBreaker.call('getData');
} catch (error) {
  // Use fallback data
  return fallbackData;
}

async function retryWithBackoff(fn, maxAttempts = 3, baseDelay = 1000) {
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
    try {
      return await fn();
    } catch (error) {
      if (attempt === maxAttempts) {
        throw error;
      }

      const delay = baseDelay * Math.pow(2, attempt - 1);
      console.log(`Attempt ${attempt} failed, retrying in ${delay}ms`);

      await sleep(delay + Math.random() * 1000); // Add jitter
    }
  }
}

// Usage
const result = await retryWithBackoff(
  () => fetch('https://api.example.com/data'),
  5,
  1000
);

class Bulkhead {
  constructor(maxConcurrent = 10) {
    this.maxConcurrent = maxConcurrent;
    this.running = 0;
    this.queue = [];
  }

  async execute(fn) {
    if (this.running >= this.maxConcurrent) {
      await new Promise(resolve => this.queue.push(resolve));
    }

    this.running++;

    try {
      return await fn();
    } finally {
      this.running--;
      const next = this.queue.shift();
      if (next) next();
    }
  }
}

// Usage
const bulkhead = new Bulkhead(5); // Max 5 concurrent requests

for (const request of requests) {
  bulkhead.execute(() => processRequest(request));
}

name: Chaos Tests

on:
  schedule:
    - cron: '0 2 * * *' # Run daily at 2 AM
  workflow_dispatch:

jobs:
  chaos-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Run chaos experiments
        run: |
          npm run chaos:test

      - name: Upload chaos report
        uses: actions/upload-artifact@v3
        with:
          name: chaos-report
          path: reports/chaos/

      - name: Create issue if experiment fails
        if: failure()
        uses: actions/github-script@v6
        with:
          script: |
            github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: 'Chaos experiment failed',
              body: 'Chaos experiment detected system weakness. Review logs.',
              labels: ['chaos', 'bug']
            });