import http from 'k6/http'; import { check, sleep } from 'k6'; import { Rate, Trend, Counter } from 'k6/metrics'; // Custom metrics for failure analysis const errorRate = new Rate('errors'); const degradedRate = new Rate('degraded_responses'); const recoveryTime = new Trend('recovery_time_ms'); const gracefulDegradation = new Counter('graceful_degradation_count'); // Failure injection test configuration // This test validates graceful degradation when dependencies fail export const options = { scenarios: { // Steady load during failure injection steady_load: { executor: 'constant-vus', vus: 30, duration: '15m', }, }, thresholds: { // During failures, we expect some errors but should still partially work http_req_failed: ['rate<0.30'], // <30% failures during injection 'errors': ['rate<0.30'], 'degraded_responses': ['rate<0.50'], // <50% degraded responses http_req_duration: ['p(95)<5000'], // Relaxed - 5s during failures }, }; const API_BASE_URL = __ENV.API_URL || 'https://api.gtmapis.com'; const API_KEY = __ENV.API_KEY || 'gtm_test_1234567890abcdef1234567890abcdef'; // Test emails for validation const testEmails = [ 'john.doe@company.com', 'jane.smith@startup.io', 'info@company.com', 'invalid-email', 'user@gmail.com', ]; // Failure injection phases (manual coordination required) // Run this script while manually injecting failures in infrastructure const PHASES = { BASELINE: { start: 0, end: 3 }, // Minutes 0-3: Normal operation REDIS_SLOW: { start: 3, end: 6 }, // Minutes 3-6: Simulate Redis latency REDIS_DOWN: { start: 6, end: 9 }, // Minutes 6-9: Simulate Redis unavailable DB_SLOW: { start: 9, end: 12 }, // Minutes 9-12: Simulate DB latency RECOVERY: { start: 12, end: 15 }, // Minutes 12-15: All systems normal }; function getCurrentPhase(elapsedMinutes) { for (const [phase, timing] of Object.entries(PHASES)) { if (elapsedMinutes >= timing.start && elapsedMinutes < timing.end) { return phase; } } return 'UNKNOWN'; } let testStartTime = 0; export default function () { if (testStartTime === 0) { testStartTime = Date.now(); } const elapsedMinutes = (Date.now() - testStartTime) / 1000 / 60; const currentPhase = getCurrentPhase(elapsedMinutes); // Single validation request const email = testEmails[Math.floor(Math.random() * testEmails.length)]; const payload = JSON.stringify({ email }); const params = { headers: { 'Content-Type': 'application/json', 'X-API-Key': API_KEY, }, tags: { name: 'Validation', phase: currentPhase, }, timeout: '10s', // Extended timeout during failures }; const startTime = Date.now(); const response = http.post(`${API_BASE_URL}/v1/validate`, payload, params); const duration = Date.now() - startTime; // Check for successful response const isSuccess = response.status === 200; // Check for graceful degradation (partial result, cached response, etc.) let isDegraded = false; let isGraceful = false; if (isSuccess) { try { const body = JSON.parse(response.body); // Check if this was a degraded response (e.g., cache-only, partial validation) // The API should indicate when it's operating in degraded mode if (body.degraded || body.from_cache || body.partial) { isDegraded = true; isGraceful = true; gracefulDegradation.add(1); } // Validate response structure check(response, { 'has result': () => body.result !== undefined, 'result is valid type': () => ['valid', 'invalid', 'risky', 'valid_role_based', 'unknown'].includes(body.result), }); } catch (e) { // Parse error errorRate.add(true); } } else { // Non-200 response errorRate.add(true); // Check if it's a graceful error (429 rate limit, 503 service unavailable with retry-after) if (response.status === 429 || response.status === 503) { isGraceful = true; gracefulDegradation.add(1); } } degradedRate.add(isDegraded); errorRate.add(!isSuccess && !isGraceful); // Log phase transitions (sample 5%) if (Math.random() < 0.05) { const status = isSuccess ? (isDegraded ? 'DEGRADED' : 'OK') : `ERROR:${response.status}`; console.log(`[${currentPhase}] ${status} - ${duration}ms`); } // Also test health endpoint to monitor service status if (Math.random() < 0.1) { // 10% of requests const healthResponse = http.get(`${API_BASE_URL}/health`, { tags: { name: 'HealthCheck', phase: currentPhase }, }); check(healthResponse, { 'health endpoint responds': (r) => r.status === 200 || r.status === 503, }); // Log health status during failure phases if (currentPhase !== 'BASELINE' && currentPhase !== 'RECOVERY') { try { const healthBody = JSON.parse(healthResponse.body); if (healthBody.services) { const redisStatus = healthBody.services.redis || 'unknown'; const dbStatus = healthBody.services.database || 'unknown'; if (Math.random() < 0.2) { console.log(`[${currentPhase}] Health - Redis: ${redisStatus}, DB: ${dbStatus}`); } } } catch (e) { // Ignore parse errors } } } sleep(0.3); } export function setup() { console.log('Failure Injection Test - Graceful Degradation Validation'); console.log(`Target API: ${API_BASE_URL}`); console.log(''); console.log('IMPORTANT: This test requires manual failure injection!'); console.log(''); console.log('Test Phases and Required Actions:'); console.log(''); console.log(' Minutes 0-3: BASELINE'); console.log(' No action needed - establish baseline metrics'); console.log(''); console.log(' Minutes 3-6: REDIS_SLOW'); console.log(' Action: Add 500ms latency to Redis'); console.log(' Railway: Use network tools or proxy'); console.log(' Local: docker-compose exec redis tc qdisc add dev eth0 root netem delay 500ms'); console.log(''); console.log(' Minutes 6-9: REDIS_DOWN'); console.log(' Action: Stop Redis container'); console.log(' Railway: Pause Redis service'); console.log(' Local: docker-compose stop redis'); console.log(''); console.log(' Minutes 9-12: DB_SLOW'); console.log(' Action: Add 1s latency to database queries'); console.log(' This may require application-level simulation'); console.log(''); console.log(' Minutes 12-15: RECOVERY'); console.log(' Action: Restore all services to normal'); console.log(' Verify metrics return to baseline'); console.log(''); console.log('Expected Behaviors:'); console.log(' - During REDIS_SLOW: Increased latency but same success rate'); console.log(' - During REDIS_DOWN: MX cache misses, fallback behavior'); console.log(' - During DB_SLOW: Increased latency, may see timeouts'); console.log(' - Health endpoint should report degraded status'); console.log(''); console.log('Success Criteria:'); console.log(' 1. No cascading failures (error rate <30%)'); console.log(' 2. Graceful degradation when cache unavailable'); console.log(' 3. Clear health status reporting'); console.log(' 4. Quick recovery when services restored'); console.log(''); // Initial health check const healthResponse = http.get(`${API_BASE_URL}/health`); check(healthResponse, { 'initial health check passed': (r) => r.status === 200, }); console.log('Starting test - begin failure injection at phase transitions!'); console.log(''); return { startTime: Date.now() }; } export function teardown(data) { const totalDuration = (Date.now() - data.startTime) / 1000 / 60; console.log(''); console.log('Failure Injection Test Complete'); console.log(`Total Duration: ${totalDuration.toFixed(2)} minutes`); console.log(''); console.log('Analysis Checklist:'); console.log(''); console.log(' [ ] BASELINE phase - Document baseline latency and error rate'); console.log(' [ ] REDIS_SLOW phase - Did latency increase proportionally?'); console.log(' [ ] REDIS_DOWN phase - Did the API gracefully degrade?'); console.log(' [ ] DB_SLOW phase - Did timeouts occur? Were they handled?'); console.log(' [ ] RECOVERY phase - Did metrics return to baseline?'); console.log(''); console.log('Key Questions:'); console.log(' 1. Did the circuit breaker activate during DB failures?'); console.log(' 2. Were cached responses served during Redis slowness?'); console.log(' 3. Did health endpoint accurately report service status?'); console.log(' 4. How long did recovery take after failures resolved?'); console.log(''); console.log('REMEMBER: Restore all services to normal operation!'); console.log(''); }

Get Started

Core Concepts

Guides

Page Not Found