AWS Lambda Production Monitoring and Debugging: Proven Strategies

// Custom metrics that saved us countless times// Compatible with Node.js 20.x and 22.x runtimesimport { CloudWatch } from '@aws-sdk/client-cloudwatch';
const cloudwatch = new CloudWatch({});
export const publishCustomMetrics = async (  functionName: string,  duration: number,  success: boolean,  businessContext?: { userId?: string, feature?: string }) => {  const metrics = [    {      MetricName: 'FunctionDuration',      Value: duration,      Unit: 'Milliseconds',      Dimensions: [        { Name: 'FunctionName', Value: functionName },        { Name: 'Feature', Value: businessContext?.feature || 'unknown' }      ]    },    {      MetricName: success ? 'FunctionSuccess' : 'FunctionFailure',      Value: 1,      Unit: 'Count',      Dimensions: [        { Name: 'FunctionName', Value: functionName }      ]    }  ];
  // Business-specific metrics  if (businessContext?.userId) {    metrics.push({      MetricName: 'UserAction',      Value: 1,      Unit: 'Count',      Dimensions: [        { Name: 'UserId', Value: businessContext.userId },        { Name: 'ActionType', Value: success ? 'completed' : 'failed' }      ]    });  }
  await cloudwatch.putMetricData({    Namespace: 'Lambda/Business',    MetricData: metrics  });};

import AWSXRay from 'aws-xray-sdk-core';import { DynamoDBClient } from '@aws-sdk/client-dynamodb';import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
// Instrument AWS SDK v3const dynamoClient = AWSXRay.captureAWSv3Client(new DynamoDBClient({}));const dynamoDB = DynamoDBDocumentClient.from(dynamoClient);
export const handler = AWSXRay.captureAsyncFunc('payment-processor', async (event) => {  // Add custom annotations for filtering  const segment = AWSXRay.getSegment();  segment?.addAnnotation('userId', event.userId);  segment?.addAnnotation('paymentMethod', event.paymentMethod);  segment?.addAnnotation('environment', process.env.STAGE);
  try {    // Trace external API calls    const subsegment = segment?.addNewSubsegment('payment-provider-api');    const paymentResult = await processPayment(event);    subsegment?.close();        // Add business metadata    segment?.addMetadata('payment', {      amount: event.amount,      currency: event.currency,      processingTime: Date.now() - event.timestamp    });
    return { success: true, paymentId: paymentResult.id };  } catch (error) {    // Capture error context    segment?.addError(error as Error);    segment?.addMetadata('errorContext', {      userId: event.userId,      errorType: error.name,      requestId: event.requestId    });    throw error;  }});

import { createLogger, format, transports } from 'winston';
const logger = createLogger({  level: process.env.LOG_LEVEL || 'info',  format: format.combine(    format.timestamp(),    format.errors({ stack: true }),    format.json()  ),  transports: [    new transports.Console()  ]});
// Lambda context-aware loggingexport const createContextLogger = (context: any, event: any) => {  const requestId = context.awsRequestId;  const functionName = context.functionName;    return {    info: (message: string, meta?: any) => logger.info({      message,      requestId,      functionName,      stage: process.env.STAGE,      ...meta    }),        error: (message: string, error?: Error, meta?: any) => logger.error({      message,      error: error?.stack || error?.message,      requestId,      functionName,      stage: process.env.STAGE,      ...meta    }),        // Business event logging    business: (event: string, data: any) => logger.info({      message: `Business Event: ${event}`,      businessEvent: event,      data,      requestId,      functionName,      timestamp: new Date().toISOString()    })  };};
// Usage in handlerexport const handler = async (event: any, context: any) => {  const log = createContextLogger(context, event);    log.info('Function invoked', { eventType: event.Records?.[0]?.eventName });    try {    const result = await processEvent(event);    log.business('order-processed', { orderId: result.orderId, amount: result.amount });    return result;  } catch (error) {    log.error('Processing failed', error as Error, { eventData: event });    throw error;  }};

// Custom metric for intelligent throttling detectionexport const detectThrottling = async (functionName: string, context: any) => {  const remainingTime = context.getRemainingTimeInMillis();  const duration = context.logStreamName; // Contains execution environment info    // Detect if we're running in a throttled environment  if (remainingTime < 1000) {    await cloudwatch.putMetricData({      Namespace: 'Lambda/Performance',      MetricData: [{        MetricName: 'NearTimeout',        Value: 1,        Unit: 'Count',        Dimensions: [          { Name: 'FunctionName', Value: functionName },          { Name: 'RemainingTime', Value: remainingTime.toString() }        ]      }]    });  }};

// Error categorization for better debuggingexport enum ErrorCategory {  TRANSIENT = 'TRANSIENT',  // Retry makes sense  CLIENT_ERROR = 'CLIENT_ERROR', // User input issue  SYSTEM_ERROR = 'SYSTEM_ERROR', // Infrastructure problem  BUSINESS_ERROR = 'BUSINESS_ERROR' // Business logic violation}
export class CategorizedError extends Error {  constructor(    message: string,    public category: ErrorCategory,    public retryable: boolean = false,    public context?: any  ) {    super(message);    this.name = 'CategorizedError';  }}
export const handleError = async (error: Error, event: any, context: any) => {  const log = createContextLogger(context, event);    if (error instanceof CategorizedError) {    // Handle categorized errors    switch (error.category) {      case ErrorCategory.TRANSIENT:        log.info('Transient error - will retry', {           error: error.message,           retryable: error.retryable         });        throw error; // Let Lambda retry mechanism handle              case ErrorCategory.CLIENT_ERROR:        log.info('Client error - no retry needed', { error: error.message });        return {           statusCode: 400,           body: JSON.stringify({ error: 'Invalid request' })        };              case ErrorCategory.SYSTEM_ERROR:        log.error('System error detected', error, {           requiresInvestigation: true         });        // Send to DLQ for investigation        throw error;              case ErrorCategory.BUSINESS_ERROR:        log.business('business-rule-violation', {          rule: error.message,          context: error.context        });        return {          statusCode: 422,          body: JSON.stringify({ error: error.message })        };    }  } else {    // Unknown error - treat as system error    log.error('Uncategorized error', error);    throw new CategorizedError(      error.message,      ErrorCategory.SYSTEM_ERROR,      false,      { originalError: error.stack }    );  }};

// DLQ processor for error pattern analysisexport const dlqProcessor = async (event: any, context: any) => {  const log = createContextLogger(context, event);    for (const record of event.Records) {    try {      const failedEvent = JSON.parse(record.body);      const errorInfo = {        functionName: record.eventSourceARN?.split(':')[6],        errorCount: record.attributes?.ApproximateReceiveCount || '1',        failureReason: record.attributes?.DeadLetterReason || 'unknown',        originalTimestamp: failedEvent.timestamp,        retryCount: parseInt(record.attributes?.ApproximateReceiveCount || '0')      };            // Pattern detection      if (errorInfo.retryCount > 3) {        log.business('recurring-failure-pattern', {          pattern: 'high-retry-count',          functionName: errorInfo.functionName,          suggestion: 'investigate-configuration'        });      }            // Store for analysis      await storeErrorPattern(errorInfo, failedEvent);          } catch (processingError) {      log.error('Failed to process DLQ record', processingError as Error);    }  }};

// Debug endpoint for production troubleshootingexport const debugHandler = async (event: any, context: any) => {  // Only allow in non-production or with special header  const allowDebug = process.env.STAGE !== 'prod' ||                      event.headers?.['x-debug-token'] === process.env.DEBUG_TOKEN;    if (!allowDebug) {    return { statusCode: 403, body: 'Debug access denied' };  }    const debugInfo = {    environment: {      stage: process.env.STAGE,      region: context.invokedFunctionArn.split(':')[3],      memorySize: context.memoryLimitInMB,      timeout: context.remainingTimeInMillis    },    runtime: {      nodeVersion: process.version,      platform: process.platform,      uptime: process.uptime()    },    lastErrors: await getRecentErrors(context.functionName),    healthChecks: {      database: await checkDatabaseConnection(),      externalAPI: await checkExternalServices(),      memory: process.memoryUsage()    }  };    return {    statusCode: 200,    body: JSON.stringify(debugInfo, null, 2)  };};

// Safe production profilingexport const profileHandler = (originalHandler: Function) => {  return async (event: any, context: any) => {    const shouldProfile = Math.random() < 0.01; // Profile 1% of requests        if (!shouldProfile) {      return originalHandler(event, context);    }        const startTime = Date.now();    const startMemory = process.memoryUsage();        try {      const result = await originalHandler(event, context);            const endTime = Date.now();      const endMemory = process.memoryUsage();            // Send profiling data      await cloudwatch.putMetricData({        Namespace: 'Lambda/Profiling',        MetricData: [          {            MetricName: 'ExecutionDuration',            Value: endTime - startTime,            Unit: 'Milliseconds'          },          {            MetricName: 'MemoryUsed',            Value: endMemory.heapUsed - startMemory.heapUsed,            Unit: 'Bytes'          }        ]      });            return result;    } catch (error) {      // Profile error scenarios too      const errorTime = Date.now();      await cloudwatch.putMetricData({        Namespace: 'Lambda/Profiling',        MetricData: [{          MetricName: 'ErrorDuration',          Value: errorTime - startTime,          Unit: 'Milliseconds'        }]      });      throw error;    }  };};

// Emergency debug checklistexport const emergencyDebugChecklist = {  step1_quickHealth: async (functionName: string) => {    const metrics = await cloudwatch.getMetricStatistics({      Namespace: 'AWS/Lambda',      MetricName: 'Errors',      Dimensions: [{ Name: 'FunctionName', Value: functionName }],      StartTime: new Date(Date.now() - 10 * 60 * 1000), // Last 10 minutes      EndTime: new Date(),      Period: 300,      Statistics: ['Sum']    });        return {      recentErrors: metrics.Datapoints?.reduce((sum, dp) => sum + (dp.Sum || 0), 0),      timeframe: 'last-10-minutes'    };  },    step2_checkDependencies: async () => {    return {      database: await checkDatabaseConnection(),      externalAPIs: await checkExternalServices(),      downstream: await checkDownstreamServices()    };  },    step3_analyzeLogs: async (functionName: string) => {    // CloudWatch Logs Insights query for recent errors    const query = `      fields @timestamp, @message, @requestId      | filter @message like /ERROR/ or @message like /TIMEOUT/      | sort @timestamp desc      | limit 20    `;        // Implementation would use CloudWatch Logs API    return { recentErrorPatterns: 'implementation-needed' };  }};

// Detect memory leaks in long-running Lambda containerslet requestCount = 0;const memorySnapshots: Array<{ count: number; memory: NodeJS.MemoryUsage }> = [];
export const memoryTrackingWrapper = (handler: Function) => {  return async (event: any, context: any) => {    requestCount++;        const beforeMemory = process.memoryUsage();    const result = await handler(event, context);    const afterMemory = process.memoryUsage();        // Track memory growth over requests    if (requestCount % 10 === 0) {      memorySnapshots.push({ count: requestCount, memory: afterMemory });            if (memorySnapshots.length > 10) {        const oldSnapshot = memorySnapshots[memorySnapshots.length - 10];        const currentSnapshot = memorySnapshots[memorySnapshots.length - 1];                const heapGrowth = currentSnapshot.memory.heapUsed - oldSnapshot.memory.heapUsed;                if (heapGrowth > 50 * 1024 * 1024) { // 50MB growth          await cloudwatch.putMetricData({            Namespace: 'Lambda/MemoryLeak',            MetricData: [{              MetricName: 'SuspectedMemoryLeak',              Value: heapGrowth,              Unit: 'Bytes',              Dimensions: [                { Name: 'FunctionName', Value: context.functionName }              ]            }]          });        }      }    }        return result;  };};

// Intelligent sampling based on business valueexport const createSampler = (baseSampleRate: number = 0.01) => {  return (event: any): boolean => {    // Always sample errors    if (event.errorType) return true;        // Always sample high-value transactions    if (event.transactionValue > 1000) return true;        // Sample new users more frequently    if (event.userType === 'new') return Math.random() < baseSampleRate * 5;        // Regular sampling    return Math.random() < baseSampleRate;  };};
const sampler = createSampler(0.005); // 0.5% base rate
export const handler = async (event: any, context: any) => {  const shouldMonitor = sampler(event);    if (shouldMonitor) {    // Full monitoring and tracing    return AWSXRay.captureAsyncFunc('handler', async () => {      return processWithFullLogging(event, context);    });  } else {    // Minimal monitoring    return processWithBasicLogging(event, context);  }};