LAB33_AWS_Event-Driven Processing: EventBridge Rules + Lambda + SQS DLQ

variable "aws_region"  { type = string; default = "us-east-1" }
variable "environment" { type = string; default = "prod" }

variable "order_events" {
  type = map(object({ detail_type = string; description = string }))
  default = {
    placed    = { detail_type = "OrderPlaced";    description = "Process new order inventory reservation" }
    updated   = { detail_type = "OrderUpdated";   description = "Send update notification to customer" }
    cancelled = { detail_type = "OrderCancelled"; description = "Release inventory and process refund" }
  }
}

provider "aws" { region = var.aws_region }

locals { tags = { Environment = var.environment, ManagedBy = "terraform" } }

data "aws_caller_identity" "current" {}
data "aws_region" "current" {}

# Custom EventBridge event bus.
resource "aws_cloudwatch_event_bus" "orders" {
  name = "orders-bus-${var.environment}"
  tags = local.tags
}

# SQS Dead-Letter Queue for events EventBridge could not deliver.
resource "aws_sqs_queue" "dlq" {
  name                      = "orders-dlq-${var.environment}"
  message_retention_seconds = 1209600  # 14 days
  tags                      = local.tags
}

# SQS resource policy — allows EventBridge to write failed events to the DLQ.
resource "aws_sqs_queue_policy" "dlq" {
  queue_url = aws_sqs_queue.dlq.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Sid       = "AllowEventBridgeDLQ"
      Effect    = "Allow"
      Principal = { Service = "events.amazonaws.com" }
      Action    = "sqs:SendMessage"
      Resource  = aws_sqs_queue.dlq.arn
      Condition = { ArnEquals = { "aws:SourceArn" = aws_cloudwatch_event_bus.orders.arn } }
    }]
  })
}

# Lambda execution role.
data "aws_iam_policy_document" "lambda_assume" {
  statement { effect = "Allow"; actions = ["sts:AssumeRole"]
    principals { type = "Service"; identifiers = ["lambda.amazonaws.com"] } }
}
resource "aws_iam_role" "lambda" {
  name               = "orders-lambda-role-${var.environment}"
  assume_role_policy = data.aws_iam_policy_document.lambda_assume.json
}
resource "aws_iam_role_policy_attachment" "lambda_logs" {
  role       = aws_iam_role.lambda.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

# One CloudWatch Log Group per event handler.
resource "aws_cloudwatch_log_group" "handlers" {
  for_each          = var.order_events
  name              = "/aws/lambda/orders-${each.key}-${var.environment}"
  retention_in_days = 30
  tags              = local.tags
}

# One Lambda function per order event type.
resource "aws_lambda_function" "handlers" {
  for_each      = var.order_events
  function_name = "orders-${each.key}-${var.environment}"
  runtime       = "python3.12"
  handler       = "handler.lambda_handler"
  role          = aws_iam_role.lambda.arn
  filename      = "${path.module}/handler.zip"
  description   = each.value.description

  environment { variables = { EVENT_TYPE = each.value.detail_type; ENV = var.environment } }
  depends_on = [aws_cloudwatch_log_group.handlers]
  tags       = merge(local.tags, { EventType = each.value.detail_type })
}

# Grant EventBridge permission to invoke each Lambda.
resource "aws_lambda_permission" "eventbridge" {
  for_each      = var.order_events
  statement_id  = "AllowEventBridge-${each.key}"
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.handlers[each.key].function_name
  principal     = "events.amazonaws.com"
  source_arn    = "arn:aws:events:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:rule/orders-bus-${var.environment}/*"
}

# EventBridge rules — one per event type.
resource "aws_cloudwatch_event_rule" "orders" {
  for_each       = var.order_events
  name           = "orders-${each.key}-rule-${var.environment}"
  event_bus_name = aws_cloudwatch_event_bus.orders.name

  event_pattern = jsonencode({
    source      = ["orders.service"]
    detail-type = [each.value.detail_type]
  })

  tags = merge(local.tags, { EventType = each.value.detail_type })
}

# Rule targets — Lambda functions with DLQ and retry policy.
resource "aws_cloudwatch_event_target" "lambda" {
  for_each       = var.order_events
  rule           = aws_cloudwatch_event_rule.orders[each.key].name
  event_bus_name = aws_cloudwatch_event_bus.orders.name
  target_id      = "lambda-${each.key}"
  arn            = aws_lambda_function.handlers[each.key].arn

  # Retry policy: 2 retries, give up after 24 hours.
  retry_policy {
    maximum_retry_attempts       = 2
    maximum_event_age_in_seconds = 86400  # 24 hours
  }

  # Events that still cannot be delivered after retries go to the DLQ.
  dead_letter_config { arn = aws_sqs_queue.dlq.arn }
}

# CloudWatch alarm — alert when DLQ messages accumulate.
resource "aws_cloudwatch_metric_alarm" "dlq_depth" {
  alarm_name          = "orders-dlq-messages-${var.environment}"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateNumberOfMessagesVisible"
  namespace           = "AWS/SQS"
  period              = 300
  statistic           = "Sum"
  threshold           = 0  # any failed event should trigger the alarm
  alarm_description   = "Failed events are accumulating in the Orders DLQ"
  dimensions          = { QueueName = aws_sqs_queue.dlq.name }
  tags                = local.tags
}

output "event_bus_arn" { value = aws_cloudwatch_event_bus.orders.arn }
output "dlq_url"       { value = aws_sqs_queue.dlq.url }
output "dlq_arn"       { value = aws_sqs_queue.dlq.arn }
output "lambda_arns" {
  value = { for k, v in aws_lambda_function.handlers : k => v.arn }
}

terraform init && terraform apply

# Publish a test event to the custom bus
aws events put-events --entries '[{
  "Source": "orders.service",
  "DetailType": "OrderPlaced",
  "Detail": "{\"orderId\": \"ORD-001\", \"customerId\": \"CUST-123\", \"amount\": 49.99}",
  "EventBusName": "orders-bus-prod"
}]'

# Check if DLQ has any messages (should be 0 for successful events)
aws sqs get-queue-attributes \
  --queue-url $(terraform output -raw dlq_url) \
  --attribute-names ApproximateNumberOfMessages

# Replay from DLQ (for reprocessing failed events)
aws events start-replay --replay-name "replay-$(date +%s)" \
  --event-source-arn $(terraform output -raw event_bus_arn) \
  --event-start-time 2024-01-01T00:00:00Z --event-end-time 2024-12-31T23:59:59Z \
  --destination '{"Arn":"'$(terraform output -raw event_bus_arn)'"}'

terraform destroy