Delete a Cloudformation stack having Lambdas in a VPC quickly

If you’ve a Cloudformation stack that creates 1 or more Lambdas in a VPC, you may have noticed that Cloudformation takes a lot of time to delete its Lambda resources. This is so because AWS requires can require upwards of 45 minutes to delete a Lambda’s Elastic Network Interface (ENI) https://repost.aws/knowledge-center/lambda-delete-cloudformation-stack

This large delay can often be frustrating for a developer, especially when deleting 1 or more Cloudformation stacks having Lambda resources. The the easiest way to resolve this is to just not start Lambdas in a VPC (atleast while a feature in development). However, your employer’s legal/security guidelines may disallow that. Fortunately, there an easy way to circumvent this delay Here’s how.

First, before deleting a stack having 1 or more Lambdas, modify a Lambda’s source code so the Lambda doesn’t do anything

echo 'export const handler = async () => { console.log("Exiting..."); };' > index.mjs
zip function.zip index.mjs
aws lambda update-function-code --function-name lambda-function-name --zip-file fileb://function.zip

Then, remove the stack’s Lambdas from their VPC

aws lambda update-function-configuration --function-name lambda-function-name --vpc-config "SubnetIds=[],SecurityGroupIds=[]"

Retrieve the network interfaces of the security groups of those Lambdas and delete the network interfaces of those security groups

aws lambda get-function-configuration --function-name lambda-function-name --query 'VpcConfig.SecurityGroupIds' --output json

aws ec2 describe-network-interfaces --filters "Name=group-id,Values=<lambda-function-security-group-id>" --query 'NetworkInterfaces[*].NetworkInterfaceId' --output json

aws ec2 delete-network-interface --network-interface-id lambda-network-interface-id

Finally, delete the Cloudformation stack

aws cloudformation delete-stack --stack-name your-cloudformation-stack-name

Obviously, doing the above often is inconvenient/impractical. Fortunately, we can use Node SDK V3 to do the above faster. Install the necessary AWS Node SDK V3 modules

npm install -g @aws-sdk/client-lambda
npm install -g @aws-sdk/client-cloudformation

and then run the Node script below

node delete-stacks.js stack1 stack2 stack3

As I delete Cloudformation stacks often, this script saves me many hours of time every week. I hope you find it useful as well.

const { CloudFormationClient, DescribeStackResourcesCommand, DescribeStacksCommand, DeleteStackCommand } = require("@aws-sdk/client-cloudformation"); 
const { EC2Client, DescribeNetworkInterfacesCommand, DetachNetworkInterfaceCommand, DeleteNetworkInterfaceCommand } = require("@aws-sdk/client-ec2");
const { LambdaClient, UpdateFunctionConfigurationCommand } = require("@aws-sdk/client-lambda");

const cloudformation = new CloudFormationClient();
const ec2 = new EC2Client();
const lambda = new LambdaClient();

function sleepForSeconds(seconds) {
  return new Promise(resolve => setTimeout(resolve, seconds * 1000));
}

const getStackStatus = async (stack) => {
  const command = new DescribeStacksCommand({ StackName: stack });
  const response = await cloudformation.send(command);
  return response.Stacks[0].StackStatus;
}

const waitForStackDeletion = async (stack) => {
  console.log(`Waiting for stack "${stack}" to be deleted...`);
  while (true) {
    await sleepForSeconds(5);
    let stackStatus;
    try {
      stackStatus = await getStackStatus(stack);
    } catch (e) {
      // When a stack is small, Cloudformation deletes it very quickly and a stack can be deleted before the getStackStatus is called
      if (e.message.includes('does not exist')) {
        break;
      }
    }
    if (stackStatus === 'DELETE_IN_PROGRESS') continue;
    if (stackStatus === 'DELETE_COMPLETE') break;
    throw new Error(`Failed to delete stack ${stack}`);
  }
};

const deleteStack = async (stack) => {
  console.log(`Deleting stack ${stack}...`);

  try {
    await cloudformation.send(new DeleteStackCommand({ StackName: stack }));
  } catch(e) {
    console.error(`Failed to delete stack ${stack}`);
    throw e;
  }

  await waitForStackDeletion(stack);
  console.log(`Stack ${stack} successfully deleted.`);
};

const deleteNetworkInterface = async (networkInterfaceId) => {
  // Retry many times as AWS often takes time to truly detach a network interface from a Lambda.
  // Until then, the `aws ec2 delete-network-interface` command will fail to delete the network interface and say that it's currently in use.
  const maxRetryCount = 10;
  const retryDelay = 15;
  for (let i = 0; i < maxRetryCount; i++) {
    try {
      await ec2.send(new DeleteNetworkInterfaceCommand({ NetworkInterfaceId: networkInterfaceId }));
    } catch (e) {
      if (e.message.includes('does not exist')) {
        break
      } else if (e.message.includes('is currently in use')) {
        console.log(`${networkInterfaceId} is still in use.`);
      } else {
        console.log(`Failed to delete network interface ${networkInterfaceId}: ${e}`);
      }
      if (i === (maxRetryCount - 1)) throw e;
      console.log(`Will retry deleting network interface ${networkInterfaceId} again in ${retryDelay} seconds.`);
    }
    await sleepForSeconds(retryDelay);
  }
}

const detachAndDeleteNetworkInterface = async (networkInterface) => {
  const networkInterfaceId = networkInterface.NetworkInterfaceId;
  console.log(`Deleting network interface ${networkInterfaceId}...`);
  await deleteNetworkInterface(networkInterfaceId);
  console.log(`Deleted network interface ${networkInterfaceId}.`);
}

const getSecurityGroupNetworkInterfaces = async (securityGroupId) => {
  const describeNetworkInterfacesResult = await ec2.send(new DescribeNetworkInterfacesCommand({
    Filters: [{ Name: 'group-id', Values: [securityGroupId] }]
  }));
  return describeNetworkInterfacesResult.NetworkInterfaces;
}

const detachLambdasFromVpc = async (stack) => {
  const { StackResources: stackResources } = await cloudformation.send(new DescribeStackResourcesCommand({ StackName: stack }));
  const lambdas = stackResources.filter(
    (r) => r.ResourceType === 'AWS::Lambda::Function'
  );
  const securityGroups = stackResources.filter(
    (resource) => resource.ResourceType === 'AWS::EC2::SecurityGroup'
  );
  if (lambdas.length !== 0) {
    for (const l of lambdas) {
      const lambdaName = l.PhysicalResourceId;
      console.log(`Detaching lambda ${lambdaName} from VPC...`);
      try {
        await lambda.send(new UpdateFunctionConfigurationCommand({
          FunctionName: lambdaName,
          VpcConfig: {
            SubnetIds: [],
            SecurityGroupIds: []
          }
        }));
      } catch (e) {
        if (e.message.includes("ResourceNotFoundException")) {
          console.log(`Lambda ${lambdaName} has already been deleted.`)
        } else {
          throw e;
        }
      }
      console.log(`Successfully detached lambda ${lambdaName} from VPC.`);
    }
  };
  if (securityGroups.length !== 0) {
    for (const securityGroup of securityGroups) {
      const securityGroupId = securityGroup.PhysicalResourceId;
      console.log(`Deleting security group ${securityGroupId} network interfaces...`);
      const networkInterfaces = await getSecurityGroupNetworkInterfaces(securityGroupId);
      for (const networkInterface of networkInterfaces) {
        await detachAndDeleteNetworkInterface(networkInterface);
      }
    }
  };
};

const detachLambdasFromVpcAndDeleteStack = async (stack) => {
  await detachLambdasFromVpc(stack);
  return deleteStack(stack);
};

const deleteStacks = async (stacks) => {
  const promises = stacks.map((stack) => detachLambdasFromVpcAndDeleteStack(stack));
  return Promise.all(promises);
};

module.exports = deleteStacks;

if (require.main === module) {
  const stacks = process.argv.slice(2);
  if (stacks.length === 0) {
    console.error('Please provide 1 or more stacks to delete.');
    process.exit(1);
  }
  deleteStacks(stacks);
}