Delete a Cloudformation stack having Lambdas in a VPC quickly

If you’ve a Cloudformation stack that creates 1 or more Lambdas in a VPC, you may have noticed that Cloudformation takes a lot of time to delete its Lambda resources. This is so because AWS requires can require upwards of 45 minutes to delete a Lambda’s Elastic Network Interface (ENI) https://repost.aws/knowledge-center/lambda-delete-cloudformation-stack

This large delay hurt a developer’s productivity (even more so when a developer has to delete multiple stacks having Lambdas). The the easiest way to resolve this is to just not start Lambdas in a VPC (atleast while a feature in development). However, your employer’s legal/security guidelines may disallow that. Fortunately, there an easy way to circumvent this delay. Here’s how.

First, before deleting a stack having 1 or more Lambdas, purge the Lambda’s source code so the Lambda doesn’t do anything when its removed from its VPC in the next step. Because the Lambda doesn’t do anything, it should be safe to remove it from its VPC (This approach may be enough to meet your employer’s legal/security guidelines).

echo 'export const handler = async () => { console.log("Exiting..."); };' > index.mjs
zip function.zip index.mjs
aws lambda update-function-code --function-name lambda-function-name --zip-file fileb://function.zip

Then, remove the stack’s Lambdas from their VPC

aws lambda update-function-configuration --function-name lambda-function-name --vpc-config "SubnetIds=[],SecurityGroupIds=[]"

Retrieve the network interfaces of the security groups of those Lambdas and delete the network interfaces of those security groups

aws lambda get-function-configuration --function-name lambda-function-name --query 'VpcConfig.SecurityGroupIds' --output json

aws ec2 describe-network-interfaces --filters "Name=group-id,Values=" --query 'NetworkInterfaces[*].NetworkInterfaceId' --output json

aws ec2 delete-network-interface --network-interface-id lambda-network-interface-id

Finally, delete the Cloudformation stack

aws cloudformation delete-stack --stack-name your-cloudformation-stack-name

Obviously, doing the above often is inconvenient/impractical. Fortunately, we can use Node SDK V3 to do the above faster. Install the necessary AWS Node SDK V3 modules

npm install -g @aws-sdk/client-lambda
npm install -g @aws-sdk/client-cloudformation

and then run the Node script below

node delete-stacks.js stack1 stack2 stack3

As I delete Cloudformation stacks often, this script saves me many hours of time every week. I hope you find it useful as well.

const { CloudFormationClient, DescribeStackResourcesCommand, DescribeStacksCommand, DeleteStackCommand } = require("@aws-sdk/client-cloudformation"); 
const { EC2Client, DescribeNetworkInterfacesCommand, DetachNetworkInterfaceCommand, DeleteNetworkInterfaceCommand } = require("@aws-sdk/client-ec2");
const { LambdaClient, UpdateFunctionConfigurationCommand } = require("@aws-sdk/client-lambda");

const cloudformation = new CloudFormationClient();
const ec2 = new EC2Client();
const lambda = new LambdaClient();

function sleepForSeconds(seconds) {
  return new Promise(resolve => setTimeout(resolve, seconds * 1000));
}

const getStackStatus = async (stack) => {
  const command = new DescribeStacksCommand({ StackName: stack });
  const response = await cloudformation.send(command);
  return response.Stacks[0].StackStatus;
}

const waitForStackDeletion = async (stack) => {
  console.log(`Waiting for stack "${stack}" to be deleted...`);
  while (true) {
    await sleepForSeconds(5);
    let stackStatus;
    try {
      stackStatus = await getStackStatus(stack);
    } catch (e) {
      // When a stack is small, Cloudformation deletes it very quickly and a stack can be deleted before the getStackStatus is called
      if (e.message.includes('does not exist')) {
        break;
      }
    }
    if (stackStatus === 'DELETE_IN_PROGRESS') continue;
    if (stackStatus === 'DELETE_COMPLETE') break;
    throw new Error(`Failed to delete stack ${stack}`);
  }
};

const deleteStack = async (stack) => {
  console.log(`Deleting stack ${stack}...`);

  try {
    await cloudformation.send(new DeleteStackCommand({ StackName: stack }));
  } catch(e) {
    console.error(`Failed to delete stack ${stack}`);
    throw e;
  }

  await waitForStackDeletion(stack);
  console.log(`Stack ${stack} successfully deleted.`);
};

const deleteNetworkInterface = async (networkInterfaceId) => {
  // Retry many times as AWS often takes time to truly detach a network interface from a Lambda.
  // Until then, the `aws ec2 delete-network-interface` command will fail to delete the network interface and say that it's currently in use.
  const maxRetryCount = 10;
  const retryDelay = 15;
  for (let i = 0; i < maxRetryCount; i++) {
    try {
      await ec2.send(new DeleteNetworkInterfaceCommand({ NetworkInterfaceId: networkInterfaceId }));
    } catch (e) {
      if (e.message.includes('does not exist')) {
        break
      } else if (e.message.includes('is currently in use')) {
        console.log(`${networkInterfaceId} is still in use.`);
      } else {
        console.log(`Failed to delete network interface ${networkInterfaceId}: ${e}`);
      }
      if (i === (maxRetryCount - 1)) throw e;
      console.log(`Will retry deleting network interface ${networkInterfaceId} again in ${retryDelay} seconds.`);
    }
    await sleepForSeconds(retryDelay);
  }
}

const detachAndDeleteNetworkInterface = async (networkInterface) => {
  const networkInterfaceId = networkInterface.NetworkInterfaceId;
  console.log(`Deleting network interface ${networkInterfaceId}...`);
  await deleteNetworkInterface(networkInterfaceId);
  console.log(`Deleted network interface ${networkInterfaceId}.`);
}

const getSecurityGroupNetworkInterfaces = async (securityGroupId) => {
  const describeNetworkInterfacesResult = await ec2.send(new DescribeNetworkInterfacesCommand({
    Filters: [{ Name: 'group-id', Values: [securityGroupId] }]
  }));
  return describeNetworkInterfacesResult.NetworkInterfaces;
}

const detachLambdasFromVpc = async (stack) => {
  const { StackResources: stackResources } = await cloudformation.send(new DescribeStackResourcesCommand({ StackName: stack }));
  const lambdas = stackResources.filter(
    (r) => r.ResourceType === 'AWS::Lambda::Function'
  );
  const securityGroups = stackResources.filter(
    (resource) => resource.ResourceType === 'AWS::EC2::SecurityGroup'
  );
  if (lambdas.length !== 0) {
    for (const l of lambdas) {
      const lambdaName = l.PhysicalResourceId;
      console.log(`Detaching lambda ${lambdaName} from VPC...`);
      try {
        await lambda.send(new UpdateFunctionConfigurationCommand({
          FunctionName: lambdaName,
          VpcConfig: {
            SubnetIds: [],
            SecurityGroupIds: []
          }
        }));
      } catch (e) {
        if (e.message.includes("ResourceNotFoundException")) {
          console.log(`Lambda ${lambdaName} has already been deleted.`)
        } else {
          throw e;
        }
      }
      console.log(`Successfully detached lambda ${lambdaName} from VPC.`);
    }
  };
  if (securityGroups.length !== 0) {
    for (const securityGroup of securityGroups) {
      const securityGroupId = securityGroup.PhysicalResourceId;
      console.log(`Deleting security group ${securityGroupId} network interfaces...`);
      const networkInterfaces = await getSecurityGroupNetworkInterfaces(securityGroupId);
      for (const networkInterface of networkInterfaces) {
        await detachAndDeleteNetworkInterface(networkInterface);
      }
    }
  };
};

const detachLambdasFromVpcAndDeleteStack = async (stack) => {
  await detachLambdasFromVpc(stack);
  return deleteStack(stack);
};

const deleteStacks = async (stacks) => {
  const promises = stacks.map((stack) => detachLambdasFromVpcAndDeleteStack(stack));
  return Promise.all(promises);
};

module.exports = deleteStacks;

if (require.main === module) {
  const stacks = process.argv.slice(2);
  if (stacks.length === 0) {
    console.error('Please provide 1 or more stacks to delete.');
    process.exit(1);
  }
  deleteStacks(stacks);
}