Skip to content

[deployer] Perform Deletion Concurrently + Activate Automatic Retry #999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 27, 2025
Merged

[deployer] Perform Deletion Concurrently + Activate Automatic Retry #999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dab0b1b
make delete faster
patrick-ogrady May 27, 2025
71fedf6
nit
patrick-ogrady May 27, 2025
41ffb64
nits
patrick-ogrady May 27, 2025
798b702
add retry
patrick-ogrady May 27, 2025
5c9e0b2
nit
patrick-ogrady May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion deployer/src/ec2/aws.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,13 @@ use tokio::time::sleep;

/// Creates an EC2 client for the specified AWS region
pub async fn create_ec2_client(region: Region) -> Ec2Client {
let retry = aws_config::retry::RetryConfig::adaptive()
.with_max_attempts(10)
.with_initial_backoff(Duration::from_millis(500))
.with_max_backoff(Duration::from_secs(30));
let config = aws_config::defaults(BehaviorVersion::v2025_01_17())
.region(region)
.retry_config(retry)
.load()
.await;
Ec2Client::new(&config)
Expand Down Expand Up @@ -842,7 +847,7 @@ pub async fn assert_arm64_support(
}
}

// Check if there’s another page
// Check if there's another page
next_token = response.next_token;
if next_token.is_none() {
break;
Expand Down
290 changes: 149 additions & 141 deletions deployer/src/ec2/destroy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crate::ec2::{
aws::*, deployer_directory, Config, Error, DESTROYED_FILE_NAME, LOGS_PORT, MONITORING_REGION,
PROFILES_PORT, TRACES_PORT,
};
use futures::future::try_join_all;
use std::collections::HashSet;
use std::fs::File;
use std::path::PathBuf;
Expand Down Expand Up @@ -41,167 +42,174 @@ pub async fn destroy(config: &PathBuf) -> Result<(), Error> {

// First pass: Delete instances, security groups, subnets, route tables, peering, IGWs, and key pairs
info!(regions=?all_regions, "removing resources");
let mut jobs = Vec::with_capacity(all_regions.len());
for region in all_regions.clone() {
let ec2_client = create_ec2_client(Region::new(region.clone())).await;
info!(region = region.as_str(), "created EC2 client");

let instance_ids = find_instances_by_tag(&ec2_client, tag).await?;
if !instance_ids.is_empty() {
terminate_instances(&ec2_client, &instance_ids).await?;
wait_for_instances_terminated(&ec2_client, &instance_ids).await?;
info!(
region = region.as_str(),
?instance_ids,
"terminated instances"
);
}

// If in the monitoring region, we need to revoke the ingress rule
let security_groups = find_security_groups_by_tag(&ec2_client, tag).await?;
let has_monitoring_sg = security_groups
.iter()
.any(|sg| sg.group_name() == Some(tag));
let has_binary_sg = security_groups
.iter()
.any(|sg| sg.group_name() == Some(&format!("{}-binary", tag)));
if region == MONITORING_REGION && has_monitoring_sg && has_binary_sg {
// Find the monitoring security group (named `tag`)
let monitoring_sg = security_groups
.iter()
.find(|sg| sg.group_name() == Some(tag))
.expect("Monitoring security group not found")
.group_id()
.unwrap();

// Find the binary security group (named `{tag}-binary`)
let binary_sg = security_groups
.iter()
.find(|sg| sg.group_name() == Some(&format!("{}-binary", tag)))
.expect("Regular security group not found")
.group_id()
.unwrap();

// Revoke ingress rule from monitoring security group to binary security group
let logging_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(LOGS_PORT as i32)
.to_port(LOGS_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(logging_permission)
.send()
.await
{
warn!(%e, "failed to revoke logs ingress rule between monitoring and binary security groups");
} else {
// Stage region teardown
let job = async move {
let ec2_client = create_ec2_client(Region::new(region.clone())).await;
info!(region = region.as_str(), "created EC2 client");

let instance_ids = find_instances_by_tag(&ec2_client, tag).await?;
if !instance_ids.is_empty() {
terminate_instances(&ec2_client, &instance_ids).await?;
wait_for_instances_terminated(&ec2_client, &instance_ids).await?;
info!(
monitoring_sg,
binary_sg,
"revoked logs ingress rule between monitoring and binary security groups"
region = region.as_str(),
?instance_ids,
"terminated instances"
);
}

// Revoke ingress rule from monitoring security group to binary security group
let profiling_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(PROFILES_PORT as i32)
.to_port(PROFILES_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(profiling_permission)
.send()
.await
{
warn!(%e, "failed to revoke profiles ingress rule between monitoring and binary security groups");
} else {
info!(
// If in the monitoring region, we need to revoke the ingress rule
let security_groups = find_security_groups_by_tag(&ec2_client, tag).await?;
let has_monitoring_sg = security_groups
.iter()
.any(|sg| sg.group_name() == Some(tag));
let has_binary_sg = security_groups
.iter()
.any(|sg| sg.group_name() == Some(&format!("{}-binary", tag)));
if region == MONITORING_REGION && has_monitoring_sg && has_binary_sg {
// Find the monitoring security group (named `tag`)
let monitoring_sg = security_groups
.iter()
.find(|sg| sg.group_name() == Some(tag))
.expect("Monitoring security group not found")
.group_id()
.unwrap();

// Find the binary security group (named `{tag}-binary`)
let binary_sg = security_groups
.iter()
.find(|sg| sg.group_name() == Some(&format!("{}-binary", tag)))
.expect("Regular security group not found")
.group_id()
.unwrap();

// Revoke ingress rule from monitoring security group to binary security group
let logging_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(LOGS_PORT as i32)
.to_port(LOGS_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(logging_permission)
.send()
.await
{
warn!(%e, "failed to revoke logs ingress rule between monitoring and binary security groups");
} else {
info!(
monitoring_sg,
binary_sg,
"revoked logs ingress rule between monitoring and binary security groups"
);
}

// Revoke ingress rule from monitoring security group to binary security group
let profiling_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(PROFILES_PORT as i32)
.to_port(PROFILES_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(profiling_permission)
.send()
.await
{
warn!(%e, "failed to revoke profiles ingress rule between monitoring and binary security groups");
} else {
info!(
monitoring_sg,
binary_sg,
"revoked profiles ingress rule between monitoring and binary security groups"
);
}

// Revoke ingress rule from monitoring security group to binary security group
let tracing_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(TRACES_PORT as i32)
.to_port(TRACES_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(tracing_permission)
.send()
.await
{
warn!(%e, "failed to revoke traces ingress rule between monitoring and binary security groups");
} else {
info!(
monitoring_sg,
binary_sg,
"revoked traces ingress rule between monitoring and binary security groups"
);
}
}

// Revoke ingress rule from monitoring security group to binary security group
let tracing_permission = IpPermission::builder()
.ip_protocol("tcp")
.from_port(TRACES_PORT as i32)
.to_port(TRACES_PORT as i32)
.user_id_group_pairs(UserIdGroupPair::builder().group_id(binary_sg).build())
.build();
if let Err(e) = ec2_client
.revoke_security_group_ingress()
.group_id(monitoring_sg)
.ip_permissions(tracing_permission)
.send()
.await
{
warn!(%e, "failed to revoke traces ingress rule between monitoring and binary security groups");
} else {
// Remove network resources
let sgs = find_security_groups_by_tag(&ec2_client, tag).await?;
for sg in sgs {
let sg_id = sg.group_id().unwrap();
wait_for_enis_deleted(&ec2_client, sg_id).await?;
info!(
monitoring_sg,
binary_sg,
"revoked traces ingress rule between monitoring and binary security groups"
region = region.as_str(),
sg_id, "ENIs deleted from security group"
);
delete_security_group(&ec2_client, sg_id).await?;
info!(region = region.as_str(), sg_id, "deleted security group");
}
}

// Remove network resources
let sgs = find_security_groups_by_tag(&ec2_client, tag).await?;
for sg in sgs {
let sg_id = sg.group_id().unwrap();
wait_for_enis_deleted(&ec2_client, sg_id).await?;
info!(
region = region.as_str(),
sg_id, "ENIs deleted from security group"
);
delete_security_group(&ec2_client, sg_id).await?;
info!(region = region.as_str(), sg_id, "deleted security group");
}

let subnet_ids = find_subnets_by_tag(&ec2_client, tag).await?;
for subnet_id in subnet_ids {
delete_subnet(&ec2_client, &subnet_id).await?;
info!(region = region.as_str(), subnet_id, "deleted subnet");
}
let subnet_ids = find_subnets_by_tag(&ec2_client, tag).await?;
for subnet_id in subnet_ids {
delete_subnet(&ec2_client, &subnet_id).await?;
info!(region = region.as_str(), subnet_id, "deleted subnet");
}

let route_table_ids = find_route_tables_by_tag(&ec2_client, tag).await?;
for rt_id in route_table_ids {
delete_route_table(&ec2_client, &rt_id).await?;
info!(region = region.as_str(), rt_id, "deleted route table");
}
let route_table_ids = find_route_tables_by_tag(&ec2_client, tag).await?;
for rt_id in route_table_ids {
delete_route_table(&ec2_client, &rt_id).await?;
info!(region = region.as_str(), rt_id, "deleted route table");
}

let peering_ids = find_vpc_peering_by_tag(&ec2_client, tag).await?;
for peering_id in peering_ids {
delete_vpc_peering(&ec2_client, &peering_id).await?;
wait_for_vpc_peering_deletion(&ec2_client, &peering_id).await?;
info!(
region = region.as_str(),
peering_id, "deleted VPC peering connection"
);
}
let peering_ids = find_vpc_peering_by_tag(&ec2_client, tag).await?;
for peering_id in peering_ids {
delete_vpc_peering(&ec2_client, &peering_id).await?;
wait_for_vpc_peering_deletion(&ec2_client, &peering_id).await?;
info!(
region = region.as_str(),
peering_id, "deleted VPC peering connection"
);
}

let igw_ids = find_igws_by_tag(&ec2_client, tag).await?;
for igw_id in igw_ids {
let vpc_id = find_vpc_by_igw(&ec2_client, &igw_id).await?;
detach_igw(&ec2_client, &igw_id, &vpc_id).await?;
info!(
region = region.as_str(),
igw_id, vpc_id, "detached internet gateway"
);
delete_igw(&ec2_client, &igw_id).await?;
info!(region = region.as_str(), igw_id, "deleted internet gateway");
}
let igw_ids = find_igws_by_tag(&ec2_client, tag).await?;
for igw_id in igw_ids {
let vpc_id = find_vpc_by_igw(&ec2_client, &igw_id).await?;
detach_igw(&ec2_client, &igw_id, &vpc_id).await?;
info!(
region = region.as_str(),
igw_id, vpc_id, "detached internet gateway"
);
delete_igw(&ec2_client, &igw_id).await?;
info!(region = region.as_str(), igw_id, "deleted internet gateway");
}

let key_name = format!("deployer-{}", tag);
delete_key_pair(&ec2_client, &key_name).await?;
info!(region = region.as_str(), key_name, "deleted key pair");
let key_name = format!("deployer-{}", tag);
delete_key_pair(&ec2_client, &key_name).await?;
info!(region = region.as_str(), key_name, "deleted key pair");
Ok::<(), Error>(())
};
jobs.push(job);
}
try_join_all(jobs).await?;

// Second pass: Delete VPCs after dependencies are removed
for region in &all_regions {
Expand Down
Loading