In this post I’ll create a site-to-site (S2S) tunnel between AWS and Azure. It’s an HA tunnel so it’s regionally resilient. I’ll use Terraform for the deployment using a set of modules (download here). The Terraform files will be separated in two different folders. One for Azure and one for AWS.

The architectural diagram looks like this.

Azure

For Azure, we will provision a public subnet (10.1.0.0/24) and a Gateway subnet (10.1.1.0/24). Then, we’ll provision one VNET Gateway with active-active BGP tunnel. The Azure setup requires that we provision 4 local network gateways and S2S connections, but the output will return only 2 public IPs. These are the files.
provider.tf

terraform {
  required_version = "~>1.9.5"
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~>3.110.0"
    }
  }
}

provider "azurerm" {
  features {
    resource_group {
      prevent_deletion_if_contains_resources = false
    }
  }
}

azure.tf

# Create a resource group
module "rg" {
  source = "../../modules/azure/rg-0.0.4"

  az_rg_name     = var.rg_ResourceGroupName
  az_rg_location = var.rgn_RegionLocation
}

# Create a VNET
module "vnet" {
  source = "../../modules/azure/vnet-4.0.0"

  vnet_name           = var.vnet_Name
  resource_group_name = var.rg_ResourceGroupName
  vnet_location       = var.rgn_RegionLocation
  use_for_each        = false
  address_space       = [var.vnet_CIDR]
  subnet_prefixes     = [var.sub_PublicCIDR, var.sub_GatewayCIDR]
  subnet_names        = [var.sub_PublicName, var.sub_GatewayName]

  depends_on = [module.rg]
}

# Create a virtual network gateway
module "vnetgw" {
  source = "../../modules/azure/vnetgw-0.5.1"

  location                  = var.rgn_RegionLocation
  name                      = var.vgw_Name
  subnet_address_prefix     = var.sub_GatewayCIDR
  sku                       = var.vgw_SKU
  type                      = var.vgw_Type
  virtual_network_id        = module.vnet.vnet_id
  vpn_generation            = var.vgw_Generation
  vpn_type                  = var.vpn_Type
  subnet_creation_enabled   = false
  vpn_active_active_enabled = true
  vpn_bgp_enabled           = true
  vpn_bgp_settings = {
    asn = var.asn_Azure
  }
  # Create virtual network gateway configurations
  ip_configurations = {
    "ip_config_01" = {
      name            = var.gwc_BGPName1
      apipa_addresses = var.ip_APIPAAddresses1
      public_ip = {
        allocation_method = var.ip_allocation_method
      }
    },
    "ip_config_02" = {
      name            = var.gwc_BGPName2
      apipa_addresses = var.ip_APIPAAddresses2
      public_ip = {
        allocation_method = var.ip_allocation_method
      }
    }
  }
  # Create local network gateways and connections
  local_network_gateways = {
    # AWS Tunnel 1 to Azure Instance 0
    "lngw_01" = {
      name            = var.lng_AWST1A0
      gateway_address = var.ip_AWST1A0
      bgp_settings = {
        asn                 = var.asn_AWS
        bgp_peering_address = var.ip_peering_AWST1A0
      }
      connection = {
        name       = var.conn_AWST1A0
        type       = var.conn_Type
        enable_bgp = true
        shared_key = var.key_SharedKey
        custom_bgp_addresses = {
          primary   = var.ip_primary_AWST1A0
          secondary = var.ip_secondary_AWST1A0
        }
      }
    },
    # AWS Tunnel 2 to Azure Instance 0
    "lngw_02" = {
      name            = var.lng_AWST2A0
      gateway_address = var.ip_AWST2A0
      bgp_settings = {
        asn                 = var.asn_AWS
        bgp_peering_address = var.ip_peering_AWST2A0
      }
      connection = {
        name       = var.conn_AWST2A0
        type       = var.conn_Type
        enable_bgp = true
        shared_key = var.key_SharedKey
        custom_bgp_addresses = {
          primary = var.ip_primary_AWST2A0
        secondary = var.ip_secondary_AWST2A0 }
      }
    },
    # AWS Tunnel 1 to Azure Instance 1
    "lngw_03" = {
      name            = var.lng_AWST1A1
      gateway_address = var.ip_AWST1A1
      bgp_settings = {
        asn                 = var.asn_AWS
        bgp_peering_address = var.ip_peering_AWST1A1
      }
      connection = {
        name       = var.conn_AWST1A1
        type       = var.conn_Type
        enable_bgp = true
        shared_key = var.key_SharedKey
        custom_bgp_addresses = {
          primary = var.ip_primary_AWST1A1
        secondary = var.ip_secondary_AWST1A1 }
      }
    },
    # AWS Tunnel 2 to Azure Instance 1
    "lngw_04" = {
      name            = var.lng_AWST2A1
      gateway_address = var.ip_AWST2A1
      bgp_settings = {
        asn                 = var.asn_AWS
        bgp_peering_address = var.ip_peering_AWST2A1
      }
      connection = {
        name       = var.conn_AWST2A1
        type       = var.conn_Type
        enable_bgp = true
        shared_key = var.key_SharedKey
        custom_bgp_addresses = {
          primary = var.ip_primary_AWST2A1
        secondary = var.ip_secondary_AWST2A1 }
      }
    }
  }

  depends_on = [module.vnet]
}

variables.tf

# Resource group
variable "rg_ResourceGroupName" {
  type        = string
  default     = "rgVPN"
  description = "Resource group name"
}

variable "rgn_RegionLocation" {
  type        = string
  default     = "eastus2"
  description = "Azure region"
}

# VNET
variable "vnet_Name" {
  type        = string
  default     = "vnetVPN"
  description = "VNET name"
}

variable "vnet_CIDR" {
  type        = string
  default     = "10.1.0.0/16"
  description = "VNET CIDR"
}

variable "sub_PublicCIDR" {
  type        = string
  default     = "10.1.0.0/24"
  description = "CIDR of the public subnet"
}

variable "sub_GatewayCIDR" {
  type        = string
  default     = "10.1.1.0/24"
  description = "CIDR of the Gateway subnet"
}

variable "sub_PublicName" {
  type        = string
  default     = "subPublic"
  description = "Name of the public subnet"
}

variable "sub_GatewayName" {
  type        = string
  default     = "GatewaySubnet"
  description = "Name of the Gateway subnet - DO NOT CHANGE. It has to stay as GatewaySubnet"
}

# Virtual network gateway
variable "vgw_Name" {
  type        = string
  default     = "vgwVPN"
  description = "Name of the virtual gateway"
}

variable "vgw_SKU" {
  type        = string
  default     = "VpnGw2AZ"
  description = "Virtual Gateway SKU"
}

variable "vgw_Type" {
  type        = string
  default     = "Vpn"
  description = "Virtual Gateway Type (VPN or ExpressRoute)"
}

variable "vgw_Generation" {
  type        = string
  default     = "Generation2"
  description = "Virtual Gateway Generation"
}

variable "vpn_Type" {
  type        = string
  default     = "RouteBased"
  description = "VPN type - RouteBased or PolicyBased"
}

variable "asn_Azure" {
  type        = string
  default     = "65000"
  description = "Azure ASN"
}

variable "asn_AWS" {
  type        = string
  default     = "64512"
  description = "AWS ASN"
}

# Virtual network gateway config 1
variable "gwc_BGPName1" {
  type        = string
  default     = "vnetGatewayConfig01"
  description = "Name of the virtual network gateway configuration for the first tunnel"
}

variable "ip_APIPAAddresses1" {
  type        = list(string)
  default     = ["169.254.21.2", "169.254.22.2"]
  description = "APIPA address for the first tunnel"
}

# Virtual network gateway config 2
variable "gwc_BGPName2" {
  type        = string
  default     = "vnetGatewayConfig02"
  description = "Name of the virtual network gateway configuration for the second tunnel"
}

variable "ip_APIPAAddresses2" {
  type        = list(string)
  default     = ["169.254.21.6", "169.254.22.6"]
  description = "APIPA address for the second tunnel"
}

variable "ip_allocation_method" {
  type        = string
  default     = "Static"
  description = "Static or Dynamic public IP"
}

# Local network gateway common settings
variable "conn_Type" {
  type        = string
  default     = "IPsec"
  description = "Connection type: IPsec or Vnet2Vnet"
}

variable "key_SharedKey" {
  type        = string
  default     = "KlimentAndreev1970_"
  description = "Make sure it's the same as 'key_SharedKey' variable in variables.tf under the AWS folder"
  # Don't go crazy with complexity. AWS side can support only 
  # between 8 and 64 characters in length and cannot start with zero (0). 
  # Allowed characters are alphanumeric characters, periods (.), and underscores (_).
}

# Local network gateway AWS Tunnel 1 to Azure Instance 0
# lngw_01
variable "lng_AWST1A0" {
  type        = string
  default     = "lng_AWSTunnel1ToAzureInstance0"
  description = "From AWS Tunnel 1 to Azure Instance 0"
}

###
### REPLACE FROM THE OUTPUT OF AWS IN STEP 2
###
variable "ip_AWST1A0" {
  type        = string
  default     = "13.58.80.164"
  description = "Enter the output of ip_AWST1IP1 from the AWS output.tf script"
}

variable "ip_peering_AWST1A0" {
  type        = string
  default     = "169.254.21.1"
  description = "BGP Peering IP for AWS Tunnel1 to Azure Instance 0"
}

variable "ip_primary_AWST1A0" {
  type        = string
  default     = "169.254.21.2"
  description = "Primary Custom BGP Address for AWS Tunnel1 to Azure Instance 0"
}

variable "ip_secondary_AWST1A0" {
  type        = string
  default     = "169.254.21.6"
  description = "Secondary Custom BGP Address for AWS Tunnel1 to Azure Instance 0"
}

variable "conn_AWST1A0" {
  type        = string
  default     = "connAWSTunnel1toAzureInstance0"
  description = "Connection name for AWS Tunnel1 to Azure Instance 0"
}

# Local network gateway AWS Tunnel 2 to Azure Instance 0
# lngw_02
variable "lng_AWST2A0" {
  type        = string
  default     = "lng_AWSTunnel2ToAzureInstance0"
  description = "From AWS Tunnel 2 to Azure Instance 0"
}

###
### REPLACE FROM THE OUTPUT OF AWS IN STEP 2
###
variable "ip_AWST2A0" {
  type        = string
  default     = "18.119.84.239"
  description = "Enter the output of ip_AWST2A0 from the AWS output.tf script"
}

variable "ip_peering_AWST2A0" {
  type        = string
  default     = "169.254.22.1"
  description = "BGP Peering IP for AWS Tunnel2 to Azure Instance 0"
}

variable "ip_primary_AWST2A0" {
  type        = string
  default     = "169.254.22.2"
  description = "Primary Custom BGP Address for AWS Tunnel2 to Azure Instance 0"
}

variable "ip_secondary_AWST2A0" {
  type        = string
  default     = "169.254.21.6"
  description = "Secondary Custom BGP Address for AWS Tunnel1 to Azure Instance 1"
}

variable "conn_AWST2A0" {
  type        = string
  default     = "connAWSTunnel2toAzureInstance0"
  description = "Connection name for AWS Tunnel2 to Azure Instance 0"
}

# Local network gateway AWS Tunnel 1 to Azure Instance 1
# lngw_03
variable "lng_AWST1A1" {
  type        = string
  default     = "lng_AWSTunnel1ToAzureInstance1"
  description = "From AWS Tunnel 1 to Azure Instance 1"
}

###
### REPLACE FROM THE OUTPUT OF AWS IN STEP 2
###
variable "ip_AWST1A1" {
  type        = string
  default     = "18.119.2.80"
  description = "Enter the output of ip_AWST1A1 from the AWS output.tf script"
}

variable "ip_peering_AWST1A1" {
  type        = string
  default     = "169.254.21.5"
  description = "BGP Peering IP for AWS Tunnel1 to Azure Instance 1"
}

variable "ip_primary_AWST1A1" {
  type        = string
  default     = "169.254.21.2"
  description = "Primary Custom BGP Address for AWS Tunnel1 to Azure Instance 1"
}

variable "ip_secondary_AWST1A1" {
  type        = string
  default     = "169.254.21.6"
  description = "Secondary Custom BGP Address for AWS Tunnel1 to Azure Instance 1"
}

variable "conn_AWST1A1" {
  type        = string
  default     = "connAWSTunnel1toAzureInstance1"
  description = "Connection name for AWS Tunnel1 to Azure Instance 1"
}

# Local network gateway AWS Tunnel 2 to Azure Instance 1
# lngw_04
variable "lng_AWST2A1" {
  type        = string
  default     = "lng_AWSTunnel2ToAzureInstance1"
  description = "From AWS Tunnel 2 to Azure Instance 1"
}

###
### REPLACE FROM THE OUTPUT OF AWS IN STEP 2
###
variable "ip_AWST2A1" {
  type        = string
  default     = "18.220.73.188"
  description = "Enter the output of ip_AWST2A1 from the AWS ouput.tf script"
}

variable "ip_peering_AWST2A1" {
  type        = string
  default     = "169.254.22.5"
  description = "BGP Peering IP for AWS Tunnel2 to Azure Instance 1"
}

variable "ip_primary_AWST2A1" {
  type        = string
  default     = "169.254.21.2"
  description = "Primary Custom BGP Address for AWS Tunnel2 to Azure Instance 1"
}

variable "ip_secondary_AWST2A1" {
  type        = string
  default     = "169.254.22.6"
  description = "Secondary Custom BGP Address for AWS Tunnel2 to Azure Instance 1"
}

variable "conn_AWST2A1" {
  type        = string
  default     = "connAWSTunnel2toAzureInstance1"
  description = "Connection name for AWS Tunnel2 to Azure Instance 1"
}

outputs.tf

output "vgw_publicip_1" {
  value       = module.vnetgw.public_ip_addresses.ip_config_01.ip_address
  description = "Use this output to populate the same variable in AWS variables.tf file."
}

output "vgw_publicip_2" {
  value       = module.vnetgw.public_ip_addresses.ip_config_02.ip_address
  description = "Use this output to populate the same variable in AWS variables.tf file."
}

Once you run the initial terraform apply it will take about 35-40 mins and you’ll get two IPs as output. You’ll need these momentarily in the file variables.tf under 02.AWS folder.

vgw_publicip_1 = "48.211.240.129"
vgw_publicip_2 = "48.211.240.186"

AWS

In AWS we’ll provision a private and a public subnet as well as two VPN tunnels. This is the Terraform code for that.
provider.tf

terraform {
  required_version = "~>1.9.5"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~>5.57.0"
    }
  }
}

provider "aws" {
  region = var.rgn_RegionLocation
}

aws.tf

# Create VPC
module "vpc" {

  source = "../../modules/aws/vpc-5.5.1"

  name = var.vpc_Name
  cidr = var.vpc_CIDR

  azs             = var.vpc_AZs
  private_subnets = var.sub_pvtCIDR
  public_subnets  = var.sub_pubCIDR

  enable_nat_gateway = true
  enable_vpn_gateway = true
  amazon_side_asn    = var.asn_AWS
}

# Change the route to enable propagation to public route table
# resource "aws_vpn_gateway_route_propagation" "rtPropg" {
#   vpn_gateway_id = module.vpc.vgw_id
#   route_table_id = module.vpc.public_route_table_ids[0]
# }

# Change the route to enable propagation to private route table
resource "aws_vpn_gateway_route_propagation" "rtPvtPropg" {
  vpn_gateway_id = module.vpc.vgw_id
  route_table_id = module.vpc.private_route_table_ids[0]
}

# Add the NAT gateway as a route to Internet to the private route table
# resource "aws_route" "r" {
#   route_table_id         = module.vpc.private_route_table_ids[0]
#   destination_cidr_block = "0.0.0.0/0"
#   nat_gateway_id         = module.vpc.natgw_ids[0]
# }

# Create 2 customer gateways
module "cgw" {
  source = "../../modules/aws/cgw-2.0.1/"

  name = var.cgw_Name

  customer_gateways = {
    IP1 = {
      bgp_asn    = var.asn_Azure
      ip_address = var.vgw_publicip_1
    },
    IP2 = {
      bgp_asn    = var.asn_Azure
      ip_address = var.vgw_publicip_2
    }
  }
}

module "vpngw1" {
  source = "../../modules/aws/vpngw-3.7.2/"

  customer_gateway_id               = module.cgw.ids[0]
  vpc_id                            = module.vpc.vpc_id
  vpn_gateway_id                    = module.vpc.vgw_id
  vpn_connection_static_routes_only = false
  local_ipv4_network_cidr           = "0.0.0.0/0"

  tunnel1_inside_cidr   = var.ip_AWST1A0
  tunnel2_inside_cidr   = var.ip_AWST2A0
  tunnel1_preshared_key = var.key_SharedKey
  tunnel2_preshared_key = var.key_SharedKey

  tags = {
    Name = var.vpn_AWSA0Name
  }

  depends_on = [module.cgw]

}

module "vpngw2" {
  source = "../../modules/aws/vpngw-3.7.2/"

  customer_gateway_id               = module.cgw.ids[1]
  vpc_id                            = module.vpc.vpc_id
  vpn_gateway_id                    = module.vpc.vgw_id
  vpn_connection_static_routes_only = false
  local_ipv4_network_cidr           = "0.0.0.0/0"

  tunnel1_inside_cidr   = var.ip_AWST1A1
  tunnel2_inside_cidr   = var.ip_AWST2A1
  tunnel1_preshared_key = var.key_SharedKey
  tunnel2_preshared_key = var.key_SharedKey

  tags = {
    Name = var.vpn_AWSA1Name
  }

  depends_on = [module.cgw]

}

variables.tf – replace the IPs in line 50 and 59 from the previous Azure output

# VPC
variable "rgn_RegionLocation" {
  type        = string
  default     = "us-east-2"
  description = "AWS region"
}

variable "vpc_Name" {
  type        = string
  default     = "vpcVPN"
  description = "Name of the VPC in AWS"
}

variable "vpc_CIDR" {
  type        = string
  default     = "10.2.0.0/16"
  description = "CIDR of the VPC in AWS"
}

variable "vpc_AZs" {
  type        = list(string)
  default     = ["us-east-2a"]
  description = "Location of available zones"
}

variable "sub_pubCIDR" {
  type        = list(string)
  default     = ["10.2.0.0/24"]
  description = "Public subnet CIDR"
}

variable "sub_pvtCIDR" {
  type        = list(string)
  default     = ["10.2.1.0/24"]
  description = "Private subnet CIDR"
}

# Customer gateways
variable "cgw_Name" {
  type        = string
  default     = "cgwVPN"
  description = "Name of the Customer Gateway"
}

####
#### REPLACE WITH OUTPUT FROM AZURE
####
variable "vgw_publicip_1" {
  type        = string
  default     = "20.10.220.96"
  description = "Use the output from previous Azure script. First public IP of the Virtual Network Gateway in Azure"
}

####
#### REPLACE WITH OUTPUT FROM AZURE
####
variable "vgw_publicip_2" {
  type        = string
  default     = "4.152.124.252"
  description = "Use the output from previous Azure script. Second public IP of the Virtual Network Gateway in Azure"
}

variable "asn_AWS" {
  type        = string
  default     = "64512"
  description = "AWS ASN"
}

variable "asn_Azure" {
  type        = string
  default     = "65000"
  description = "Azure ASN"
}

# VPN site to site 1
variable "key_SharedKey" {
  type        = string
  default     = "KlimentAndreev1970_"
  description = "Make sure it's the same as vartiables.tf in  Azure folder"
  # Don't go crazy with complexity. AWS side can support only 
  # between 8 and 64 characters in length and cannot start with zero (0). 
  # Allowed characters are alphanumeric characters, periods (.), and underscores (_).

}

variable "vpn_AWSA0Name" {
  type        = string
  default     = "vpnToAzureInstance0"
  description = "AWS Tunnel to Azure Instance 0 name"
}

variable "ip_AWST1A0" {
  type    = string
  default = "169.254.21.0/30"
}

variable "ip_AWST2A0" {
  type    = string
  default = "169.254.22.0/30"
}

# VPN site to site 2
variable "vpn_AWSA1Name" {
  type        = string
  default     = "vpnToAzureInstance1"
  description = "AWS Tunnel to Azure Instance 1 name"
}

variable "ip_AWST1A1" {
  type    = string
  default = "169.254.21.4/30"
}

variable "ip_AWST2A1" {
  type    = string
  default = "169.254.22.4/30"
}

outputs.tf

output "ip_AWST1A0" {
  value = module.vpngw1.vpn_connection_tunnel1_address
}

output "ip_AWST2A0" {
  value = module.vpngw1.vpn_connection_tunnel2_address
}

output "ip_AWST1A1" {
  value = module.vpngw2.vpn_connection_tunnel1_address
}

output "ip_AWST2A1" {
  value = module.vpngw2.vpn_connection_tunnel2_address
}

Once you run terraform apply you’ll get 4 IPs as an output.

ip_AWST1A0 = "3.137.10.4"
ip_AWST1A1 = "3.20.47.74"
ip_AWST2A0 = "18.220.233.198"
ip_AWST2A1 = "18.190.37.23"

You have to go back and replace these 4 IPs in variables.tf in the previous Azure folder. So, you run terraform apply in 01.Azure folder first, then 02.AWS second and then again terraform apply in 01.Azure.

Azure

Edit variables.tf file and add the 4 IPs. Make sure that the variables match, do not just assign IPs randomly. Replace the IPs in the lines 155, 196, 237 and 278. The second time you run terraform apply it will take about a minute or two.

Verification

Now, go to AWS and go to VPC | VPN | Site-to-Site VPN connections. You’ll see two tunnels there. For each one select the tunnel and then click on the Tunnel details tab. Both should be up for each tunnel.

In Azure, go to Virtual Network Gateways, click on the vgw, then Monitoring and BGP peers.
You should see something like this.

If you have a use case where you need to transfer a lot of data back-and-forth between various resources and an S3 bucket, you will definitely benefit if you use gateway or interface endpoints for S3. The monthly bill for regional and zonal transfer will be much less and on top of that it’s much more secure.
From the AWS documentation, “Amazon S3 supports both gateway endpoints and interface endpoints. With a gateway endpoint, you can access Amazon S3 from your VPC, without requiring an internet gateway or NAT device for your VPC, and with no additional cost. However, gateway endpoints do not allow access from on-premises networks, from peered VPCs in other AWS Regions, or through a transit gateway. For those scenarios, you must use an interface endpoint, which is available for an additional cost.”
Keep this table handy as it explains the differences between gateway and interface S3 endpoint.

Now that we know the use case and the differences, we’ll see three different scenarios. For this, you’ll need access to two different accounts and two different regions.

• An S3 bucket in account A and an ec2 instance in the same account A and the same region A
• An S3 bucket in account A and an ec2 instance in a different account B, but the same region A
• An S3 bucket in account A and an ec2 instance in a different account B and a different region B

The diagram looks like this.

An S3 bucket in account A, ec2 in account A, both in the same region

Let’s create an S3 bucket first. We’ll use AWS CLI with different profiles.

BUCKET_NAME="myuniquenameforthebucket"
REGION_A="us-east-2"
aws s3api create-bucket \
    --bucket $BUCKET_NAME \
    --region $REGION_A \
    --create-bucket-configuration LocationConstraint=$REGION_A

Let’s copy a file there.

dd if=/dev/zero of=somefile bs=1024 count=1
aws s3 cp somefile "s3://$BUCKET_NAME/somefile"

Using the same profile, create a VPC with one public subnet with the same CIDR as the VPC.

CIDR_A="192.168.10.0/24"
VPCA_ID=$(aws ec2 create-vpc \
    --cidr-block $CIDR_A \
    --region $REGION_A \
    --tag-specification 'ResourceType=vpc,Tags=[{Key=Name,Value=VPC-A}]' \
    --output text --query 'Vpc.VpcId')
echo $VPCA_ID

Enable DNS and hostnames resolution, needed for SSM. Two separate lines needed.

aws ec2 modify-vpc-attribute --enable-dns-hostnames "{\"Value\":true}"  --vpc-id $VPCA_ID
aws ec2 modify-vpc-attribute --enable-dns-support "{\"Value\":true}"  --vpc-id $VPCA_ID

Create a default network ACL.

aws ec2 create-network-acl --vpc-id $VPCA_ID

Create an Internet Gateway and attach it to the VPC.

IGWA_ID=$(aws ec2 create-internet-gateway \
    --region $REGION_A \
    --tag-specifications 'ResourceType=internet-gateway,Tags=[{Key=Name,Value=igw-A}]' \
    --output text --query 'InternetGateway.InternetGatewayId')
echo $IGWA_ID
aws ec2 attach-internet-gateway \
    --internet-gateway-id $IGWA_ID \
    --vpc-id $VPCA_ID

Let’s create a subnet.

SUBA_ID=$(aws ec2 create-subnet \
    --vpc-id $VPCA_ID \
    --cidr-block 192.168.10.0/24 \
    --region $REGION_A \
    --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=subPubA}]' \
    --output text --query 'Subnet.SubnetId')
echo $SUBA_ID

Create a route table.

RTA_ID=$(aws ec2 create-route-table \
     --vpc-id $VPCA_ID \
     --output text --query 'RouteTable.RouteTableId')
echo $RTA_ID
aws ec2 create-route --route-table-id $RTA_ID --destination-cidr-block 0.0.0.0/0 --gateway-id $IGWA_ID

Associate this route table to the public subnet.

aws ec2 associate-route-table --route-table-id $RTA_ID --subnet-id $SUBA_ID

Now, we have to create an EC2 instance where we can test the S3 access. We won’t be using keys and SSH to access, we’ll use SSM. For that we need the instance to have access to the Internet which means we need a security group that allows access to Internet.

SGA_ID=$(aws ec2 create-security-group \
    --group-name sgAllowICMPandOutboundAccess --description "Allows ICMP and outbound access" \
    --vpc-id $VPCA_ID \
    --output text --query 'GroupId')
echo $SGA_ID

Allow unrestricted egress. By default this rule is there, so don’t execute the code below.

#aws ec2 authorize-security-group-egress \
    --group-id $SGA_ID \
    --protocol all \
    --cidr "0.0.0.0/0" 

Do this one, so we can ping instances from each other when we do VPC peering.

aws ec2 authorize-security-group-ingress \
  --group-id $SGA_ID \
  --ip-permissions IpProtocol=icmp,FromPort=-1,ToPort=-1,IpRanges="[{CidrIp=192.168.10.0/24},{CidrIp=192.168.20.0/24},{CidrIp=192.168.30.0/24}]"

Let’s create a role so we can access the EC2 over SSM. First we need a policy that the role can assume.

cat << 'EOF' >> trust-policy.json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
EOF

Then create the role and attach the AWS Managed policy for SSM (AmazonSSMManagerInstanceCore).

aws iam create-role --role-name rolSSM --assume-role-policy-document file://trust-policy.json
aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore --role-name rolSSM

Create instance profile.

aws iam create-instance-profile --instance-profile-name ipEC2

Add the role.

aws iam add-role-to-instance-profile --role-name rolSSM --instance-profile-name ipEC2

Create an instance.

AMI_ID="ami-033fabdd332044f06"
INS_TYPE="t3.micro"
aws ec2 run-instances \
    --image-id $AMI_ID \
    --instance-type $INS_TYPE \
    --security-group-ids $SGA_ID \
    --subnet-id $SUBA_ID \
    --iam-instance-profile Name="ipEC2" \
    --associate-public-ip-address  \
    --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ec2-A}]'

We’ll need an IAM user that we’ll use to access the S3 bucket.

IAM_USER="usrMyBucket"
aws iam create-user --user-name $IAM_USER

We’ll also need a policy that gives access to that specific bucket only.

cat >> bucket-policy.json << EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": "s3:*",
            "Resource": [
                "arn:aws:s3:::$BUCKET_NAME",
                "arn:aws:s3:::$BUCKET_NAME/*"
            ]
        },
        {
            "Effect": "Deny",
            "NotAction": "s3:*",
            "NotResource": [
                "arn:aws:s3:::$BUCKET_NAME",
                "arn:aws:s3:::$BUCKET_NAME/*"
            ]
        }
    ]
}
EOF

Create the policy.

POL_ARN=$(aws iam create-policy \
    --policy-name polMyBucketAccess \
    --policy-document file://bucket-policy.json \
    --output text --query 'Policy.Arn')
echo $POL_ARN

Attach the policy to the user.

aws iam attach-user-policy --policy-arn $POL_ARN --user-name $IAM_USER

Create access keys.

aws iam create-access-key --user-name $IAM_USER --output table --query ['AccessKey.AccessKeyId','AccessKey.SecretAccessKey']

Copy these two values.
Now, go to AWS console, select the EC2 instance and click Connect. Use the 2nd tab Session Manager and click Connect.
Type aws configure and paste the copied values for the IAM user you just created.

If you try to do aws s3 ls, you’ll get access denied, but if you try aws s3 ls s3://myuniquenameforthebucket, you’ll see that it works.

Now, this type of access goes over the Internet and you’ll be charged for the data transfers. In order to avoid that, we’ll create an S3 gateway endpoint.
Go to VPC menu and from the left side choose Endpoints.
Create the endpoint, name the endpoint, choose AWS services, filter by Type = Gateway and select the S3 service name. Then select the VPC where you created your EC2 and the route table that’s associated with the subnet where EC2 resides. For policy choose Full Access. This means, the S3 gateway endpoint will have access to all S3 buckets. Finally, click to create the endpoint.

If you check your route table, you’ll see that there is another entry there. Wait 10 seconds or less if you don’t see it.

If you go back to the EC2 instance and do the aws s3 ls s3://myuniquenameforthebucket command again, you’ll see that nothing changed. So, how do you know that we did the right thing? It’s simple. Click on the endpoint that you’ve created, click on the Policy tab and then click the Edit Policy button.

Change line 5 from Allow to Deny and click Save.
Go back to the EC2 instance and do the aws s3 ls s3://myuniquenameforthebucket again. You’ll get access denied. There you go…Traffic goes over the endpoint. Revert the changes back from Deny to Allow. Let’s move to the 2nd use case.

An S3 bucket in account A, an ec2 instance in a different account B, but both in the same region A

NOTE:Make sure you switch your AWS CLI to a different profile, because we’ll be building the EC2 instance in another account. The S3 bucket stays the same.
Let’s build the 2nd VPC and EC2.

REGION_A="us-east-2"
CIDR_B="192.168.20.0/24"
VPCB_ID=$(aws ec2 create-vpc \
    --cidr-block $CIDR_B \
    --region $REGION_A \
    --tag-specification 'ResourceType=vpc,Tags=[{Key=Name,Value=VPC-B}]' \
    --output text --query 'Vpc.VpcId')
echo $VPCB_ID

Enable DNS and hostnames resolution, needed for SSM. Two separate lines needed.

aws ec2 modify-vpc-attribute --enable-dns-hostnames "{\"Value\":true}"  --vpc-id $VPCB_ID
aws ec2 modify-vpc-attribute --enable-dns-support "{\"Value\":true}"  --vpc-id $VPCB_ID

Create a default network ACL.

aws ec2 create-network-acl --vpc-id $VPCB_ID

Create an Internet Gateway and attach it to the VPC.

IGWB_ID=$(aws ec2 create-internet-gateway \
    --region $REGION_A \
    --tag-specifications 'ResourceType=internet-gateway,Tags=[{Key=Name,Value=igw-B}]' \
    --output text --query 'InternetGateway.InternetGatewayId')
echo $IGWB_ID
aws ec2 attach-internet-gateway \
    --internet-gateway-id $IGWB_ID \
    --vpc-id $VPCB_ID

Let’s create a subnet.

SUBB_ID=$(aws ec2 create-subnet \
    --vpc-id $VPCB_ID \
    --cidr-block 192.168.20.0/24 \
    --region $REGION_A \
    --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=subPubB}]' \
    --output text --query 'Subnet.SubnetId')
echo $SUBB_ID

Create a route table.

RTB_ID=$(aws ec2 create-route-table \
     --vpc-id $VPCB_ID \
     --output text --query 'RouteTable.RouteTableId')
echo $RTB_ID
aws ec2 create-route --route-table-id $RTB_ID --destination-cidr-block 0.0.0.0/0 --gateway-id $IGWB_ID

Associate this route table to the public subnet.

aws ec2 associate-route-table --route-table-id $RTB_ID --subnet-id $SUBB_ID

Now, we have to create an EC2 instance where we can test the S3 access. We won’t be using keys and SSH to access, we’ll use SSM. For that we need the instance to have access to the Internet which means we need a security group that allows access to Internet.

SGB_ID=$(aws ec2 create-security-group \
    --group-name sgAllowICMPandOutboundAccess --description "Allows ICMP and outbound access" \
    --vpc-id $VPCB_ID \
    --output text --query 'GroupId')
echo $SGB_ID

Allow unrestricted egress. By default this rule is there, so don’t execute the code below.

#aws ec2 authorize-security-group-egress \
    --group-id $SGB_ID \
    --protocol all \
    --cidr "0.0.0.0/0" 

Do this one, so we can ping instances from each other when we do VPC peering.

aws ec2 authorize-security-group-ingress \
  --group-id $SGB_ID \
  --ip-permissions IpProtocol=icmp,FromPort=-1,ToPort=-1,IpRanges="[{CidrIp=192.168.10.0/24},{CidrIp=192.168.20.0/24},{CidrIp=192.168.30.0/24}]"

We can reuse the same policy to create a role that will allow us to use SSM.

cat trust-policy.json

Then create the role and attach the AWS Managed policy for SSM (AmazonSSMManagerInstanceCore).

aws iam create-role --role-name rolSSM --assume-role-policy-document file://trust-policy.json
aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore --role-name rolSSM

Create instance profile. We’ll use the same names, because they are different accounts.

aws iam create-instance-profile --instance-profile-name ipEC2

Add the role.

aws iam add-role-to-instance-profile --role-name rolSSM --instance-profile-name ipEC2

Create an instance. The AMI_ID is the same because it’s the same region.

AMI_ID="ami-033fabdd332044f06"
INS_TYPE="t3.micro"
aws ec2 run-instances \
    --image-id $AMI_ID \
    --instance-type $INS_TYPE \
    --security-group-ids $SGB_ID \
    --subnet-id $SUBB_ID \
    --iam-instance-profile Name="ipEC2" \
    --associate-public-ip-address  \
    --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ec2-B}]'

Wait for a couple of minutes and then connect from SSM, same as you did with the first instance. Configure AWS again using the same set of credentials that you did before and then try to list the content of the bucket. You should be able to see the file.

BUCKET_NAME="myuniquenameforthebucket"
aws s3 ls s3://$BUCKET_NAME

Again, the traffic goes over the Internet. So we need another gateway S3 endpoint. You create the gateway S3 endpoint where the client is located, not the destination.
Let’s create another S3 gateway endpoint as described above.

Check the route table and you’ll see the route created. At this point, you are accessing the bucket over the endpoint. Do the same test. Change the policy to Deny for the endpoint and you’ll get access denied.

An S3 bucket in account A, an ec2 instance in a different account B and a different region B

For the 3rd use case, we’ll use an interface endpoint (PrivateLink), not a gateway endpoint. This is what AWS documentation has to say. “With AWS PrivateLink for Amazon S3, you can provision interface VPC endpoints (interface endpoints) in your virtual private cloud (VPC). These endpoints are directly accessible from applications that are on premises over VPN and AWS Direct Connect, or in a different AWS Region over VPC peering. Interface endpoints are represented by one or more elastic network interfaces (ENIs) that are assigned private IP addresses from subnets in your VPC. Requests to Amazon S3 over interface endpoints stay on the Amazon network. You can also access interface endpoints in your VPC from on-premises applications through AWS Direct Connect or AWS Virtual Private Network (AWS VPN).”
Let’s create the resources. VPC and EC2. Change the AWS CLI profile so it’s using the same account B and a new region B.

REGION_B="us-east-1"
CIDR_C="192.168.30.0/24"
VPCC_ID=$(aws ec2 create-vpc \
    --cidr-block $CIDR_C \
    --region $REGION_B \
    --tag-specification 'ResourceType=vpc,Tags=[{Key=Name,Value=VPC-C}]' \
    --output text --query 'Vpc.VpcId')
echo $VPCC_ID

Enable DNS and hostnames resolution, needed for SSM. Two separate lines needed.

aws ec2 modify-vpc-attribute --enable-dns-hostnames "{\"Value\":true}"  --vpc-id $VPCC_ID
aws ec2 modify-vpc-attribute --enable-dns-support "{\"Value\":true}"  --vpc-id $VPCC_ID

Create a default network ACL.

aws ec2 create-network-acl --vpc-id $VPCC_ID

Create an Internet Gateway and attach it to the VPC.

IGWC_ID=$(aws ec2 create-internet-gateway \
    --region $REGION_B \
    --tag-specifications 'ResourceType=internet-gateway,Tags=[{Key=Name,Value=igw-C}]' \
    --output text --query 'InternetGateway.InternetGatewayId')
echo $IGWC_ID
aws ec2 attach-internet-gateway \
    --internet-gateway-id $IGWC_ID \
    --vpc-id $VPCC_ID

Let’s create a subnet.

SUBC_ID=$(aws ec2 create-subnet \
    --vpc-id $VPCC_ID \
    --cidr-block 192.168.30.0/24 \
    --region $REGION_B \
    --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=subPubC}]' \
    --output text --query 'Subnet.SubnetId')
echo $SUBC_ID

Create a route table.

RTC_ID=$(aws ec2 create-route-table \
     --vpc-id $VPCC_ID \
     --output text --query 'RouteTable.RouteTableId')
echo $RTC_ID
aws ec2 create-route --route-table-id $RTC_ID --destination-cidr-block 0.0.0.0/0 --gateway-id $IGWC_ID

Associate this route table to the public subnet.

aws ec2 associate-route-table --route-table-id $RTC_ID --subnet-id $SUBC_ID

Now, we have to create an EC2 instance where we can test the S3 access. We won’t be using keys and SSH to access, we’ll use SSM. For that we need the instance to have access to the Internet which means we need a security group that allows access to Internet.

SGC_ID=$(aws ec2 create-security-group \
    --group-name sgAllowICMPandOutboundAccess --description "Allows ICMP and outbound access" \
    --vpc-id $VPCC_ID \
    --output text --query 'GroupId')
echo $SGC_ID

Allow unrestricted egress. By default this rule is there, so don’t execute the code below.

#aws ec2 authorize-security-group-egress \
    --group-id $SGB_ID \
    --protocol all \
    --cidr "0.0.0.0/0" 

Do this one, so we can ping instances from each other when we do VPC peering.

aws ec2 authorize-security-group-ingress \
  --group-id $SGC_ID \
  --ip-permissions IpProtocol=icmp,FromPort=-1,ToPort=-1,IpRanges="[{CidrIp=192.168.10.0/24},{CidrIp=192.168.20.0/24},{CidrIp=192.168.30.0/24}]"

We don’t have to create any roles. We’ll use the existing ones that we built for the 2nd use case.
Create the EC2. The AMI ID will change this time.

AMI_ID="ami-08a0d1e16fc3f61ea"
INS_TYPE="t3.micro"
aws ec2 run-instances \
    --image-id $AMI_ID \
    --instance-type $INS_TYPE \
    --security-group-ids $SGC_ID \
    --subnet-id $SUBC_ID \
    --iam-instance-profile Name="ipEC2" \
    --associate-public-ip-address  \
    --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ec2-C}]'

Same thing as we did before, connect from SSM, configure AWS CLI using the previous credentials and try to list the bucket.

BUCKET_NAME="myuniquenameforthebucket"
aws s3 ls s3://$BUCKET_NAME

You should see the file there. But, this access goes over Internet. Now, we have to create an interface endpoint in the destination VPC which is VPC-A. When we create an interface endpoint a private IP will be generated for this interface endpoint. In order this private IP to access the bucket, we need some sort of connection between regions, e.g. VPC peering or Transit Gateway. Let’s do the peering between account A (VPC-A) and account B (VPC-C). Enter the 12 digit account number for account A where the S3 bucket resides.

echo $VPCA_ID
echo $VPCC_ID
echo $REGION_A
ACCOUNT_A="123456789012"
echo $ACCOUNT_A

Make sure all of the above returns some value. Do the peering.

aws ec2 create-vpc-peering-connection \
  --vpc-id $VPCC_ID --peer-vpc-id $VPCA_ID \
  --peer-owner-id $ACCOUNT_A --peer-region $REGION_A

Log to the account A and under VPC | Peering menu on the left, select the peering and accept it. But this is not enough. You need to tell the VPC to use the routes over the peering connection. Go to account A, VPC A, find the route for the subnet where the EC2 instance is and add a route for destination 192.168.30.0/24 to go over peering. As you can see this is the same route that we modified initially in our first use case.

Now, log to account B and VPC C and modify the route but this time, the destination is VPC A (192.168.10.0/24). At this point you should be able to ping EC2-A from EC2-C. Don’t worry if you can’t ping EC2-C from EC2-A.
At this point we can create the interface endpoint where the bucket is. This time choose interface as type, VPC-A, our security group, full access policy and the subnet where EC2 is. This subnet has nothing to do with the bucket access, we are just creating a network interface there. We can choose any subnet in that VPC, we just need the IP assigned to the endpoint interface.

There will be 2 DNS names created for you. One is referring the zone and the other the region. Use the regional DNS name, the first one and add the word bucket instead of the * character.

From instance EC2-C, this is how you will access the bucket.

aws s3 ls s3://myuniquenameforthebucket --endpoint-url https://bucket.vpce-038d5f94d78d8b216-vyvk5uon.s3.us-east-2.vpce.amazonaws.com

If you try from the instance EC2-C, this won’t work. Why? Because of the security group that we used. We have to allow HTTPS traffic from 192.168.30.0/24. Go find the first security group, that’s the one we used for the interface endpoint and allow HTTPS from 192.168.30.0/24. After the change, you’ll be able to access the bucket.

In this post I’ll describe how to integrate Okta and your AWS account. In order everything to work, you’ll have to enable AWS Organization on the master account and then you can add multiple sub accounts in AWS.
Best practice says that the AWS master account should be used for SSO only and you shouldn’t run any workloads. First thing first, let’s start with Okta.

Okta

Log to your Okta account or sign up for a 30 day trial. Once logged in, if you haven’t already done so, change the sign-in domain from trail*-okta.com to your domain. Or you can skip this step.
Click on Admin from the upper right corner and then under Customizations, click on Domain. I’ve used the Okta-managed customization because it’s easier to manage.

After you click Next, you’ll be prompted to add two DNS entries for your domain.

Change the DNS. This will allow us to access Okta dashboard using our domain in my case it’s okta.andreev.it. From the same menu on the left, click on Applications, then Applications again and click on Browse App Catalog.

Search for AWS IAM and choose AWS IAM Identity Center.

Click on Add Integration and then Done.
Click on the Sign On tab and scroll all the way down to SAML Signing Certificates. Click on Actions drop-down and then View IdP metadata.

This will open a new tab with an XML file. Save the file as metadata.xml.

AWS

Go to console.aws.amazon.com and log with your root account. Go to IAM Identity Center and click on Enable and then select Enable with AWS Organizations.


Click on Settings on the left, then Actions drop-down and click on Customize AWS access portal URL. Change the URL so it’s something that you can easily remember, e.g. yourorg or aws-master.

Click on Settings on the left, then Actions drop-down and click on Change identity source.

Choose External identity provider and click Next.

You’ll land on a page with two parts, Service provider metadata (that’s AWS) and Identity provider metadata (that’s Okta).
Click on Download metadata file from upper right. You’ll need this in Okta. The file will be named with some date prefix and some characters plus metadata.xml
Copy the values for the 2nd and 3rd entry. You’ll also need these for Okta.

Click on Choose file under Identity provider metadata and upload the metadata.xml file that you’ve downloaded earlier from Okta.

Click Next, type ACCEPT and finally click Change identity source.

Okta

Go back to Okta, Sign On tab and right bellow click Edit. Enter the two URL entries that you copied from AWS. Click Save after.

AWS

Go back to AWS, then IAM Identity Center, click on Settings and click to Enable the Automatic Provisioning.

Click on Show Token and copy both values for the SCIM endpoint and the Access Token.

Okta

Go back to Okta and for the AWS app, click on Provisioning and then click on Configure API Integration.

Click Enable API Integration and enter the SCIM endpoint and the token value in the corresponding fields. Click Save. Remove the trailing slash from the end of the Base URL (SCIM endpoint) if you get an error (Base URL: Does not match required pattern)
Under Provisioning tab, click on To App, click Edit and select all three checkmarks. Click Save.

Then, go to Assignments tab and assign a group or a user to this AWS app.

AWS

Go back to AWS and under IAM Identity Center and then Users, you can see the user that we just assigned.

But this user has no rights in AWS, so we have to create a Permission set and then assign this user to that permission set and the account that this user has access to. Click on Permission sets on the left and then Create permission set.

I chose AdministratorAccess from the Predefined permission sets.

Go back to AWS accounts, click on the account and choose Assign users or groups.

Select the User tab, click on the user, click Next.

Select the permission set that we just created and click Next then Submit.

We just told AWS that the user xyz will have access to the account abc with AdministratorAccess permissions.

Test SSO

Open a new tab and go to the URL that you configured earlier (Customize AWS access portal). It should be https://whatever.awsapps.com/start.
AWS will redirect you to Okta where you’ll enter your Okta username.

Once authenticated, you’ll see the AWS accounts in your org and the permissions that you have.

I wrote about this topic a couple of years ago, but some things changed so I am writing this post again. In this post I’ll explain how to log to AWS Console and AWS cli with Entra ID (former Azure AD) credentials. For this you’ll need admin access to both the AWS account and Entra ID.

IAM Identity Center

The former AWS Single Sign-on is now IAM Identity Center. You need to enable this in order to configure SSO. Go to IAM Identity Center and click on the Enable button.

Enable it with AWS Organizations.

What you want to do first is to change the AWS access portal URL. Click on the Edit button and change the URL. It has to be unique, so don’t expect you can type aws and be done with it.

Another setting that you have to change is the Identity source. On the left side of the IAM Identity Center, you are on the Dashboard screen. Click on Settings and then click on Change identity source.

Choose External identity provider and click Next.

You will see this screen which consists of two parts, service provider metatada (that’s AWS) and Identity provider metadata (that’s Entra ID). Click on Download metadata file. You’ll get an XML file.

Keep the page open and in a new browser window go to portal.azure.com.

Entra ID

Once logged to Azure, go to Enterprise Applications and search for AWS IAM Identity Center (successor to AWS Single Sign-On). Click to install. When you are presented with the Overview of the application, click on Assign users and groups. Depending on your Entra ID level (P1 or P2) you might not be able to assign groups, so you have to assign users.

The next step is to setup the SSO. Click on 2. Set up single sign on and then SAML. You’ll be presented with this page and eventually you’ll see a prompt if you want to save some SSO settings. If you don’t see the prompt, no worries. If you see the prompt, click Yes.

Click on Upload metadata file and point to the XML file that you saved from AWS. Click Save and don’t worry about any of the fields there. If you see a popup to test the config, just click No, I’ll test later. Under step 3, SAML Certificates, click to Download Federation Metadata XML.

You’ll get another XML file called AWS IAM Identity Center (successor to AWS Single Sign-On).xml.

Back to AWS

Go back to the same page where you’ve downloaded the XML metadata file, but this time look at the bottom part where it says Identity provider metadata and IdP SAML metadata. Click on Choose file and upload the file from Azure. Click Next, type ACCEPT and click Change identity source.
Once completed click to Enable Automatic provisioning. Copy the SCIM endpoint and the access token.

Back to Azure

Under the AWS SSO application. on the left side click on Provisioning. Change the Provisioning Mode to Automatic. Expand Admin credentials and for Tenant ID enter the SCIM endpoint and for Secret Token, enter the token. Click Test Connection and upon successful test, click Save. If you see clientsecret instead of Tenant ID, it means you are not using the right AWS application and you use the AWS Single Account application instead. Start from scratch.
Anytime you assign a new user or a group to AWS application, Azure will sync the user to AWS. But, instead of waiting (40 mins by default) for the initial sync, click on the Overview and click Start provisioning.

Back to AWS

If your provisioning works fine, you’ll see your assigned Azure user under Users in IAM Identity Center. On the left side of the IAM Identity Center, click on Permission sets. Click Create Permission set and choose Predefined permission set, then AdministratorAccess. Change the name, description, session duration if needed and then click Next and then Create.
Finally, click on AWS accounts, select an account from the organization, click Assign users or groups, click Users or Groups, select a user and click Next. Assign a permission set, click Next and Submit.

Now, test the SSO by going to whatever.awsapps.com/start# and you’ll be redirected to the Microsoft sign in prompt. Follow the authentication process for Entra ID (username, password, MFA etc) and once you pass the authentication process you’ll see the AWS account that you have access to.

AWS CLI and SSO

Follow the instructions to install AWS CLI for your OS, then configure SSO. Name your session, enter the URL you configured earlier, enter the region where you configured SSO and hit enter for the registration scopes.

aws configure sso
SSO session name (Recommended): sso
SSO start URL [None]: https://your_url.awsapps.com/start
SSO region [None]: us-east-1
SSO registration scopes [sso:account:access]:
Attempting to automatically open the SSO authorization page in your default browser.
If the browser does not open or you wish to use a different device to authorize this request, open the following URL:

https://device.sso.us-east-1.amazonaws.com/

Then enter the code:

PWKG-CAFZ

After successful authentication, you’ll be presented with the accounts available to you. Pick one.

You’ll be prompted to choose the roles available to you and then specify your default region, default output and the profile. \
NOTE: If you’ll be working with multiple accounts, maybe you should specify a different profile than the default. It’s up to you. E.g. specify prod and dev profiles.

There are 2 AWS accounts available to you.
Using the account ID 123456789012
The only role available to you is: AdministratorAccess
Using the role name "AdministratorAccess"
CLI default client Region [None]: us-east-2
CLI default output format [None]: json
CLI profile name [AdministratorAccess-123456789012]: default

To use this profile, specify the profile name using --profile, as shown:

aws s3 ls --profile default

After your SSO session expires, you have to log back in.

aws sso login

If you are like me and always uses the default profile so I don’t have to type –profile parameter, then anytime you want to use a different account, just aws configure sso again, but specify the other account now.

In one of my previous posts, I wrote about accessing DynamoDB and AWS Gateway/Lambda using IAM roles and secrets from EC2/EKS Node.js application. The idea is to reference passwords in your code and never store and secrets/passwords in the code. By using roles, we can access the secrets from the AWS Secrets Manager and use them dynamically in the code. However, that solution has its own pros and cons. The pros are that you don’t have to do anything extra except creating an IAM role and secrets. The cons are that you need AWS SDK as part of your application. You will use AWS SDK for Node.js to get the credentials from the Secrets Manager. This means you have to maintain up-to-date AWS SDK and add extra code. Another way is to use the k8s CSI driver and ASCP (provider) that pretty much gets the secret and mounts it as a file. You still need IAM roles to access the Secrets Manager but you don’t need AWS SDK. You can read the secrets as you read a file.

Create an EKS cluster

We’ll create an EKS cluster using the eksctl tool.

CLUSTER_NAME="eksDemoCluster"
REGION="us-east-2"
NAMESPACE="default"

eksctl create cluster --name $CLUSTER_NAME \
  --region $REGION --version 1.28 \
  --node-type t3.small --nodes 2

We’ll use helm to install the CSI driver and the specific ASCP provider for AWS.

helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
helm repo add aws-secrets-manager https://aws.github.io/secrets-store-csi-driver-provider-aws
helm repo update
helm install -n kube-system csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver
helm install -n kube-system secrets-provider-aws aws-secrets-manager/secrets-store-csi-driver-provider-aws

Check if everything is OK.

kubectl --namespace=kube-system get pods -l "app=secrets-store-csi-driver"
NAME                                               READY   STATUS    RESTARTS   AGE
csi-secrets-store-secrets-store-csi-driver-nhq7b   3/3     Running   0          19s
csi-secrets-store-secrets-store-csi-driver-tft6q   3/3     Running   0          19s

Create a secret

Let’s create a secret that we’ll be using in a mock-up app.

SECRET_NAME="demoapp/mysecret"
SECRET_ARN=$(aws secretsmanager create-secret  --name $SECRET_NAME \
  --secret-string '{"username":"Administrator", "password":"SuperSecretPassword$"}' \
  --output text --query 'ARN' --region $REGION)
echo $SECRET_ARN
arn:aws:secretsmanager:us-east-2:123456789012:secret:demoapp/mysecret-lboyfd

Create an IAM policy

We also need to define an IAM policy that will allow our EKS pods to access only this particular secret and nothing else.

cat << EOF > polDemoSecret.json
{
    "Version": "2012-10-17",
    "Statement": [ {
        "Effect": "Allow",
        "Action": ["secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret"],
        "Resource": ["$SECRET_ARN"]
    } ]
}
EOF

Create the policy.

POLICY_ARN=$(aws iam create-policy --policy-name polDemoSecret \
  --policy-document file://polDemoSecret.json \
  --output text --query 'Policy.Arn')
echo $POLICY_ARN
arn:aws:iam::123456789012:policy/polDemoSecret

Create an IAM OIDC provider

Let’s determine whether we have an existing IAM OIDC provider for our cluster.

aws eks describe-cluster --name $CLUSTER_NAME --region $REGION \
  --query "cluster.identity.oidc.issuer" --output text
https://oidc.eks.us-east-2.amazonaws.com/id/D7FAD6F2BEF430FC1CB673777A9E4FED

When we provisioned the cluster with eksctl, the OIDC was created automatically. We need the ID of the OIDC which is the hex value at the end.

OIDCID=$(aws eks describe-cluster --name $CLUSTER_NAME --region $REGION \
  --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
echo $OIDCID
E9B5E80C37A71FA3932B94EFF5BA3437

Check if the IAM OIDC provider is already configured. It shouldn’t if you provisioned a brand new cluster.

aws iam list-open-id-connect-providers | grep $OIDCID 

If for whatever reason, you had an output from the command above, skip this associate-iam-oidc-provider step below.
If you don’t have any output, associate the OIDC provider with the cluster.

eksctl utils associate-iam-oidc-provider --cluster $CLUSTER_NAME --region $REGION --approve
2023-12-22 12:27:47 [ℹ]  will create IAM Open ID Connect provider for cluster "eksDemoCluster" in "us-east-2"
2023-12-22 12:27:47 [✔]  created IAM Open ID Connect provider for cluster "eksDemoCluster" in "us-east-2"

Cluster service account

We also need a service account for the cluster and we need to attache the policy for the secrets that we created before.

SA_NAME="sademosecret"
eksctl create iamserviceaccount --name $SA_NAME --namespace $NAMESPACE \
--cluster $CLUSTER_NAME  --region $REGION \
--attach-policy-arn $POLICY_ARN --approve --override-existing-serviceaccounts
kubectl get sa

Service Provider Class

…and finally we need a service provider class that we’ll deploy on the EKS cluster.

cat << EOF > spc.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
  name: spc-demosecret
  namespace: $NAMESPACE
spec:
  provider: aws
  parameters:
    objects: |
        - objectName: "$SECRET_NAME"
          objectType: "secretsmanager"
EOF
kubectl apply -f spc.yaml

Test with busybox pod

Create this YAML file that will deploy a pod. Pay attention to highlighted lines. We have to provide the namespace, the service account and the service provider class that specifies the secret.Check this link on how to mount and specify different secrets.

cat << EOF > app.yaml
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: $NAMESPACE
spec:
  serviceAccountName: $SA_NAME
  volumes:
  - name: secretsvolume
    csi:
       driver: secrets-store.csi.k8s.io
       readOnly: true
       volumeAttributes:
         secretProviderClass: "spc-demosecret"
  containers:
  - image: public.ecr.aws/docker/library/busybox:1.36
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
    name: busybox
    volumeMounts:
    - name: secretsvolume
      mountPath: "/mnt/secrets-store"
      readOnly: true
  restartPolicy: Always
EOF
kubectl apply -f app.yaml

Test the secrets retrieval. As you can see, the secret that we created was demoapp/mysecret, but in this case the forward slash would mean a directory, so the actual secret is demoapp_mysecret… and it’s simple as that. The secret is just a file under the /mnt/secrets-store directory.

kubectl exec -it $(kubectl get pods | awk '/busybox/{print $1}' | head -1) -- cat /mnt/secrets-store/demoapp_mysecret; echo
{"username":"Administrator", "password":"SuperSecretPassword$"}

Node.js app

Create a folder and add this file. Save it as server.js.

'use strict';
const express = require('express')
const app = express()
const port = 3000
 
var fs = require('fs');
try {  
    var data = fs.readFileSync('/mnt/secrets-store/demoapp_mysecret', 'utf8');
    console.log(data.toString());    
} catch(e) {
    console.log('Error:', e.stack);
}

app.get('/', (req, res) => {
    res.send(JSON.stringify(data));
})
 
app.listen(port, () => {
  console.log(`Example app listening on port ${port}`)
})

Let’s create the Dockerfile so we can build the image.

FROM node:20
  
# Create app directory
WORKDIR /usr/src/app
  
# Install app dependencies
# A wildcard is used to ensure both package.json AND package-lock.json are copied
# where available (npm@5+)
COPY package*.json ./
  
RUN npm install
# If you are building your code for production
# RUN npm ci --only=production
  
# Bundle app source
COPY . .
  
EXPOSE 3000
CMD [ "node", "server.js" ]

Create a .dockerignore file.

node_modules
npm-debug.log

Now, we can create the image and push to Dockerhub or some other image repo. Use your own repo here at line 1.

sudo docker build . -t klimenta/secretsdemo
sudo docker images
sudo docker push klimenta/secretsdemo

Create a file to deploy 2 replicas and a load balancer. Save it as something.yaml and deploy it. Analyze the file and see how we use the service account, secret provider class and the image.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demosecret
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      run: demosecret
  template:
    metadata:
      labels:
        run: demosecret
    spec:
      serviceAccountName: sademosecret
      volumes:
      - name: secretsvolume
        csi:
          driver: secrets-store.csi.k8s.io
          readOnly: true
          volumeAttributes:
            secretProviderClass: "spc-demosecret"
      containers:
      - name: demosecret
        image: klimenta/secretsdemo
        volumeMounts:
        - name: secretsvolume
          mountPath: "/mnt/secrets-store"
          readOnly: true
        ports:
        - containerPort: 3000
---
apiVersion: v1
kind: Service
metadata:
  name: loadbalancer
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: nlb
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
spec:
  ports:
    - port: 80
      targetPort: 3000
      protocol: TCP
  type: LoadBalancer
  selector:
    run: demosecret

Once you deploy with kubectl apply -f something.yaml, type kubectl get svc and you’ll see a public URL.
If you go to that URL, you’ll see our secret.

In this post, I’ll explain how to create an EKS cluster in AWS and add an Ingress Controller. We’ll use GitLab as a repo and execute the CI/CD pipeline from there.
For this you’ll need:

• GitLab and AWS account
• Terraform, AWS CLI, helm installed

AWS

From the IAM console, create a new Identity Provider in AWS and use https://gitlab.com for both Provider URL and Audience. For self hosted GitLab use gitlab.domain.com or whatever is your GitLab URL.
Click on Get thumbprint before clicking on Add provider.

In the IAM console, create a role. I’ll name the role rolGitLab, name it whatever you like. When you create the policy, choose Custom trust policy.

Now, here is what you have to decide. If you want to manage the cluster from GitLab only, you can ommit lines 16-23 where I allow the IAM user called gitlab to assume this role. But if you want some IAM user to manage the cluster using aws cli or kubectl, add the ARN of that IAM user in line 20. Add AdministratorAccess to that gitlab user but you can fine-tune those policies later. If you want to manage the cluster from GitLab, you have to register an agent and provide a config file. You can read how to do that here.
Replace the default custom trust policy with this JSON.

{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Effect": "Allow",
			"Principal": {
				"Federated": "arn:aws:iam::123456789012:oidc-provider/gitlab.com"
			},
			"Action": "sts:AssumeRoleWithWebIdentity",
			"Condition": {
				"StringLike": {
					"gitlab.com:sub": "project_path:*/eks-gitlab:ref_type:branch:ref:main"
				}
			}
		},
		{
			"Sid": "",
			"Effect": "Allow",
			"Principal": {
				"AWS": "arn:aws:iam::123456789012:user/gitlab"
			},
			"Action": "sts:AssumeRole"
		}
	]
}

In line 7 and 20, change the 123456789012 with your account number and in line 7 change gitlab.com with however you named the Identity Provider (IdP). In line 12 replace eks-gitlab with how you plan to name your GitLab project. In line 20 change the gitlab user to however you named your IAM user to access the EKS cluster. Click Next and when asked for a permission policy, choose AdministratorAccess. Best practice is to limit access to only what’s needed, but EKS has so many policies, it will take a lot to list them all.
In case you created the gitlab IAM user, attach the AdministratorPolicy (fine tune later) and this inline policy. Change the ARN in line 7. Name this inline policy something like polAssumeGitLabRole.

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": "sts:AssumeRole",
            "Resource": "arn:aws:iam::123456789012:role/rolGitLab"
        }
    ]
}

GitLab

Create a new blank project in Gitlab with the default settings. In my case the project name is called eks-gitlab.

Go to Settings on the left, then CI/CD and then Variables. Add these two variables.

• AWS_PROFILE and a value of oidc. Yes, the value is oidc. Just type these 4 characters for the value.
• ROLE_ARN and a value of the role ARN from the role you just created for rolGitLab

Create a .gitlab-ci.yml file with the following content. Change the domain for gitlab.com at line 7 if you use self-hosted GitLab.

assume role:
  image: 
    name: amazon/aws-cli:latest
    entrypoint: [""]
  id_tokens:
    GITLAB_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - mkdir -p ~/.aws
    - echo "${GITLAB_OIDC_TOKEN}" > /tmp/web_identity_token
    - echo -e "[profile oidc]\nrole_arn=${ROLE_ARN}\nweb_identity_token_file=/tmp/web_identity_token" > ~/.aws/config
  script:
      - aws sts get-caller-identity
      - aws s3 ls

By the time you commit changes, the pipeline will start and if everything is OK, you’ll see your S3 buckets in the logs of the job if you go to Build | Jobs.

One more thing related to Terraform state as we’ll keep the state in GitLab is that we have to create an Access Token. If you have a self-hosted or paid GitLab account, create an Access Token from the Project | Settings | Access Tokens.
If you have a free GitLab account, you have to create a Personal Access Token and you do that if you click on your avatar in the upper left corner, then select Edit Profile and then click Access Tokens on the left. Click Add new token, name it GITLAB_PAT and select API only. Click Create personal access token. Once the token has been created click on the eye icon where it says Your new personal access token and copy & paste somewhere safe.

Go to the project settings and add these 2 variables.

• TF_USERNAME which is your username, click on the avatar, look at the @ under your name, your username is without the @ sign
• TF_PASSWORD which is the value of your personal access token you just created

Local machine

Now that we know the pipeline is working properly, let’s clone the project on our local machine.
If you plan to manage the cluster from your local machine, now it’s time to configure the gitlab IAM user on your local machine using access keys.
You need to have a valid key in order to clone your private repo.
Upload your .pub key to GitLab by clicking your avatar, then Edit Profile, click SSH Keys and then Add new key.
Finally, check the connection with the following command. It doesn’t matter what your local Linux user is, just use git@gitlab.com as the username@hostname. Type as is.

ssh -T git@gitlab.com
Welcome to GitLab, @[your_gitlab_username_here]

Once you clone the repo add the following files locally.
FILE: .gitignore

# Local .terraform directories
**/.terraform/*

# .tfstate files
*.tfstate
*.tfstate.*

# Crash log files
crash.log
crash.*.log

# Exclude all .tfvars files, which are likely to contain sensitive data, such as
# password, private keys, and other secrets. These should not be part of version 
# control as they are data points which are potentially sensitive and subject 
# to change depending on the environment.
*.tfvars
*.tfvars.json

# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json

# Include override files you do wish to add to version control using negated pattern
# !example_override.tf

# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*

# Ignore CLI configuration files
.terraformrc
terraform.rc

FILE: LICENSE

This is free and unencumbered software released into the public domain.

Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.

In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

For more information, please refer to <https://unlicense.org>

FILE: alb_ingress_iam_policy.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "iam:CreateServiceLinkedRole"
            ],
            "Resource": "*",
            "Condition": {
                "StringEquals": {
                    "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:DescribeAccountAttributes",
                "ec2:DescribeAddresses",
                "ec2:DescribeAvailabilityZones",
                "ec2:DescribeInternetGateways",
                "ec2:DescribeVpcs",
                "ec2:DescribeVpcPeeringConnections",
                "ec2:DescribeSubnets",
                "ec2:DescribeSecurityGroups",
                "ec2:DescribeInstances",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DescribeTags",
                "ec2:GetCoipPoolUsage",
                "ec2:DescribeCoipPools",
                "elasticloadbalancing:DescribeLoadBalancers",
                "elasticloadbalancing:DescribeLoadBalancerAttributes",
                "elasticloadbalancing:DescribeListeners",
                "elasticloadbalancing:DescribeListenerCertificates",
                "elasticloadbalancing:DescribeSSLPolicies",
                "elasticloadbalancing:DescribeRules",
                "elasticloadbalancing:DescribeTargetGroups",
                "elasticloadbalancing:DescribeTargetGroupAttributes",
                "elasticloadbalancing:DescribeTargetHealth",
                "elasticloadbalancing:DescribeTags"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "cognito-idp:DescribeUserPoolClient",
                "acm:ListCertificates",
                "acm:DescribeCertificate",
                "iam:ListServerCertificates",
                "iam:GetServerCertificate",
                "waf-regional:GetWebACL",
                "waf-regional:GetWebACLForResource",
                "waf-regional:AssociateWebACL",
                "waf-regional:DisassociateWebACL",
                "wafv2:GetWebACL",
                "wafv2:GetWebACLForResource",
                "wafv2:AssociateWebACL",
                "wafv2:DisassociateWebACL",
                "shield:GetSubscriptionState",
                "shield:DescribeProtection",
                "shield:CreateProtection",
                "shield:DeleteProtection"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:AuthorizeSecurityGroupIngress",
                "ec2:RevokeSecurityGroupIngress"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:CreateSecurityGroup"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:CreateTags"
            ],
            "Resource": "arn:aws:ec2:*:*:security-group/*",
            "Condition": {
                "StringEquals": {
                    "ec2:CreateAction": "CreateSecurityGroup"
                },
                "Null": {
                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:CreateTags",
                "ec2:DeleteTags"
            ],
            "Resource": "arn:aws:ec2:*:*:security-group/*",
            "Condition": {
                "Null": {
                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:AuthorizeSecurityGroupIngress",
                "ec2:RevokeSecurityGroupIngress",
                "ec2:DeleteSecurityGroup"
            ],
            "Resource": "*",
            "Condition": {
                "Null": {
                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:CreateLoadBalancer",
                "elasticloadbalancing:CreateTargetGroup"
            ],
            "Resource": "*",
            "Condition": {
                "Null": {
                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:CreateListener",
                "elasticloadbalancing:DeleteListener",
                "elasticloadbalancing:CreateRule",
                "elasticloadbalancing:DeleteRule"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:AddTags",
                "elasticloadbalancing:RemoveTags"
            ],
            "Resource": [
                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
            ],
            "Condition": {
                "Null": {
                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:AddTags",
                "elasticloadbalancing:RemoveTags"
            ],
            "Resource": [
                "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
                "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
                "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
                "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:ModifyLoadBalancerAttributes",
                "elasticloadbalancing:SetIpAddressType",
                "elasticloadbalancing:SetSecurityGroups",
                "elasticloadbalancing:SetSubnets",
                "elasticloadbalancing:DeleteLoadBalancer",
                "elasticloadbalancing:ModifyTargetGroup",
                "elasticloadbalancing:ModifyTargetGroupAttributes",
                "elasticloadbalancing:DeleteTargetGroup"
            ],
            "Resource": "*",
            "Condition": {
                "Null": {
                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:AddTags"
            ],
            "Resource": [
                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
            ],
            "Condition": {
                "StringEquals": {
                    "elasticloadbalancing:CreateAction": [
                        "CreateTargetGroup",
                        "CreateLoadBalancer"
                    ]
                },
                "Null": {
                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:RegisterTargets",
                "elasticloadbalancing:DeregisterTargets"
            ],
            "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "elasticloadbalancing:SetWebAcl",
                "elasticloadbalancing:ModifyListener",
                "elasticloadbalancing:AddListenerCertificates",
                "elasticloadbalancing:RemoveListenerCertificates",
                "elasticloadbalancing:ModifyRule"
            ],
            "Resource": "*"
        }
    ]
}

FILE: backend.tf

terraform {
  backend "http" {
  }
}

FILE: data.tf

data "aws_availability_zones" "available" {
}

data "aws_eks_cluster" "cluster" {
  name = module.eks.cluster_name
  depends_on = [module.eks.cluster_name]
}

data "aws_eks_cluster_auth" "cluster" {
  name = module.eks.cluster_name
  depends_on = [module.eks.cluster_name]
}

FILE: main.tf NOTE: Make sure you add lines 40-41 and specify the rolGitlab role created earlier. If you don’t do this when you destroy the cluster, the KMS key for EKS will stay there and there is no way you can delete it unless you call AWS support.

# Create a VPC
module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "5.0.0"

  name = var.vpcName

  cidr = var.vpcCIDR
  azs  = slice(data.aws_availability_zones.available.names, 0, 3)

  private_subnets = var.privateSubnets
  public_subnets  = var.publicSubnets

  enable_nat_gateway   = var.enable_nat_gateway
  single_nat_gateway   = var.single_nat_gateway
  enable_dns_hostnames = var.enable_dns_hostnames

  public_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/elb"                    = 1
  }

  private_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/internal-elb"           = 1
  }
}

# Create an EKS cluster
module "eks" {
  source  = "terraform-aws-modules/eks/aws"
  version = "19.15.3"

  cluster_name    = var.cluster_name
  cluster_version = var.cluster_version

  vpc_id                         = module.vpc.vpc_id
  subnet_ids                     = module.vpc.private_subnets
  cluster_endpoint_public_access = var.cluster_endpoint_public_access
  kms_key_administrators         = ["arn:aws:iam::123456789012:role/rolGitLab"]
  kms_key_enable_default_policy  = true

  eks_managed_node_group_defaults = {
    ami_type = var.ami_type
  }

  eks_managed_node_groups = {
    one = {
      name = var.nodeGroup

      instance_types = var.instanceTypes

      min_size     = var.minSize
      max_size     = var.maxSize
      desired_size = var.desiredSize
    }
  }
}

# Create an IAM policy for the ALB ingress
resource "aws_iam_policy" "worker_policy" {
  name        = "worker-policy"
  description = "Worker policy for the ALB Ingress"

  policy = file(var.iamALBIngressPolicyFileName)
}

resource "aws_iam_role_policy_attachment" "additional" {
  for_each = module.eks.eks_managed_node_groups

  policy_arn = aws_iam_policy.worker_policy.arn
  role       = each.value.iam_role_name
}

# Crete the Ingress using Helm chart from AWS
resource "helm_release" "ingress" {
  name       = "ingress"
  chart      = "aws-load-balancer-controller"
  repository = "https://aws.github.io/eks-charts"
  version    = "1.6.1"

  set {
    name  = "autoDiscoverAwsRegion"
    value = "true"
  }

  set {
    name  = "autoDiscoverAwsVpcID"
    value = "true"
  }

  set {
    name  = "clusterName"
    value = module.eks.cluster_name
  }
}

FILE: outputs.tf

output "cluster_endpoint" {
  description = "Endpoint for EKS control plane"
  value       = module.eks.cluster_endpoint
}

output "cluster_security_group_id" {
  description = "Security group ids attached to the cluster control plane"
  value       = module.eks.cluster_security_group_id
}

output "region" {
  description = "AWS region"
  value       = var.region
}

output "cluster_name" {
  description = "Kubernetes Cluster Name"
  value       = module.eks.cluster_name
}

FILE: providers.tf

provider "aws" {
  region = var.region

  shared_config_files      = ["~/.aws/config"]  
  profile                  = "oidc"
}

provider "helm" {
  kubernetes {
    host                   = data.aws_eks_cluster.cluster.endpoint
    token                  = data.aws_eks_cluster_auth.cluster.token
    cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
  }
}

FILE: variables.tf

variable "region" {
  type    = string
  default = "us-east-2"
}

variable "vpcName" {
  type    = string
  default = "vpcFromGitLab"
}

variable "vpcCIDR" {
  type    = string
  default = "10.0.0.0/16"
}

variable "privateSubnets" {
  type    = list(string)
  default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
}

variable "publicSubnets" {
  type    = list(string)
  default = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
}

variable "enable_nat_gateway" {
  type    = bool
  default = true
}

variable "single_nat_gateway" {
  type    = bool
  default = false
}

variable "enable_dns_hostnames" {
  type    = bool
  default = true
}

variable "cluster_name" {
  type    = string
  default = "eksFromGitLab"
}

variable "cluster_version" {
  type    = string
  default = "1.27"
}

variable "cluster_endpoint_public_access" {
  type    = bool
  default = true
}

variable "ami_type" {
  type    = string
  default = "AL2_x86_64"
}

variable "nodeGroup" {
  type    = string
  default = "ngFromGitLab"
}

variable "instanceTypes" {
  type    = list(string)
  default = ["t3.small"]
}

variable "minSize" {
  type    = number
  default = 1
}

variable "maxSize" {
  type    = number
  default = 3
}

variable "desiredSize" {
  type    = number
  default = 3
}


variable "iamALBIngressPolicyFileName" {
  type    = string
  default = "alb_ingress_iam_policy.json"
}

…and finally your .gitlab-ci.yml file.
FILE: .gitlab-ci.yml. NOTE: Remove line 38 if you don’t use it, don’t keep it with a comment or you’ll get an error in the pipeline.

stages:          
  - validate
  - plan
  - apply
  - destroy

image:
  name: hashicorp/terraform:1.6
  entrypoint:
    - "/usr/bin/env"
    - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

variables:
  TF_DIR: ${CI_PROJECT_DIR}
  TF_STATE_NAME: "eks-tf-state"
  TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}"

default:
  id_tokens:
    GITLAB_OIDC_TOKEN:
      aud: https://gitlab.com

  before_script:
    - terraform --version
    - cd ${TF_DIR}
    - terraform init
      -reconfigure 
      -backend-config="address=${TF_ADDRESS}" 
      -backend-config="lock_address=${TF_ADDRESS}/lock" 
      -backend-config="unlock_address=${TF_ADDRESS}/lock" 
      -backend-config="username=${TF_USERNAME}" 
      -backend-config="password=${TF_PASSWORD}" 
      -backend-config="lock_method=POST" 
      -backend-config="unlock_method=DELETE" 
      -backend-config="retry_wait_min=5"
    - mkdir -p ~/.aws
    - echo "${GITLAB_OIDC_TOKEN}" > /tmp/web_identity_token
    - # echo "${GITLAB_OIDC_TOKEN}" | base64 - If you want to see the token, reveal it this way and then use base64 --decode to get the real one
    - echo -e "[profile oidc]\nrole_arn=${ROLE_ARN}\nweb_identity_token_file=/tmp/web_identity_token" > ~/.aws/config

job-validate:
  stage: validate
  script:
    - terraform validate

job-plan:
  stage: plan
  script:
    - terraform plan -out "tf_plan"
  dependencies:
    - job-validate
  artifacts:
    paths:
      - tf_plan

job-apply:
  stage: apply
  script:
    - terraform apply "tf_plan"
  dependencies:
    - job-plan
  allow_failure: true

job-destroy:
  stage: destroy
  script:
    - terraform destroy  -auto-approve
  dependencies:
    #- job-plan
    #- job-apply
  when: manual

Push the code back to Gitlab (git add, git commit, git push) and once everything completes successfully, you’ll have a working EKS cluster after 10-15 mins.
If you choose to manage the EKS from GitLab, you can skip to the next section. If you decide to manage the cluster outside GitLab and you already have another IAM user added to the role rolGitLab as described above, then do the following. Replace account # in line 1.

aws eks update-kubeconfig --region us-east-2 --name eksFromGitLab --role-arn arn:aws:iam::123456789012:role/rolGitLab
aws eks list-clusters
eksctl get cluster --name=eksFromGitLab --region us-east-2
kubectl get svc

Manage EKS from GitLab

Generally, you would want this to be in another project. Our first project was to build the cluster, but our second project builds on top of that and manages the cluster. In addition, the second project also uses the CI/CD pipeline file .gitlab-ci.yml so in order to avoid a single complex project – just create a new project. I’ll name my new project – project1. Clone the project on your local machine using git clone.
Create an empty config file for the agent on your local machine. It has to be under this folder structure.

mkdir -p .gitlab/agents/eks-agent && touch .gitlab/agents/eks-agent/config.yaml

In my case the name of the agent is eks-agent. Use whatever name you want but follow DNS naming convention.
Push the local changes to GitLab.
On the left side in GitLab, go to Operate | Kubernetes clusters and then click Connect a cluster from the upper right corner.
Select the agent from the drop down and click Register.

You’ll get an access token and a bunch of helm commands to install the agent on the cluster. Copy and paste and run those commands.

helm repo add gitlab https://charts.gitlab.io
helm repo update
helm upgrade --install eks-agent gitlab/gitlab-agent \
    --namespace gitlab-agent-eks-agent \
    --create-namespace \
    --set image.tag=v16.9.0-rc2 \
    --set config.token=glagent-2r_uW63e1w_mkNzR_keU2hmaNhhcKauSpujg8OdLQG74wrF73g \
    --set config.kasAddress=wss://kas.gitlab.com

Refresh the GitLab screen after the helm chart has been installed and you’ll see your cluster connected.

Now you need to authorize the agent to access our project.
In the blank config.yaml file on the local machine under .gitlab/agents/eks-agent/ directory, add these lines. Change the path the project in line 3 (look at the URL, that’s your project name after gitlab.com) and change the manifest file if you want in line 6. You can also change the namespace in line 5 as well.

gitops:
  manifest_projects:
  - id: andreevkliment/project1
    default_namespace: default
    paths:
      - glob: 'manifest.yaml'

Then create the manifest file under the root of the project. Save it as manifest.yaml or however you referenced it in line 6 in the previous file.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 2
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.14.2
        ports:
        - containerPort: 80

If you push this code back to GitLab, the agent will execute it and you’ll (with kubect get pods) see the nginx pods running on your cluster.
Another way of managing the EKS cluster is using .gitlab-ci.yml file.
Create .gitlab-ci.yml with these lines. Replace your username, project and the agent name in line 7.

deploy:
  image:
    name: bitnami/kubectl:latest
    entrypoint: ['']
  script:
    - kubectl config get-contexts
    - kubectl config use-context andreevkliment/project1:eks-agent
    - kubectl get pods

If you push this back to GitLab, the pipeline will start and you’ll see the pods from the cluster in the logs. This example doesn’t execute manifest.yaml. You can have both ways of interacting with the cluster or you can use them separately.

In this post I’ll explain how to access DynamoDB table and Secrets Manager from an EC2 instance running a small Node.js/Express app. The app will access the DynamoDB and print an entry from the table and also access an API Gateway that will call a lambda function that returns the current date and time. The API Gateway will be protected with a header authentication and we’ll store the header auth token in Secrets Manager, which means our app will have to access the Secret Manager as well. Finally, we’ll containerize that app as a Docker image and run it as a Deployment behind a load balancer on EKS.
There are a lot of steps, so make sure to use the same variable names and pay attention to the outputs.
NOTE: There are multiple ways of accessing Secrets Manager. The one described here is with roles and AWS SDK. There is another way of accessing Secrets Manager from EKS using Kubernetes CSI driver, so make sure you read this post too if you use EKS and Secrets Manager only.

Lambda and API Gateway

Let’s create a Lambda function and an API Gateway so when the API Gateway is called with a parameter, it will write the parameter, current date/time and on top of that we’ll protect the API Gateway so it can be called only if we pass a password/token.
Define the default region.

AWS_DEFAULT_REGION="us-east-2"

Get the account number.

ACCT_NO=$(aws sts get-caller-identity --query "Account" --output text)
echo $ACCT_NO

This is the Hello World Node.js Lambda function source.

cat <<'EOF' > helloworld.js
'use strict';
 
exports.handler = async (event) => {
    let name = "";
    let responseCode = 200;
    const date = new Date();
    
    if (event.queryStringParameters && event.queryStringParameters.name) {
        name = event.queryStringParameters.name;
    }
    
    if (event.body) {
        let body = JSON.parse(event.body)
    }
 
    let greeting = `Hello ${name}, it's ${date}`;
    

    let responseBody = {
        message: greeting,
        input: event
    };
    
    let response = {
        statusCode: responseCode,
        headers: {
            "x-custom-header" : "my custom header value"
        },
        body: JSON.stringify(responseBody)
    };
    console.log("response: " + JSON.stringify(response))
    return response;
};
EOF

Zip the function so we can deploy it.

zip helloworld.zip helloworld.js

We need a Lambda role that allows execution.

cat <<EOF > lambda-role.json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
      "Service": [
        "lambda.amazonaws.com"
      ]
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
EOF

Create the role and attach it to a policy.

ROLE_ARN=$(aws iam create-role --role-name rolLambdaExecutionRole \
  --assume-role-policy-document file://lambda-role.json --output text --query 'Role.Arn')
echo $ROLE_ARN
aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole \
  --role-name rolLambdaExecutionRole

Create the Lambda function.

LAMBDA_HELLOWORLD_ARN=$(aws lambda create-function --function-name HelloWorld \
  --zip-file fileb://helloworld.zip --runtime nodejs18.x --role $ROLE_ARN \
  --handler helloworld.handler --output text --query 'FunctionArn')
echo $LAMBDA_HELLOWORLD_ARN

We also need another Lambda function that will be used as an authorizer. This will allow us to protect the API Gateway with authentication. The authentication parameters are in line 27 here on the screen.(headerauth1, password).

cat << EOF > authorize.js
exports.handler = function(event, context, callback) {
    console.log('Received event:', JSON.stringify(event, null, 2));

    // Retrieve request parameters from the Lambda function input:
    var headers = event.headers;
        
    // Parse the input for the parameter values
    var tmp = event.methodArn.split(':');
    var apiGatewayArnTmp = tmp[5].split('/');
    var awsAccountId = tmp[4];
    var region = tmp[3];
    var restApiId = apiGatewayArnTmp[0];
    var stage = apiGatewayArnTmp[1];
    var method = apiGatewayArnTmp[2];
    var resource = '/'; // root resource
    if (apiGatewayArnTmp[3]) {
        resource += apiGatewayArnTmp[3];
    }
        
    // Perform authorization to return the Allow policy for correct parameters and 
    // the 'Unauthorized' error, otherwise.
    var authResponse = {};
    var condition = {};
    condition.IpAddress = {};
     
    if (headers.headerauth1 === "password") 
    {
        callback(null, generateAllow('me', event.methodArn));
    }  else {
        callback("Unauthorized");
    }
}
     
// Help function to generate an IAM policy
var generatePolicy = function(principalId, effect, resource) {
    // Required output:
    var authResponse = {};
    authResponse.principalId = principalId;
    if (effect && resource) {
        var policyDocument = {};
        policyDocument.Version = '2012-10-17'; // default version
        policyDocument.Statement = [];
        var statementOne = {};
        statementOne.Action = 'execute-api:Invoke'; // default action
        statementOne.Effect = effect;
        statementOne.Resource = resource;
        policyDocument.Statement[0] = statementOne;
        authResponse.policyDocument = policyDocument;
    }
    // Optional output with custom properties of the String, Number or Boolean type.
    authResponse.context = {
        "stringKey": "stringval",
        "numberKey": 123,
        "booleanKey": true
    };
    return authResponse;
}
     
var generateAllow = function(principalId, resource) {
    return generatePolicy(principalId, 'Allow', resource);
}
     
var generateDeny = function(principalId, resource) {
    return generatePolicy(principalId, 'Deny', resource);
}
EOF

Zip it so we can deploy it.

zip authorize.zip authorize.js

Create the Lambda authorizer function and use the same role as with the Hello World function.

LAMBDA_AUTH_ARN=$(aws lambda create-function --function-name Authorize1 --zip-file fileb://authorize.zip --runtime nodejs18.x \
  --role $ROLE_ARN --handler authorize.handler --output text --query 'FunctionArn')
echo $LAMBDA_AUTH_ARN

Create an API Gateway.

APIGW_ID=$(aws apigateway create-rest-api --name 'gwapihelloworld' \
  --endpoint-configuration '{"types": ["REGIONAL"]}' --output text --query 'id')
echo $APIGW_ID

Create a top level resource.

ROOT_ID=$(aws apigateway get-resources --rest-api-id $APIGW_ID --output text --query 'items[0].id')
echo $ROOT_ID
RES_ID=$(aws apigateway create-resource --rest-api-id $APIGW_ID \
      --parent-id $ROOT_ID \
      --path-part helloworld --output text --query 'id')
echo $RES_ID

Create an ANY method without authorization.

aws apigateway put-method --rest-api-id $APIGW_ID \
       --resource-id $RES_ID \
       --http-method ANY \
       --authorization-type "NONE"

Integrate the Lambda function with the API Gateway.

aws apigateway put-integration \
        --rest-api-id $APIGW_ID \
        --resource-id $RES_ID \
        --http-method ANY \
        --type AWS_PROXY \
        --integration-http-method POST \
        --uri arn:aws:apigateway:$AWS_DEFAULT_REGION:lambda:path/2015-03-31/functions/$LAMBDA_HELLOWORLD_ARN/invocations 

Allow Lambda function to be triggered by the API Gateway.

aws lambda add-permission --function-name $LAMBDA_HELLOWORLD_ARN \
        --action lambda:InvokeFunction --statement-id 'api_gateway' \
        --principal apigateway.amazonaws.com \
        --source-arn "arn:aws:execute-api:$AWS_DEFAULT_REGION:$ACCT_NO:$APIGW_ID/*/*/helloworld"

Create an authorizer in the API Gateway.

AUTH_ID=$(aws apigateway create-authorizer --rest-api-id $APIGW_ID --name 'Authorizer1' \
  --type REQUEST \
  --authorizer-uri "arn:aws:apigateway:$AWS_DEFAULT_REGION:lambda:path/2015-03-31/functions/$LAMBDA_AUTH_ARN/invocations"  \
  --authorizer-result-ttl-in-seconds 300 \
  --identity-source 'method.request.header.headerauth1' \
  --output text --query 'id')
echo $AUTH_ID

Tell the API Gateway to use the authorizer.

aws apigateway update-method --rest-api-id $APIGW_ID --resource-id $RES_ID \
    --patch-operations "op=replace,path=/authorizationType,value=CUSTOM" \
    "op=replace,path=/authorizerId,value=$AUTH_ID" \
    --http-method ANY

Create a deployment and a stage.

STAGE="test"
aws apigateway create-deployment --rest-api-id $APIGW_ID --stage-name $STAGE

Allow Lambda to be triggered by the Authorizer API Gateway.

aws lambda add-permission --function-name $LAMBDA_AUTH_ARN \
        --action lambda:InvokeFunction --statement-id 'api_gateway' \
        --principal apigateway.amazonaws.com \
        --source-arn "arn:aws:execute-api:$AWS_DEFAULT_REGION:$ACCT_NO:$APIGW_ID/authorizers/$AUTH_ID"

Build the API Gateway URL.

APIURL="https://${APIGW_ID}.execute-api.${AWS_DEFAULT_REGION}.amazonaws.com/${STAGE}"
echo $APIURL

Test without and with authorization.

curl -X POST $APIURL/helloworld?name=there
{"message":"Unauthorized"}

These are the parameters that we specified in line 27 in authorize.js file above.

curl -X POST $APIURL/helloworld?name=there -H 'headerauth1 : password'

…and this is the JSON output.

{
  "message": "Hello there, it's Sat Aug 19 2023 12:52:54 GMT+0000 (Coordinated Universal Time)",
  "input": {
    "resource": "/helloworld",
    "path": "/helloworld",
    "httpMethod": "POST",
    "headers": {
      "accept": "*/*",
      "headerauth1": "password",
      "Host": "p8d859ec50.execute-api.us-east-2.amazonaws.com",
      "User-Agent": "curl/7.76.1",
      "X-Amzn-Trace-Id": "Root=1-64e0bb26-5ea6f1225951c3d316e1de77",
      "X-Forwarded-For": "63.81.58.96",
      "X-Forwarded-Port": "443",
      "X-Forwarded-Proto": "https"
    },
    "multiValueHeaders": {
      "accept": [
        "*/*"
      ],
      "headerauth1": [
        "password"
      ],
      "Host": [
        "p8d859ec50.execute-api.us-east-2.amazonaws.com"
      ],
      "User-Agent": [
        "curl/7.76.1"
      ],
      "X-Amzn-Trace-Id": [
        "Root=1-64e0bb26-5ea6f1225951c3d316e1de77"
      ],
      "X-Forwarded-For": [
        "63.81.58.96"
      ],
      "X-Forwarded-Port": [
        "443"
      ],
      "X-Forwarded-Proto": [
        "https"
      ]
    },
    "queryStringParameters": {
      "name": "there"
    },
    "multiValueQueryStringParameters": {
      "name": [
        "there"
      ]
    },
    "pathParameters": null,
    "stageVariables": null,
    "requestContext": {
      "resourceId": "gter0y",
      "authorizer": {
        "numberKey": "123",
        "booleanKey": "true",
        "stringKey": "stringval",
        "principalId": "me",
        "integrationLatency": 0
      },
      "resourcePath": "/helloworld",
      "httpMethod": "POST",
      "extendedRequestId": "J6IuBFugiYcFq7Q=",
      "requestTime": "19/Aug/2023:12:52:54 +0000",
      "path": "/test/helloworld",
      "accountId": "261910724432",
      "protocol": "HTTP/1.1",
      "stage": "test",
      "domainPrefix": "p8d859ec50",
      "requestTimeEpoch": 1692449574298,
      "requestId": "1e8461ad-e3ba-494c-8c19-169ce7a9a174",
      "identity": {
        "cognitoIdentityPoolId": null,
        "accountId": null,
        "cognitoIdentityId": null,
        "caller": null,
        "sourceIp": "72.82.158.196",
        "principalOrgId": null,
        "accessKey": null,
        "cognitoAuthenticationType": null,
        "cognitoAuthenticationProvider": null,
        "userArn": null,
        "userAgent": "curl/7.76.1",
        "user": null
      },
      "domainName": "p8d859ec50.execute-api.us-east-2.amazonaws.com",
      "apiId": "p8d859ec50"
    },
    "body": null,
    "isBase64Encoded": false
  }
}

Create a secret in Secrets Manager

We don’t want to store our parameters when we call the API in the code, so we’ll use Secrets Manager to keep the parameters (headerauth1, password) there.

SECRET_ARN=$(aws secretsmanager create-secret  --name "helloworld/secret" \
  --secret-string "{\"headerauth1\":\"password\"}" --output text --query 'ARN')
echo $SECRET_ARN

Create a DynamoDB table

Let’s create a single DynamoDB table and add two records.

TABLE_ARN=$(aws dynamodb create-table \
  --table-name COMPUTERS \
  --attribute-definitions \
    AttributeName=COMPUTER,AttributeType=S \
    AttributeName=MEMORY_MB,AttributeType=N \
  --key-schema \
    AttributeName=COMPUTER,KeyType=HASH \
    AttributeName=MEMORY_MB,KeyType=RANGE \
  --provisioned-throughput ReadCapacityUnits=1,WriteCapacityUnits=1 \
  --table-class STANDARD \
  --output text \
  --query 'TableDescription.TableArn')
echo $TABLE_ARN

Add some data.

aws dynamodb put-item \
 --table-name COMPUTERS \
 --item \
 '{"COMPUTER": {"S": "ZX SPECTRUM"}, "MEMORY_MB": {"N": "48"}}'

aws dynamodb put-item \
 --table-name COMPUTERS \
 --item \
 '{"COMPUTER": {"S": "COMMODORE"}, "MEMORY_MB": {"N": "64"}}'

Create a VPC and EC2 instance

We have to create a VPC where we’ll provision an Ubuntu EC2 instance and install Node.js on it. If you have a VPC and EC2 that you want to use, you can skip this step. For the sake of simplicity, the VPC will be with one public subnet. Make sure there is an output from each ECHO command. It means everything is OK.

# Create a VPC
VPCID=$(aws ec2 create-vpc --cidr-block 192.168.100.0/24 \
  --region us-east-2 \
  --tag-specifications 'ResourceType="vpc",Tags=[{Key="Name",Value="demo"}]' \
  --output text --query 'Vpc.VpcId')
echo $VPCID

# Create an Internet Gateway
IGWID=$(aws ec2 create-internet-gateway --region us-east-2 \
  --output text --query 'InternetGateway.InternetGatewayId')
echo $IGWID

# Attach the Internet Gatewa to the VPC
aws ec2 attach-internet-gateway \
    --internet-gateway-id $IGWID \
    --vpc-id $VPCID

# Create a subnet
SUBID=$(aws ec2 create-subnet --vpc-id $VPCID \
  --tag-specifications 'ResourceType="subnet",Tags=[{Key="Name",Value="subPublic"}]' \
  --cidr-block 192.168.100.0/24  \
  --output text --query 'Subnet.SubnetId')
echo $SUBID

# Get the main route ID that was created for the VPC
RTID=$(aws ec2 describe-route-tables \
  --filters Name=vpc-id,Values=$VPCID --output text \
  --query 'RouteTables[*].RouteTableId')
echo $RTID

# Use the Internet Gateway for the Internet route 0.0.0.0/0
aws ec2 create-route --route-table-id $RTID --destination-cidr-block 0.0.0.0/0 --gateway-id $IGWID

# Create a security group for the EC2 instance
SGID=$(aws ec2 create-security-group --group-name sgEC2 \
  --description "Security group for the EC2 instance" \
  --vpc-id $VPCID \
  --output text)
echo $SGID

# Get my home IP
MYIP=$(curl -s ifconfig.me)
echo $MYIP

# Allow SSH from my home IP
aws ec2 authorize-security-group-ingress \
    --group-id $SGID \
    --protocol tcp \
    --port 22 \
    --cidr $MYIP/32

# ... and port 3000 from my home IP again (used for Node.js app testing)
aws ec2 authorize-security-group-ingress \
    --group-id $SGID \
    --protocol tcp \
    --port 3000 \
    --cidr $MYIP/32

# Create a key pair
KEYNAME="keydemo"
aws ec2 create-key-pair --key-name $KEYNAME \
  --query 'KeyMaterial' \
  --output text > ~/.ssh/$KEYNAME
chmod 0600 ~/.ssh/$KEYNAME

# Create an Ubuntu instance
IMGID="ami-024e6efaf93d85776"

EC2ID=$(aws ec2 run-instances \
    --image-id $IMGID \
    --count 1 \
    --instance-type t2.small \
    --key-name $KEYNAME \
    --security-group-ids $SGID \
    --subnet-id $SUBID \
    --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ec2demo}]' 'ResourceType=volume,Tags=[{Key=Name,Value=volec2demo}]' \
    --output text \
    --query 'Instances[*].InstanceId' \
    --associate-public-ip-address)

# Get the public IP
PUBLICIP=$(aws ec2 describe-instances --instance-ids $EC2ID \
  --output text \
  --query 'Reservations[*].Instances[*].NetworkInterfaces[*].PrivateIpAddresses[*].Association.PublicIp')
echo $PUBLICIP

Wait for a minute or two and SSH to the instance with:

ssh -i ~/.ssh/$KEYNAME ubuntu@$PUBLICIP

Exit from the SSH session!!!
We’ll need to create a role for the EC2 instance.

Create an IAM role

In order for the EC2 instance to access the DynamoDB and the Secrets Manager, we’ll create an IAM role and attach it to the EC2 instance. We don’t want to deal with IAM users and passwords, accessing the resources with a role is the proper way. Create a trust policy first.

cat << EOF > trust-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "sts:AssumeRole"
            ],
            "Principal": {
                "Service": [
                    "ec2.amazonaws.com"
                ]
            }
        }
    ]
}
EOF

Then create two policies. One for DynamoDB access and the second one for access to Secrets Manager.

cat << EOF > polHelloWorldDynamoDB.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ListAndDescribe",
            "Effect": "Allow",
            "Action": [
                "dynamodb:List*",
                "dynamodb:DescribeReservedCapacity*",
                "dynamodb:DescribeLimits",
                "dynamodb:DescribeTimeToLive"
            ],
            "Resource": "*"
        },
        {
            "Sid": "SpecificTable",
            "Effect": "Allow",
            "Action": [
                "dynamodb:BatchGet*",
                "dynamodb:DescribeStream",
                "dynamodb:DescribeTable",
                "dynamodb:Get*",
                "dynamodb:Query",
                "dynamodb:Scan",
                "dynamodb:BatchWrite*",
                "dynamodb:CreateTable",
                "dynamodb:Delete*",
                "dynamodb:Update*",
                "dynamodb:PutItem"
            ],
            "Resource": "$TABLE_ARN"
        }
    ]
}
EOF

POL_DYNAMODB_ARN=$(aws iam create-policy --policy-name polHelloWorldDynamoDB \
  --policy-document file://polHelloWorldDynamoDB.json \
  --output text --query 'Policy.Arn')
echo $POL_DYNAMODB_ARN

cat << EOF > polHelloWorldSecretsManager.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "secretsmanager:GetResourcePolicy",
                "secretsmanager:GetSecretValue",
                "secretsmanager:DescribeSecret",
                "secretsmanager:ListSecretVersionIds"
            ],
            "Resource": [
                "$SECRET_ARN"
            ]
        },
        {
            "Effect": "Allow",
            "Action": "secretsmanager:ListSecrets",
            "Resource": "*"
        }
    ]
}
EOF

POL_SECRETSMGR_ARN=$(aws iam create-policy --policy-name polHelloWorldSecretsManager \
  --policy-document file://polHelloWorldSecretsManager.json \
  --output text --query 'Policy.Arn')
echo $POL_SECRETSMGR_ARN

Then create a role called rolAccessToResources and attach the AWS policy that gives DynamoDB and Secrets Manager access.

ROLE_ACCESS_ARN=$(aws iam create-role --role-name rolAccessToResources \
  --assume-role-policy-document file://trust-policy.json --output text --query 'Role.Arn')
echo $ROLE_ACCESS_ARN

…attach the policies.

aws iam attach-role-policy --policy-arn $POL_DYNAMODB_ARN --role-name rolAccessToResources
aws iam attach-role-policy --policy-arn $POL_SECRETSMGR_ARN --role-name rolAccessToResources

We need to attach this role to the EC2 instance.

aws iam create-instance-profile --instance-profile-name ipnAccessToResources

…then

aws iam add-role-to-instance-profile --role-name rolAccessToResources --instance-profile-name ipnAccessToResources

Finally…

aws ec2 associate-iam-instance-profile --instance-id $EC2ID --iam-instance-profile Name=ipnAccessToResources

Create JavaScript files for testing

Connect to the instance first.

ssh -i ~/.ssh/$KEYNAME ubuntu@$PUBLICIP

Install Node.js v18. Download and import the Nodesource GPG key

sudo apt-get update
sudo apt-get install -y ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg

Then create the deb repo.

NODE_MAJOR=18
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | \
  sudo tee /etc/apt/sources.list.d/nodesource.list

Run update & install.

sudo apt-get update
sudo apt-get install nodejs -y

Check.

node --version

Create a working folder.

mkdir nodejs 
cd nodejs
npm install aws-sdk

This is a script that will dump the DynamoDB table.

cat << EOF > scan.js
'use strict';
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set the region 
AWS.config.update({region: 'us-east-2'});

// Create the DynamoDB service object
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});

var params = {
  TableName: 'COMPUTERS'
};

// Call DynamoDB to add the item to the table
ddb.scan(params, function(err, data) {
  if (err) {
    console.log("Error", err);
  } else {
    console.log("Success", data.Items);
  }
});
EOF

Check if everything is there.

node scan.js

You’ll get a response like this.

Success [
  { MEMORY_MB: { N: '48' }, COMPUTER: { S: 'ZX SPECTRUM' } },
  { MEMORY_MB: { N: '64' }, COMPUTER: { S: 'COMMODORE' } }
]

This is a script that will check the API Gateway/Lambda. Replace the URL at line 7 with the value of $APIURL from your local computer + add the header.

cat << EOF > fetch.js
'use strict';

(async () => {

        const requestOptions = { method: 'POST', headers: { 'headerauth1':'password' } };
        const url = 'https://p8d859ec50.execute-api.us-east-2.amazonaws.com/test/helloworld?name=there';

        const response = await fetch(url, requestOptions);
        //console.log(response);
        const data = await response.json();
        console.log(data.message);
})();
EOF

Test the API.

node fetch.js

The output should be something like this.

Hello there, it's Sat Aug 19 2023 22:06:22 GMT+0000 (Coordinated Universal Time)

This is a script that will retrieve the secrets from the Secrets Manager.

cat << EOF > secret.js
const AWS = require('aws-sdk');
const client = new AWS.SecretsManager({ region: "us-east-2" });

const getMySecret = async (SecretId) => {
  const s = await client.getSecretValue({ SecretId }).promise();
  return s.SecretString;
};

(async() => {
  const secret_101 = await getMySecret('helloworld/secret');
  console.log('My secret:', secret_101);
})();
EOF

And if you execute you’ll get the output.

node secret.js
My secret: {"headerauth1":"password"}

The final code that combines all is this one. Replace the URL at line 35 with the value of $APIURL.

cat << 'EOF' > index.js
'use strict';

const AWS = require('aws-sdk');
AWS.config.update({region: 'us-east-2'});
const ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
const client = new AWS.SecretsManager({ region: "us-east-2" });
 
var params = {
  TableName: 'COMPUTERS'
};

const getMySecret = async (SecretId) => {
  const s = await client.getSecretValue({ SecretId }).promise();
  return s.SecretString;
};
 
var mysecret = '';
 
params = {
  TableName: 'COMPUTERS'
};
 
ddb.scan(params, function(err, data) {
  if (err) {
    console.log("Error", err);
  } else {
    console.log("Success", data.Items);
  }
});
 
(async () => {
        mysecret = await getMySecret('helloworld/secret');
        var requestOptions = { method: 'POST', headers: JSON.parse(mysecret) };
        const url = 'https://3utbhdiri4.execute-api.us-east-2.amazonaws.com/test/helloworld?name=there';
 
        const response = await fetch(url, requestOptions);
        const data = await response.json();
        console.log(data.message);
})();
EOF

…and the final output.

node index.js
Success [
  { MEMORY_MB: { N: '48' }, COMPUTER: { S: 'ZX SPECTRUM' } },
  { MEMORY_MB: { N: '64' }, COMPUTER: { S: 'COMMODORE' } }
]
Success [
  { MEMORY_MB: { N: '48' }, COMPUTER: { S: 'ZX SPECTRUM' } },
  { MEMORY_MB: { N: '64' }, COMPUTER: { S: 'COMMODORE' } }
]
Hello there, it's Wed Sep 20 2023 20:02:29 GMT+0000 (Coordinated Universal Time)

As you can see, I don’t have any secrets/passwords in the code and it’s still working. That’s because I am getting the secret at line 33 and pass it in line 34.

Dockerizing the app

If we want to run this app as a container under EKS, we have to make sure it runs as a Node.js service and then dockerize it.
The Express version of the same app is this. I am far from an expert for Node.js/Express, so I’ll give it a best shot.
While you are at the nodejs folder, install Express.

npm install express

Then…Make sure you replace line 41 with the $APIURL from your local computer!!!

cat <<'EOF' > server.js
'use strict';
const express = require('express')
const app = express()
const port = 3000

const AWS = require('aws-sdk');
AWS.config.update({region: 'us-east-2'});
const ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
const client = new AWS.SecretsManager({ region: "us-east-2" });

var params = {
  TableName: 'COMPUTERS'
};

const getMySecret = async (SecretId) => {
  const s = await client.getSecretValue({ SecretId }).promise();
  return s.SecretString;
};

var mysecret = '';
var dbitems = '';

params = {
  TableName: 'COMPUTERS'
};

ddb.scan(params, function(err, data) {
  if (err) {
    console.log("Error", err);
  } else {
    console.log("Success", data.Items);
    dbitems = data.Items;
  }
});

var data1 = '';
(async () => {
        mysecret = await getMySecret('helloworld/secret');
        var requestOptions = { method: 'POST', headers: JSON.parse(mysecret) };
        const url = 'https://3utbhdiri4.execute-api.us-east-2.amazonaws.com/test/helloworld?name=there';

        const response = await fetch(url, requestOptions);
        data1 = await response.json();
        console.log(data1);
})();

app.get('/', (req, res) => {
    res.send(JSON.stringify(dbitems) + " " + data1.message);
})

app.listen(port, () => {
  console.log(`Example app listening on port ${port}`)
})
EOF

Run this application with node server.js and the app will listen on port 3000. Open a new session to the terminal and go to the public IP of the instance https://$PUBLICIP:3000 and you’ll see something like this. Or just open up a browser and go to $PUBLICIP:3000. We opened port 3000 earlier on the EC2 instance for this reason. Replace the $PUBLICIP with the public IP of the EC2 Ubuntu instance.

curl $PUBLICIP:3000

CTRL-C out of the app and let’s install Docker.

sudo apt install -y docker.io

Create a Dockerfile.

cat <<EOF > Dockerfile 
FROM node:18
 
# Create app directory
WORKDIR /usr/src/app
 
# Install app dependencies
# A wildcard is used to ensure both package.json AND package-lock.json are copied
# where available (npm@5+)
COPY package*.json ./
 
RUN npm install
# If you are building your code for production
# RUN npm ci --only=production
 
# Bundle app source
COPY . .
 
EXPOSE 3000
CMD [ "node", "server.js" ]
EOF

Create a .dockerignore file.

cat <<EOF > .dockerignore
node_modules
npm-debug.log
EOF

Now, we can create the image and push to Dockerhub. Use your own repo here.

sudo docker build . -t klimenta/awsdemo
sudo docker images
sudo docker run -p 3000:3000 -d klimenta/awsdemo

GO to the same $PUBLIC of the Ubuntu and you’ll see the same output but this time from the Docker container.
Push the image to the Dockerhub.

sudo docker login
sudo docker push kliment/awsdemo

Run as a container in EKS cluster

Now that we have the container and we know it’s working fine, let’s provision an EKS cluster and deploy our Docker image.
Do this from your computer, not the Ubuntu server in AWS.

CLUSTER_NAME="eksAWSDemo"
eksctl create cluster --name $CLUSTER_NAME --region us-east-2 --instance-types t3.medium --nodes 2 --managed --version 1.27

And this is our deployment that creates a public load balancer so we can access the deployment.

cat <<EOF > demo.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: demo
spec:
  replicas: 1
  selector:
    matchLabels:
      run: demo
  template:
    metadata:
      labels:
        run: demo
    spec:
      containers:
      - name: demo
        image: klimenta/awsdemo
        ports:
        - containerPort: 3000
EOF

Deploy the solution.

kubectl apply -f demo.yaml
deployment.apps/demo created
service/loadbalancer created

And you’ll get this if you want to check the container(pod).

kubectl get pods
NAME                    READY   STATUS   RESTARTS      AGE
demo-5655f58d9d-z9pnz   0/1     Error    2 (17s ago)   20s

It errored out.
…and if you want to see why the container crashed, it is very obvious.

kubectl logs demo-5655f58d9d-z9pnz
(Use `node --trace-warnings ...` to show where the warning was created)
Error AccessDeniedException: User: arn:aws:sts::261123456789:assumed-role/eksctl-eksDemoAWS-nodegroup-ng-0c-NodeInstanceRole-UUU04JSKQOJR/i-09df331c5f755bc7e is not authorized to perform: dynamodb:Scan on resource: arn:aws:dynamodb:us-east-2:261910724432:table/COMPUTERS because no identity-based policy allows the dynamodb:Scan action
    at Request.extractError (/usr/src/app/node_modules/aws-sdk/lib/protocol/json.js:80:27)
    at Request.callListeners (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
    at Request.emit (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
    at Request.emit (/usr/src/app/node_modules/aws-sdk/lib/request.js:686:14)
    at Request.transition (/usr/src/app/node_modules/aws-sdk/lib/request.js:22:10)
    at AcceptorStateMachine.runTo (/usr/src/app/node_modules/aws-sdk/lib/state_machine.js:14:12)
    at /usr/src/app/node_modules/aws-sdk/lib/state_machine.js:26:10
    at Request.<anonymous> (/usr/src/app/node_modules/aws-sdk/lib/request.js:38:9)
    at Request.<anonymous> (/usr/src/app/node_modules/aws-sdk/lib/request.js:688:12)
    at Request.callListeners (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:116:18) {
  code: 'AccessDeniedException',
  '[__type]': 'See error.__type for details.',
  '[Message]': 'See error.Message for details.',
  time: 2023-09-21T20:46:01.784Z,
  requestId: 'SDK8R0LSTASAIIBIO3UUU1RC3RVV4KQNSO5AEMVJF66Q9ASUAAJG',
  statusCode: 400,
  retryable: false,
  retryDelay: 32.90504606038562
}
/usr/src/app/node_modules/aws-sdk/lib/protocol/json.js:80
  resp.error = util.error(new Error(), error);
                          ^

AccessDeniedException: User: arn:aws:sts::261987654321:assumed-role/eksctl-eksDemoAWS-nodegroup-ng-0c-NodeInstanceRole-UUU04JSKQOJR/i-09df331c5f755bc7e is not authorized to perform: secretsmanager:GetSecretValue on resource: helloworld/secret because no identity-based policy allows the secretsmanager:GetSecretValue action
    at Request.extractError (/usr/src/app/node_modules/aws-sdk/lib/protocol/json.js:80:27)
    at Request.callListeners (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
    at Request.emit (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
    at Request.emit (/usr/src/app/node_modules/aws-sdk/lib/request.js:686:14)
    at Request.transition (/usr/src/app/node_modules/aws-sdk/lib/request.js:22:10)
    at AcceptorStateMachine.runTo (/usr/src/app/node_modules/aws-sdk/lib/state_machine.js:14:12)
    at /usr/src/app/node_modules/aws-sdk/lib/state_machine.js:26:10
    at Request.<anonymous> (/usr/src/app/node_modules/aws-sdk/lib/request.js:38:9)
    at Request.<anonymous> (/usr/src/app/node_modules/aws-sdk/lib/request.js:688:12)
    at Request.callListeners (/usr/src/app/node_modules/aws-sdk/lib/sequential_executor.js:116:18) {
  code: 'AccessDeniedException',
  '[__type]': 'See error.__type for details.',
  '[Message]': 'See error.Message for details.',
  time: 2023-09-21T20:46:01.800Z,
  requestId: '035cbe99-6826-4f46-b4f8-115cfa90d88b',
  statusCode: 400,
  retryable: false,
  retryDelay: 58.98522014520893
}

We created a role (rolAccessToResources) and two policies (polHelloWorldDynamoDB and polHelloWorldSecretsManager) to allow access to the AWS resources, but we assigned them to the Ubuntu instance. Now, the EKS cluster and the nodes are trying to do the same with their default role (eksctl-eksDemoAWS-nodegroup-ng-0c-NodeInstanceRole-UUU04JSKQOJR), but that role has no access to the DynamoDB table and Secrets Manager, hence the error.
Giving access to the resources to this EKS role is not recommended because all the nodes will have access and we might have other apps running in this cluster.
So we need to give access to the pod, its replicas and all new pods in case the original pods crashes or it’s removed by the autoscaler.

Grant access to AWS resources for a pod

This is a three steps process. We’ll have to:

• Create an IAM OIDC provider
• Configure the role and a service account
• Configure the pods

Let’s do one by one.

Create an IAM OIDC provider

Let’s determine whether we have an existing IAM OIDC provider for our cluster.

aws eks describe-cluster --name eksAWSDemo --query "cluster.identity.oidc.issuer" --output text
https://oidc.eks.us-east-2.amazonaws.com/id/D7FAD6F2BEF430FC1CB673777A9E4FED

When we provisioned the cluster with eksctl, the OIDC was created automatically. We need the ID of the OIDC which is the hex value at the end.

OIDCID=$(aws eks describe-cluster --name $CLUSTER_NAME --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
echo $OIDCID

Check if the IAM OIDC provider is already configured. It shouldn’t if you provisioned a brand new cluster.

aws iam list-open-id-connect-providers | grep $OIDCID 

If for whatever reason, you had an output from the command above, skip this associate-iam-oidc-provider step.

eksctl utils associate-iam-oidc-provider --cluster $CLUSTER_NAME --approve
2023-09-22 12:27:47 [ℹ]  will create IAM Open ID Connect provider for cluster "eksAWSDemo" in "us-east-2"
2023-09-22 12:27:47 [✔]  created IAM Open ID Connect provider for cluster "eksAWSDemo" in "us-east-2"

Configure the role and a service account

We have the role created, that’s rolAccessToResources, that we did initially. This role has two policies that give us access to DynamoDB and Secrets Manager. All we have to do is to create the service account. You can’t use uppercase in the name of the service account.

SA_NAME="sapodshelloworld"
eksctl create iamserviceaccount --name $SA_NAME --namespace default \
  --cluster $CLUSTER_NAME  --attach-role-arn $ROLE_ACCESS_ARN --approve
2023-09-22 12:44:50 [ℹ]  1 iamserviceaccount (default/sapodshelloworld) was included (based on the include/exclude rules)
2023-09-22 12:44:50 [!]  serviceaccounts that exist in Kubernetes will be excluded, use --override-existing-serviceaccounts to override
2023-09-22 12:44:50 [ℹ]  1 task: { create serviceaccount "default/sapodshelloworld" }
2023-09-22 12:44:50 [ℹ]  created serviceaccount "default/sapodshelloworld"

Check the service account.

kubectl get sa
NAME               SECRETS   AGE
default            0         93m
sapodshelloworld   0         27m

Check if the role has the proper policies.

aws iam list-attached-role-policies --role-name rolAccessToResources --query AttachedPolicies[].PolicyArn
[
    "arn:aws:iam::261123456789:policy/polHelloWorldSecretsManager",
    "arn:aws:iam::261987654321:policy/polHelloWorldDynamoDB"
]

In addition, we also have to change the trust policy for our role rolAccessToResources. Currently, it looks like this.

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Service": "ec2.amazonaws.com"
            },
            "Action": "sts:AssumeRole"
        }
    ]
}

This is the proper access policy.

cat<<EOF > trust-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Federated": "arn:aws:iam::$ACCT_NO:oidc-provider/oidc.eks.$AWS_DEFAULT_REGION.amazonaws.com/id/$OIDCID"
            },
            "Action": "sts:AssumeRoleWithWebIdentity",
            "Condition": {
                "StringEquals": {
                    "oidc.eks.$AWS_DEFAULT_REGION.amazonaws.com/id/$OIDCID:aud": "sts.amazonaws.com",
                    "oidc.eks.$AWS_DEFAULT_REGION.amazonaws.com/id/$OIDCID:sub": "system:serviceaccount:default:$SA_NAME"
                }
            }
        }
    ]
}
EOF

Modify the role’s trust policy.

aws iam update-assume-role-policy --role-name rolAccessToResources --policy-document file://trust-policy.json

Check if correct.

aws iam get-role --role-name rolAccessToResources --query Role.AssumeRolePolicyDocument

Check the service account and the role.

kubectl describe serviceaccount $SA_NAME -n default | grep Anno
Annotations:         eks.amazonaws.com/role-arn: arn:aws:iam::261987654321:role/rolAccessToResources

Configure the pods

Actually, we don’t need to configure the pods, we have to configure the deployment that creates the pods.
The only change that you have to make is to specify the service account in the YAML file.
Kill the pods if you have them running.

kubectl delete -f demo.yaml

Create a new deployment demo.yaml file. You only have to add one line (line 16 to specify the service account) + a part with the load balancer.

cat<<EOF > demo-new.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: demo
spec:
  replicas: 1
  selector:
    matchLabels:
      run: demo
  template:
    metadata:
      labels:
        run: demo
    spec:
      serviceAccountName: sapodshelloworld 
      containers:
      - name: demo
        image: klimenta/awsdemo
        ports:
        - containerPort: 3000
---
apiVersion: v1
kind: Service
metadata:
  name: loadbalancer
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: nlb
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
spec:
  ports:
    - port: 80
      targetPort: 3000
      protocol: TCP
  type: LoadBalancer
  selector:
    run: demo
EOF

Deploy the solution again.

kubectl apply -f demo-new.yaml

Get the ULR of the load balancer.

kubectl get svc

…and if you go to that URL after 2-3 mins, you’ll see that everthing works as expected.

curl ab4fdd725967a438ea5a10b4866b720b-b76f9507fafb4eed.elb.us-east-2.amazonaws.com
[{"MEMORY_MB":{"N":"48"},"COMPUTER":{"S":"ZX SPECTRUM"}},{"MEMORY_MB":{"N":"64"},"COMPUTER":{"S":"COMMODORE"}}] Hello there, it's Fri Sep 22 2023 18:49:49 GMT+0000 (Coordinated Universal Time)

When I was doing some proof of concept using EKS, I’ve noticed that there is a limitation on how many pods can a node run and how EKS CNI assigns the subnet IPs. Apparently, a node can run a maximum of XX pods (based on type) which is very low, e.g. a t3.large instance with 2 CPUs and 8GB RAM can run only 35 pods. On top of that, the IPs that are assigned for the pods are used from the existing pool of IPs and if you have a /24 subnet for the EKS cluster, you’ll run out of IPs in no time. Here is an example of what I did and options how to solve this issue.

Option 1 – Calico 3rd party CNI

I’ll provision an EKS cluster in a /24 subnet with 3 public and 3 private subnets.

eksctl create cluster --name eksECIC --region us-east-2 --instance-types t3.large \
    --managed --vpc-cidr 192.168.100.0/24 --node-private-networking --version 1.24 --without-nodegroup

I am using the Ohio region which has 3 availability zones. The public and private subnets (total of 6) have /27 subnet which is 32 IPs. Not all of them are usable of course. Let’s create a node group with 3 nodes, and because t3.large is a Nitro instance, we can set the max number of pods to be 110.

eksctl create nodegroup --cluster eksECIC --name old-nodegroup --nodes 3 \
    --node-type t3.large --node-private-networking --managed --max-pods-per-node 200

These are the available 31 IPs that I have in the private subnets.

Let’s provision a small deployment with a single pod and 10 replicas and a network load balancer.

kubectl apply -f demo.yaml

…and this is the deployment file demo.yaml.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demo
spec:
  replicas: 10
  selector:
    matchLabels:
      run: demo
  template:
    metadata:
      labels:
        run: demo
    spec:
      containers:
      - name: demo
        image: klimenta/serverip
        ports:
        - containerPort: 3000
---
apiVersion: v1
kind: Service
metadata:
  name: loadbalancer
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: nlb
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
spec:
  ports:
    - port: 80
      targetPort: 3000
      protocol: TCP
  type: LoadBalancer
  selector:
    run: demo

If you look at the available address now, there are 7. So from 31 to 7 with just 10 pods. Check this guy’s blog to see what’s going on.

Let’s scale up the cluster to 100 pods.

kubectl scale --replicas=100 deployment/demo

After a minute or two, check the running pods.

kubectl get pods --field-selector=status.phase=Running | wc -l
69

The console will show 0 available subnets.

If you check the pods with kubectl get pods, you’ll see the remaining containers have a ContainerCreating status.
And if you check one of them, you’ll see this message.

kubectl describe pod demo-698f6fc958-z8smb
....
d9b65ccfeccb7e9e7433ec61fef78faf83d1bae": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container
  Warning  FailedCreatePodSandBox  2m29s (x17 over 5m59s)  kubelet            (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "edfa987c3eddf415068691f914d740fa9ca9ca6ffdfb8db44d1340d913ece0b0": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container

Get the load balancer service.

kubectl get svc
NAME           TYPE           CLUSTER-IP      EXTERNAL-IP                                                                     PORT(S)        AGE
kubernetes     ClusterIP      10.100.0.1      <none>                                                                          443/TCP        37m
loadbalancer   LoadBalancer   10.100.25.241   a105789cd3a264ac596c367fce463640-80d2b88acdc17771.elb.us-east-2.amazonaws.com   80:32396/TCP   17m

If you go to a105789cd3a264…amazonaws.com URL, you’ll see the load balancer hitting different pods. Wait for 2-3 minutes if you see that the page can’t be opened. It takes time for DNS to propagate.

Let’s delete the deployment and the old node group.

kubectl delete -f demo.yaml
eksctl delete nodegroup --name old-nodegroup --cluster eksECIC

At this point, we still have the EKS cluster, the etcd database, networking etc, we just don’t have the nodes. Now it’s time to remove the EKS CNI, install Calico and add nodes that will use Calico.
Installation is very simple. Remove the AWS CNI.

kubectl delete daemonset -n kube-system aws-node

Check the pods in all namespaces.

kubectl get pods --all-namespaces
NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE
kube-system   coredns-5c5677bc78-2k88b   0/1     Pending   0          4m
kube-system   coredns-5c5677bc78-2zz2n   0/1     Pending   0          4m

You will see coredns in pending state, it can’t be deployed anywhere.
Deploy Calico.

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml

…keep going…

kubectl create -f - <<EOF
kind: Installation
apiVersion: operator.tigera.io/v1
metadata:
  name: default
spec:
  kubernetesProvider: EKS
  cni:
    type: Calico
  calicoNetwork:
    bgp: Disabled
EOF

Deploy the new node group.

eksctl create nodegroup --cluster eksECIC --name new-nodegroup --nodes 3 \
    --node-type t3.large --node-private-networking --managed --max-pods-per-node 200

Let’s deploy the same demo deployment now.

kubectl apply -f demo.yaml

…scale it up to 601 pods.

kubectl scale --replicas=601 deployment/demo

Wait 2-3 mins and check the running ones.

kubectl get pods --field-selector=status.phase=Running | wc -l

There will be 585 pods running, 195 per node. Much, much better. This time the remaining containers will be in Pending state.

kubectl describe pod demo-698f6fc958-smlbq
....
Events:
  Type     Reason            Age                  From               Message
  ----     ------            ----                 ----               -------
  Warning  FailedScheduling  3m27s (x3 over 13m)  default-scheduler  0/3 nodes are available: 3 Too many pods. preemption: 0/3 nodes are available: 3 No preemption victims found for incoming pod.

Option 2 – CNI Custom networking and prefix assignment mode

In this case, we’ll keep the AWS CNI network plugin, but we’ll use what is called CNI Custom networking, meaning we’ll use another CIDR attached to the VPC where pods will run. NOTE: The nodes will still run in the 192.168/24 subnets, it’s just the pods that will run in a much bigger IP space. In addition, this CNI Custom networking won’t solve the density of the pods that we can run on the nodes. We’ll need something called prefix assignment mode that will allow us to run much more pods on a node than usual. See this and this links for more information. For more info regarding prefix assignment mode check out this link. The solution described below is much more complicated than if you just use Calico.
Let’s provision a cluster with the same VPC CIDR and no nodegroups.

eksctl create cluster --name eksECIC --region us-east-2 --instance-types t3.large \
    --managed --vpc-cidr 192.168.100.0/24 --node-private-networking --version 1.24 --without-nodegroup

Add a new CIDR and associate it with the VPC. I am using the console but you can use aws ec2 associate-vpc-cidr-block command.Go to the VPC where the EKS cluster resides. From the Actions button, select Edit CIDRs and then click the Add new IPv4 CIDR button.

I’ll add 100.64.0.0/16 CIDR and then I’ll create 3 /19 subnets.

These 3 subnets will be where the pods will run. I also want them to initiate connections to Internet, so I’ll modify the route table and add the Internet Gateway. The three new subnets 100.64/19 that we created have the same route table.

Just add the route to 0.0.0.0/0 to go over Internet Gateway.

Let’s configure the custom networking on the cluster.

kubectl set env daemonset aws-node -n kube-system AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true

Get the EKS cluster security group.

sec_grp=$(aws eks describe-cluster --name eksECIC --query cluster.resourcesVpcConfig.clusterSecurityGroupId --output text)
echo $sec_grp

For each of the subnets create a custom resource. Define the three 100.64 subnet-ids as variables.

export sub1=subnet-0f3ec5d75caf1b981
export sub2=subnet-0fc39edc5a1ce91c0
export sub3=subnet-0bd6bdfd5543aa086

cat >us-east-2a.yaml <<EOF
apiVersion: crd.k8s.amazonaws.com/v1alpha1
kind: ENIConfig
metadata: 
  name: us-east-2a
spec: 
  securityGroups: 
    - $sec_grp
  subnet: $sub1
EOF

2nd subnet.

cat >us-east-2b.yaml <<EOF
apiVersion: crd.k8s.amazonaws.com/v1alpha1
kind: ENIConfig
metadata: 
  name: us-east-2b
spec: 
  securityGroups: 
    - $sec_grp
  subnet: $sub2
EOF

3rd subnet.

cat >us-east-2c.yaml <<EOF
apiVersion: crd.k8s.amazonaws.com/v1alpha1
kind: ENIConfig
metadata: 
  name: us-east-2c
spec: 
  securityGroups: 
    - $sec_grp
  subnet: $sub3
EOF

NOTE:Make sure you name the metadata in line 5 as I did based on your region and AZ.
Deploy the custom resources.

kubectl apply -f us-east-2a.yaml
kubectl apply -f us-east-2b.yaml
kubectl apply -f us-east-2c.yaml

Confirm it looks good.

kubectl get ENIConfigs
NAME         AGE
us-east-2a   16s
us-east-2b   12s
us-east-2c   8s

Update your aws-node DaemonSet to automatically apply the ENIConfig for an Availability Zone to any new Amazon EC2 nodes created in your cluster.

kubectl set env daemonset aws-node -n kube-system ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone

Enable prefix assignment mode. This will allow much more pods per node.

kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true

Another change.

kubectl set env ds aws-node -n kube-system WARM_PREFIX_TARGET=1

With the default setting, WARM_PREFIX_TARGET will allocate one additional complete (/28) prefix even if the existing prefix is used by only one pod. If the ENI does not have enough space to assign a prefix, a new ENI is generated.
Provision the node group.

eksctl create nodegroup --cluster eksECIC --name my-nodegroup --nodes 3 \
    --node-type t3.large --node-private-networking --managed --max-pods-per-node 200

Check the nodes, they are running in the 192.168/24 original VPC CIDR. See the IP in the names of the nodes.

kubectl get nodes
NAME                                            STATUS   ROLES    AGE     VERSION
ip-192-168-100-122.us-east-2.compute.internal   Ready    <none>   7m39s   v1.24.13-eks-0a21954
ip-192-168-100-155.us-east-2.compute.internal   Ready    <none>   7m39s   v1.24.13-eks-0a21954
ip-192-168-100-167.us-east-2.compute.internal   Ready    <none>   7m37s   v1.24.13-eks-0a21954

Check the pods.

kubectl get pods -A -o wide
NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE    IP                NODE                                            NOMINATED NODE   READINESS GATES
kube-system   aws-node-txwrd             1/1     Running   0          9m4s   192.168.100.167   ip-192-168-100-167.us-east-2.compute.internal   <none>           <none>
kube-system   aws-node-vdmv7             1/1     Running   0          9m6s   192.168.100.155   ip-192-168-100-155.us-east-2.compute.internal   <none>           <none>
kube-system   aws-node-xss7c             1/1     Running   0          9m6s   192.168.100.122   ip-192-168-100-122.us-east-2.compute.internal   <none>           <none>
kube-system   coredns-5c5677bc78-2lkcx   1/1     Running   0          80m    100.64.76.225     ip-192-168-100-122.us-east-2.compute.internal   <none>           <none>
kube-system   coredns-5c5677bc78-mj6wr   1/1     Running   0          80m    100.64.76.224     ip-192-168-100-122.us-east-2.compute.internal   <none>           <none>
kube-system   kube-proxy-lt7v4           1/1     Running   0          9m6s   192.168.100.122   ip-192-168-100-122.us-east-2.compute.internal   <none>           <none>
kube-system   kube-proxy-m8pf6           1/1     Running   0          9m4s   192.168.100.167   ip-192-168-100-167.us-east-2.compute.internal   <none>           <none>
kube-system   kube-proxy-nlvw7           1/1     Running   0          9m6s   192.168.100.155   ip-192-168-100-155.us-east-2.compute.internal   <none>           <none>

Look at coredns pods. They are running in the new CIDR.
Deploy the demo.

kubectl apply -f demo.yaml

Get the load balancer service.

kubectl get svc
NAME           TYPE           CLUSTER-IP       EXTERNAL-IP                                                                     PORT(S)        AGE
kubernetes     ClusterIP      10.100.0.1       <none>                                                                          443/TCP        83m
loadbalancer   LoadBalancer   10.100.206.187   a7ad3ecf1d52e4fc58dbbdc550237009-ac4778e11af2c0ad.elb.us-east-2.amazonaws.com   80:30230/TCP   45s

If you go to a7ad3ecf1d…amazonaws.com URL, you’ll see the load balancer hitting different pods. Wait for 2-3 minutes if you see that the page can’t be opened. It takes time for DNS to propagate.

The pods are running in the 100.64 subnet.
Scale up to 601 pods.

kubectl scale --replicas=601 deployment/demo

Wait 2-3 mins and check the running ones.

kubectl get pods --field-selector=status.phase=Running | wc -l

I got 593 total pods. Slightly better than 585 with Calico. The ones that are in pending state show up an error that there are no more resources.

Events:
  Type     Reason            Age    From               Message
  ----     ------            ----   ----               -------
  Warning  FailedScheduling  3m49s  default-scheduler  0/3 nodes are available: 3 Too many pods. preemption: 0/3 nodes are available: 3 No preemption victims found for incoming pod.

In this post I’ll explain several things related to EKS monitoring and logging.

– How to create an EKS cluster
– Enable the EKS control plane logs and send them to CloudWatch
– Send these logs from CloudWatch to OpenSearch cluster
– Install Container Insights and FluentBit and send logs to CloudWatch
– Install Prometheus and Grafana to monitor and visualize EKS cluster metrics
– WordPress and CloudWatch log group

In order to do that, we’ll need the following CLI tools.

– eksctl
– kubectl
– helm
– aws cli

You can easily find how to install and configure these tools for various OSes.

Create the EKS cluster

I’ll create a managed cluster called eksWordPress in us-east-2 region with two t3.medium nodes.

eksctl create cluster --name eksWordPress --region us-east-2 --instance-types t3.medium --nodes 2 --managed --version 1.22

If you get an error that the last supported version is 1.21, update eksctl tool. The cluster creation took about 20 mins for me.

Send EKS logs to CloudWatch

Once the cluster was created and up and running, enabling the EKS control plane logs is easy. There are 5 log types (API server, Audit, Authenticator, Controller Manager and Scheduler logs). The logged info is a lot, so it’s up to you what you want to log. There is no way to choose what to log, e.g. info, warning, errors. Everything is logged. To enable logging from the console, go to the EKS cluster, select it, click the Logging tab and then click on the Manage logging button.

If you want to do it from a CLI, type this command and specify what types of logs you want sent to CloudWatch. Specify the region and the cluster name.

aws eks update-cluster-config \
    --region <REGION> \
    --name <CLUSTER_NAME> \
    --logging '{"clusterLogging":[{"types":["api","audit","authenticator","controllerManager","scheduler"],"enabled":true}]}'

In addition, because we provisioned the cluster with the eksctl tool, we can also enable the logs using eksctl.

eksctl utils update-cluster-logging --enable-types all --cluster <CLUSTER_NAME> --approve

Or for certain types, use…

eksctl utils update-cluster-logging --enable-types <LOG_TYPE> --cluster <CLUSTER_NAME> --approve

You can use the same types as in the AWS CLI command above (api, audit, scheduler…).
If you go to CloudWatch and then the Log groups, you’ll see the log group with the name of the EKS cluster (/aws/eks/eksWordPress/cluster).
Make sure you change the retention from Never to some value. You probably don’t want to keep these logs indefinitely. If you look at the logs streams, you’ll see that there is a lot of info there. Most of the stuff is useless.

Send the logs to OpenSearch cluster

Let’s create a public OpenSearch cluster with anonymous access that only you can access. Replace the IP at the end of the statement with your IP address. It takes less than 10 mins for the OpenSearch cluster to be provisioned.

aws opensearch create-domain --domain-name oswordpress --engine-version OpenSearch_1.3 \
    --auto-tune-options DesiredState="ENABLED" --cluster-config InstanceType=t3.small.search,InstanceCount=2 \
    --ebs-options EBSEnabled=true,VolumeType=gp3,VolumeSize=10,Iops=3000 \
    --access-policies '{"Version": "2012-10-17", "Statement": [{"Action": "es:*", "Principal":"*","Effect": "Allow", "Condition": {"IpAddress":{"aws:SourceIp":["2.18.2.19/32"]}}}]}'

Once the cluster is provisioned you can get the public URL with:

echo `aws es describe-elasticsearch-domain --domain-name oswordpress --output text --query "DomainStatus.Endpoint"`\\_dashboards

You can also get the dashboard URL from the cluster settings as well.

Create the following IAM policy. Save the policy below as policy-file.json but change the account_no in line 9 to match your AWS account.

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "es:*"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:es:us-east-2:<account_no>:domain/oswordpress/*"
        }
    ]
}

And then create the IAM policy.

aws iam create-policy --policy-name polOpenSearch --policy-document file://policy-file.json

Create the role. Save the policy below as policy-trust.json.

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

And then create the role.

aws iam create-role --role-name rolOpenSearch --assume-role-policy file://policy-trust.json

Finally, attach the policy to the role.

aws iam put-role-policy --role-name rolOpenSearch --policy-name polOpenSearch --policy-document file://policy-file.json 

Go to CloudWatch, select the Log group, from the Actions button select Subscription filters and then Create Amazon OpenSearch service subscription filter.

Choose the OpenSearch cluster that we created, the log format is JSON, subscription filter pattern is ” “ for all events and type all subscription filter name or whatever you want to name this pattern. Click Start streaming button.
Now, go to OpenSearch dashboard and from the hamburger menu in the upper left corner, click on Visualize. In the middle of the screen, you’ll be prompted to create an index pattern. Type cwl* and click Next step.

Select @timestamp from the drop down menu and click on Create index pattern. You’ll see the fields and the index.

Click on the Discover from the hamburger menu and you’ll see your data there. You can use queries to search your data, but that’s out of the scope of this post.

Container Insights and FluentBit

Container Insights is a CloudWatch agent that we’ll install on the EKS cluster. It will collect all kinds of metrics and then FluentBit as a log forwarder will ship those logs to CloudWatch. See this link for more info.
Let’s do the prerequisites work first.
Get the nodes. Type the first line only, the rest is my output.

kubectl get nodes                                                                                                                                                               
NAME                                           STATUS   ROLES    AGE     VERSION
ip-192-168-56-250.us-east-2.compute.internal   Ready    <none>   3h39m   v1.22.12-eks-ba74326
ip-192-168-66-251.us-east-2.compute.internal   Ready    <none>   3h39m   v1.22.12-eks-ba74326

Get the instance ID from any of the instances, it doesn’t matter. Replace the name accordingly after Values=.

aws ec2 describe-instances --filters 'Name=private-dns-name,Values=ip-192-168-56-250.us-east-2.compute.internal' \
    --output text --query 'Reservations[*].Instances[*].InstanceId'
i-05adaa0a823a8549a

Once you have the instance ID of any node, get the Arn of the IAM role that’s the attached to that node.

aws ec2 describe-instances --region us-east-2 --instance-ids i-05adaa0a823a8549a --query 'Reservations[*].Instances[*].IamInstanceProfile.Id'
[
    [
        "AIPAXFRN6SYD75I5N4BUN"
    ]
]

Get the role name. Replace the ID (AIPAX…) with your value above.

aws iam list-instance-profiles --query 'InstanceProfiles[?InstanceProfileId==`AIPAXFRN6SYD75I5N4BUN`].Roles[*].RoleName' 
[
    [
        "eksctl-eksWordPress-nodegroup-ng-NodeInstanceRole-LWJZESQ468Y2"
    ]
]

We want to add a policy that allows nodes to write to CloudWatch logs group. Replace the –role-name with the value that you got above.

aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy --role-name eksctl-eksWordPress-nodegroup-ng-NodeInstanceRole-LWJZESQ468Y2

oidc_id=$(aws eks describe-cluster --name eksWordPress --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)

Type this line and see if there is any output. If no output, execute the command after. If yes-output, then do nothing.

aws iam list-open-id-connect-providers | grep $oidc_id

If no output, type this.

eksctl utils associate-iam-oidc-provider --cluster eksWordPress --approve

Then install CloudWatch Container Insights and FluentBit. Change the CLUSTER_NAME and REGION in lines 1 and 2.

ClusterName=<CLUSTER_NAME>
RegionName=<REGION>
FluentBitHttpPort='2020'
FluentBitReadFromHead='Off'
[[ ${FluentBitReadFromHead} = 'On' ]] && FluentBitReadFromTail='Off'|| FluentBitReadFromTail='On'
[[ -z ${FluentBitHttpPort} ]] && FluentBitHttpServer='Off' || FluentBitHttpServer='On'
curl https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml | sed 's/{{cluster_name}}/'${ClusterName}'/;s/{{region_name}}/'${RegionName}'/;s/{{http_server_toggle}}/"'${FluentBitHttpServer}'"/;s/{{http_server_port}}/"'${FluentBitHttpPort}'"/;s/{{read_from_head}}/"'${FluentBitReadFromHead}'"/;s/{{read_from_tail}}/"'${FluentBitReadFromTail}'"/' | kubectl apply -f - 

Download a config map and edit it the file called cwagent-configmap.yaml.

curl -O https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-configmap.yaml

In line 11, change the variable so it points to your cluster. In my case it looks like this.

"cluster_name": "{{eksWordPress}}",

Save the changes and apply the config.

kubectl apply -f cwagent-configmap.yaml

Then deploy it as a DaemonSet.

kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml

Verify that it’s running. Type the first line only. The rest is my output.

kubectl get pods -n amazon-cloudwatch
NAME                     READY   STATUS    RESTARTS   AGE
cloudwatch-agent-dfkzv   1/1     Running   0          18m
cloudwatch-agent-nfnf7   1/1     Running   0          18m
fluent-bit-8vd2n         1/1     Running   0          18m
fluent-bit-tvtpv         1/1     Running   0          18m

Check the logs.

kubectl logs <POD_NAME>  -n amazon-cloudwatch

Or in my case…

kubectl logs cloudwatch-agent-dfkzv -n amazon-cloudwatch
[2022/10/03 13:28:56] [ info] [output:cloudwatch_logs:cloudwatch_logs.2] Created log stream ip-192-168-73-121.us-east-2.compute.internal.host.messages
[2022/10/03 13:29:06] [ info] [output:cloudwatch_logs:cloudwatch_logs.0] Creating log stream ip-192-168-73-121.us-east-2.compute.internal-application.var.log.containers.cloudwatch-agent-7fvxd_amazon-cloudwatch_cloudwatch-agent-5409bed9d4733a51602e4bc0cccde5e5580eb3e9282cd5cc4c1a4f2d2e28e8ea.log in log group /aws/containerinsights/eksWordPress/application
[2022/10/03 13:29:06] [ info] [output:cloudwatch_logs:cloudwatch_logs.0] Created log stream ip-192-168-73-121.us-east-2.compute.internal-application.var.log.containers.cloudwatch-agent-7fvxd_amazon-cloudwatch_cloudwatch-agent-5409bed9d4733a51602e4bc0cccde5e5580eb3e9282cd5cc4c1a4f2d2e28e8ea.log

If you go to CloudWatch now you’ll see 4 new log groups.

Prometheus and Grafana

The Kubernetes API can also be monitored using Prometheus. We’ll install it using helm.
Create a namespace first.

kubectl create namespace prometheus

Add the Prometheus repo.

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

Deploy Prometheus.

helm upgrade -i prometheus prometheus-community/prometheus --namespace prometheus --set alertmanager.persistentVolume.storageClass="gp2",server.persistentVolume.storageClass="gp2"

Check if everything is OK.

kubectl get pods -n prometheus
NAME                                            READY   STATUS    RESTARTS   AGE
prometheus-alertmanager-5c57cc6945-v9lcb        2/2     Running   0          4m55s
prometheus-kube-state-metrics-77ddf69b4-jsgrp   1/1     Running   0          4m55s
prometheus-node-exporter-68dk9                  1/1     Running   0          4m55s
prometheus-node-exporter-m98xk                  1/1     Running   0          4m55s
prometheus-pushgateway-ff89cc976-4sfhl          1/1     Running   0          4m55s
prometheus-server-6c99667b9b-mpw97              2/2     Running   0          4m55s

Type this command and open up a browser and go to localhost:9090.

kubectl --namespace=prometheus port-forward deploy/prometheus-server 9090

You can CTRL-C out of the command prompt once you verify it’s OK.
Let’s add the Grafana repo first.

helm repo add grafana https://grafana.github.io/helm-charts

Copy, paste and save this as grafana.yaml.

datasources:
  datasources.yaml:
    apiVersion: 1
    datasources:
    - name: Prometheus
      type: prometheus
      url: http://prometheus-server.prometheus.svc.cluster.local
      access: proxy
      isDefault: true

Create a namespace for grafana.

kubectl create namespace grafana

Deploy using helm. Look at the adminPassword parameter. Change it to something else. In my case it’s admin123!.

helm install grafana grafana/grafana --namespace grafana --set persistence.storageClassName="gp2" --set persistence.enabled=true --set adminPassword='admin123!' --values grafana.yaml --set service.type=LoadBalancer

Check if everything is OK.

kubectl get all -n grafana

Get the URL of the classic LB that was just created.

export ELB=$(kubectl get svc -n grafana grafana -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
echo "http://$ELB"

Go to that URL and you’ll see the Grafana URL.
You should see the Grafana landing page. Log as admin and the password you specified when pods were deployed.
In Grafana create a new dashboard and Import the dashboard with ID 3119. Choose Prometheus as source.

For pods monitoring use the same method but this time specify 6417 as a dashboard ID. This is how it looks like under my account.

WordPress

Let’s deploy WordPress.

kubectl create namespace wordpress

Add the Bitnami helm chart.

helm repo add bitnami https://charts.bitnami.com/bitnami

Deploy WordPress in its own namespace

helm -n wordpress install understood-zebu bitnami/wordpress

Wait for 3-4 mins and do this command. This is your ELB, get the URL, something like *zdasdfa*.elb.amazonaws.com

kubectl get svc --namespace wordpress -w understood-zebu-wordpress

The username is user and get the password with:

echo Password: $(kubectl get secret --namespace wordpress understood-zebu-wordpress -o jsonpath="{.data.wordpress-password}" | base64 -d)

If you go to the ELB URL, you’ll hit WordPress main page, if you want to login, add /wp-login.php as suffix to the above URL.
Go to CloudWatch and check the /aws/containerinsights/eksWordPress/application log group. You’ll see a bunch of references for WordPress. You can ship those to OpenSearch if you want and alert on errors or whatever you want to do.

Delete EKS and OpenSearch cluster

Detach the policy, delete the daemonset and delete the EKS cluster.

aws iam detach-role-policy --policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy --role-name eksctl-eksWordPress-nodegroup-ng-NodeInstanceRole-LWJZESQ468Y2
ClusterName=<CLUSTER_NAME>
RegionName=<REGION>
FluentBitHttpPort='2020'
FluentBitReadFromHead='Off'
[[ ${FluentBitReadFromHead} = 'On' ]] && FluentBitReadFromTail='Off'|| FluentBitReadFromTail='On'
[[ -z ${FluentBitHttpPort} ]] && FluentBitHttpServer='Off' || FluentBitHttpServer='On'
curl https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluent-bit-quickstart.yaml | sed 's/{{cluster_name}}/'${ClusterName}'/;s/{{region_name}}/'${LogRegion}'/;s/{{http_server_toggle}}/"'${FluentBitHttpServer}'"/;s/{{http_server_port}}/"'${FluentBitHttpPort}'"/;s/{{read_from_head}}/"'${FluentBitReadFromHead}'"/;s/{{read_from_tail}}/"'${FluentBitReadFromTail}'"/' | kubectl delete -f -
eksctl delete cluster --name eksWordPress --region=us-east-2

Delete policy.

aws iam delete-policy --policy-arn arn:aws:iam::492943873543:policy/polOpenSearch

Delete CloudWatch log groups.

EKS_CLUSTER=eksWordPress
aws logs delete-log-group --log-group-name "/aws/containerinsights/$EKS_CLUSTER/application"
aws logs delete-log-group --log-group-name "/aws/containerinsights/$EKS_CLUSTER/dataplane"
aws logs delete-log-group --log-group-name "/aws/containerinsights/$EKS_CLUSTER/host"
aws logs delete-log-group --log-group-name "/aws/containerinsights/$EKS_CLUSTER/performance"
aws logs delete-log-group --log-group-name "/aws/eks/$EKS_CLUSTER/cluster"

Delete the OpenSearch cluster.

aws opensearch delete-domain --domain-name oswordpress

Here is the deal. Our vendor wants to access our Aurora database. We can’t do VPC peering because our CIDRs overlap. On top of that, we don’t want peering with 3rd party vendors and we don’t want to host our data in the vendor’s account. So, we have to use PrivateLink.
In this post I’ll guide you through creating an RDS database in one account and accessing it from another account over PrivateLink. In order to do this, we’ll need:

– Two AWS accounts (Provider and Customer). We are the provider, the vendor is the customer.
– One VPC with 2 public/private subnets in each account. The vendor’s (customer) VPC config doesn’t really matter.
– One EC2 instance in each account (public subnet) for testing access

We will create an endpoint service in the Provider VPC and an endpoint in the Customer VPC. The endpoint service requires a network load balancer as well. Here is a problem. When we create a NLB, the target of the NLB are IP addresses. The RDS database has DNS entries as endpoints. Technically, we can resolve the DNS URLs in IPs and use them as targets. But these IPs can change. So we have to put something in between the RDS and NLB. That’s the job of the RDS proxy. The DNS URLs of an RDS Proxy always resolve to the same IPs. Another problem is that RDS proxy can be deployed for MySQL and PostgreSQL databases only. So keep this in mind.

VPC

I’ll create two VPCs in each account. The first VPC will have a 192.168.100.0/22 CIDR and the second VPC will have 192.168.200.0/22 CIDR with 4 x /24 private and public subnets. Here is the breakdown of the subnets. It’s a standard build with two NAT gateways and one Internet gateway.
The first VPC is called vpc-Provider.

The second VPC is called vpc-Consumer.

Security groups

Create two security groups under vpc-Provider VPC, one called sgEC2 and the other called sgRDS.
For the sgEC2 group, allow SSH from some IP, e.g. your IP from work/home. The EC2 instance that we’ll create will be in a public subnet.
For the sgRDS group, allow port 3306 (MySQL) from the actual subnet ID for sgEC2 group and another rule to allow access to 3306 from itself. This is how the rules for sgRDS look like.

Now, do the same for the vpc-Consumer account, but create only the sgEC2 group.

RDS

For the RDS, use pretty much default settings. In my case, I used:

– Standard create
– Amazon Aurora
– Amazon Aurora MySQL-Compatible Edition
– Capacity type: Provisioned
– Aurora (MySQL 5.7) 2.07.2
– Templates: Dev/Test
– Name the cluster and enter your admin credentials (remember these)
– Burstable db.t3.small
– Don’t create an Aurora Replica
– Choose the VPC that we created
– Public access: No
– Security group: sgRDS
– Availability Zone: No preference

Then click to Create database.

EC2

While the database is creating, go to EC2 and create an instance that we’ll use for testing. It can be any Linux and any size. I’ve used t2.micro Amazon Linux with telnet/MySQL client installed (sudo yum install telnet mysql) in a public subnet. I’ve named the instance ec2Provider. Create another instance in the second account with the same specs as ec2Provider and name it ec2Customer.

Test 1

Go to the RDS and get the DNS record for the Aurora cluster. It’s at the bottom.

Log to the instance in the same account (ec2Provider) and try to telnet to that port. You should be able to. CTRL-C to get out of there and log to MySQL using the client.

mysql -h database-1.cluster-cdbzhqzxu6yp.us-east-2.rds.amazonaws.com -u admin -p

Replace the hostname with yours. This the output of my connection.

I’ll create a database while I am there.

create database test_db;

AWS Secrets Manager

In order to deploy an RDS proxy, we need a secret (user/password combo) stored in the AWS Secrets Manager. Use the same username and password that you used for the RDS database, which is admin and whatever password you used. Make sure you select the database that is listed at the bottom of the first screen.

Name the secret and proceed with the defaults.

RDS Proxy

Go to the RDS menu and select and click on the database cluster. Scroll all the way down to the RDS Proxy section. Click on Create proxy.

– Name the proxy (Proxy identifier)
– Engine – MySQL
– Choose the database
– Uncheck the Add reader point.
– Choose the secret that you created before and specify two private subnets
– Leave AWS to create the IAM role.
– Under Additional connectivity configuration make sure you assign the sgRDS security group and remove the default.

Click on Create proxy.

Test 2

Let’s try to access our database through the proxy now. In the main menu for the RDS, click on the Proxies on the left.

Click on the proxy and scroll down to see the proxy endpoint. Copy that value.

Go to the EC2 instance in the vpc-Provider, which is in the same account as the RDS and try to access the database over proxy.

Great job if you made it so far!

Network Load Balancer (NLB)

OK, so if everything works, we need to create an NLB so we can create the endpoint service. The NLB that we’ll create requires the IP targets of the RDS proxy. You got the DNS for the RDS proxy, the one that you just tested right above. Do an nslookup on it and get the IPs.

In my case the proxy RDS returns 192.168.103.210 and 192.168.102.160, which tells me they are in the private availability zones (AZs). Good.
Create an NLB, name it and make sure it’s internal.

Choose the VPC and the two internal subnets.

When you reach the Listeners and routing, change the port to 3306 and click on Create target group.

Choose IP addresses, name the Target group name and change the port to 3306. The correct VPC should be selected.

Leave the health checks, they are automatically populated for the right port and click Next.
On the next screen below, enter the IPs that you got from the nslookup for the RDS proxy. In my case it looks like this. Make sure the port is 3306 and click on Include as pending below and finally click on Create target group.

Go back to the original browser tab where you create the NLB and click the refresh button. You’ll be able to select the target group now.

Click on Create load balancer at the bottom. It takes a couple of minutes to provision the NLB. Once done, click on the Listeners tab and then the target group.

It will take another couple of minutes, but you should see that the targets are unhealthy. Why is that? Because our RDS proxy can get requests from the EC2 instances only. Thе NLB can’t access the RDS proxy. Per AWS, we have to allow the CIDRs of the NLB to access the RDS proxy. That’s easy. We created an internal NLB with 192.168.102.0/24 and 192.168.103.0/24 subnets, so we have to modify the sgRDS security group. Do the same. This is how my security group sgRDS looks like now.

And if you check the target group, you should see both RDS proxy instance as healthy.

Test 3

Let’s check the access on the NLB. Get the DNS name. In my case it’s rds-nlb-40d1b9bfc867b2fb.elb.us-east-2.amazonaws.com.
Go back to the EC2 instance in the same account and try to access the database.

Yep, we are doing great. Let’s recap. We can’t access the RDS directly from another account because – well, it’s another account that we don’t trust and our RDS DB is in a private subnet. So, we have to create an endpoint service that requires a network load balancer. But, the NLB requires IP targets and our RDS endpoints are DNS names. Which means that the DNS names can change its IPs, so we used an RDS proxy DNS names converted to IPs which AWS claims that they won’t change.

Endpoint service

The Provider (we) creates an Endpoint service, the Customer (the vendor) creates the Endpoint. Simple as that. It doesn’t have to be a vendor. Maybe the proper term is the Consumer. So, we (the provider) expose some service as Endpoint Service and someone else tries to access it.
An Endpoint service requires a network load balancer. We did that. So let’s create the endpoint service. It’s under the VPC menu.

Click on Endpoint Services and then Create endpoint service.
Name the endpoint service and check the network load balancer. Make sure that the Acceptance required is checked.

Once created (in a second), get the service name. Copy that URL highlighted.

Select the endpoint service and from the Actions menu, choose Allow principals. Add * for ARN which means everyone. Now this is really bad. For the sake of simplicity, I’ve added * (everyone) but you want to add an ARN role from the other account.

Endpoint

The endpoint is the Consumer, the vendor or some other VPC. Go to that account. We are no longer working on the original account, log to the second account and go to the Endpoint under VPC menu. Click on Create endpoint. Enter the name for endpoint, click on Other endpoint services and paste the endpoint that you got in the previous step. Click on Verify service and you should see that it’s OK.

Scroll down and choose the VPC where you want to create the endpoint. In our case, we want to use the EC2 instance that’s in the public subnet. Add the two public subnets.

And for the security group, add the EC2 security group.

NOTE: If you go to the Interfaces section under EC2, you’ll see three NICs. One is for the EC2 instance that we’ve created, the other two are the endpoints.

Anyway, once you create the endpoint, you’ll see this.

Go to the other account (Provider) and under the endpoint services (VPC menu, Endpoint connections tab), you’ll see a request for the connection. That’s what Accepance required means when we created the Endpoint Service. Accept the connection. It takes about 30 seconds to complete the acceptance.

Test 4

Go to the Consumer account and click on Endpoint and then select the endpoint. Under the Details tab you’ll see three DNS entries.

Copy one of them and log to the EC2 instance in that account and try to access the RDS.

mysql -h vpce-08e3fbbe49e5f241b-q4e8rrlt.vpce-svc-01cc20f911a4789ff.us-east-2.vpce.amazonaws.com -u admin -p

You won’t be able to do so. Finally, edit the security group sgEC2 that we assigned to the endpoint and add a reference to itself for port 3306. It looks like this.

And if you try to access the RDS now, you’ll be able to do so.

Endpoint service

You can also go the the Provider account and delete the principal under Allowed principals.

It’s not needed anymore.