Hi
After creating an ECS cluster and service similar to the one described here. My "healthy" service is unable to use the boto3 client to connect to a DynamoDB table in the account. The connection times out.
The experience is seems exactly like a stack overflow user's question here.
I have created my own instance via the CLI and put it into the same VPC (in the same public subnets) with the ecs instance and it is able to connect to dynamo without issue.
I have tried adding a vpc-endpoint for the service (despite that seeming like a pray in the wind), I have looked at the security groups at each stage and seen no issue.
It is important that I do not want to use the Fargate service. What is wrong with my configuration in CDK that is causing there to be no connectivity from my service to DynamoDB?
class MyConstruct(Construct):
def __init__(
self,
scope: Construct,
construct_id: str,
*,
environment: EnvController,
api_cert,
dynamo_table: aws_dynamodb.Table,
bucket: aws_s3.Bucket
):
super().__init__(scope, construct_id)
defaultVPC = aws_ec2.Vpc.from_lookup(self, "defaultVPC", is_default=True)
# Create the application load balancer and its security group
alb = aws_elasticloadbalancingv2.ApplicationLoadBalancer(
self,
"StreamProcessingLoadBalancer",
internet_facing=True,
vpc=defaultVPC,
)
albSecurityGroup = aws_ec2.SecurityGroup(self, 'alb-sg', vpc=defaultVPC, allow_all_outbound=True)
albSecurityGroup.add_ingress_rule(aws_ec2.Peer.any_ipv4(), aws_ec2.Port.tcp(443), "Allow ingress")
alb.add_security_group(albSecurityGroup)
# Define a target group for the ecs containers
target_group = aws_elasticloadbalancingv2.ApplicationTargetGroup(
self,
"StreamALBTargetGroup",
vpc=defaultVPC,
target_type=aws_elasticloadbalancingv2.TargetType.IP,
protocol=aws_elasticloadbalancingv2.ApplicationProtocol.HTTP,
protocol_version=aws_elasticloadbalancingv2.ApplicationProtocolVersion.HTTP1,
targets=[],
port=8765
)
target_group.configure_health_check(
path="/health",
protocol=aws_elasticloadbalancingv2.Protocol.HTTP
)
# Connect the target group to the ALB
listener = alb.add_listener(
'alb-listener',
open=True,
port=443,
certificates=[api_cert]
)
listener.add_target_groups('alb-target-group', target_groups=[target_group])
# Build the ESC cluster to house the processing containers
ecsCluster = aws_ecs.Cluster(self, 'ESPCluster', cluster_name=f"{environment.stage}-ESPCluster", vpc=defaultVPC)
# Add ec2 resource to the ecs cluster
asg = aws_autoscaling.AutoScalingGroup(
self,
'ClusterASG',
vpc=defaultVPC,
instance_type=aws_ec2.InstanceType.of(aws_ec2.InstanceClass.T4G, aws_ec2.InstanceSize.MICRO),
machine_image=aws_ecs.EcsOptimizedImage.amazon_linux2(aws_ecs.AmiHardwareType.ARM),
min_capacity=1,
max_capacity=10
)
capacity_provider =aws_ecs.AsgCapacityProvider(self, 'ClusterASGProvider', auto_scaling_group=asg)
ecsCluster.add_asg_capacity_provider(capacity_provider)
# Define upload the processor code to ECR and fetch the ECR repository to reference in the task definition
image = aws_ecr_assets.DockerImageAsset(
self,
'StreamProcessingImage',
directory='.',
platform=aws_ecr_assets.Platform.LINUX_ARM64
)
repo = aws_ecr.Repository.from_repository_name(self, 'ECRRepository', '*****')
# Create the task definition for the cluster
taskRole = aws_iam.Role(
self,
'ESPTaskRole',
assumed_by=aws_iam.ServicePrincipal("ecs-tasks.amazonaws.com")
)
taskRole.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonDynamoDBFullAccess'))
taskRole.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonS3FullAccess'))
taskDefinition = aws_ecs.TaskDefinition(
self,
'ESPTaskDefinition',
compatibility=aws_ecs.Compatibility.EC2,
task_role=taskRole,
network_mode=aws_ecs.NetworkMode.AWS_VPC,
cpu="1024"
)
container = taskDefinition.add_container(
'container',
image=aws_ecs.EcrImage(repo, image.image_tag),
cpu=1024,
memory_reservation_mib=500,
logging=aws_ecs.LogDriver.aws_logs(stream_prefix=f'{environment.stage}-ESP-ecs'),
environment={
"TABLE_NAME": dynamo_table.table_name,
"S3_MEASUREMENT_ROOT_PATH": f"s3://{bucket.bucket_name}",
"AWS_DEFAULT_REGION": (environment.region or 'eu-west-1')
}
)
container.add_port_mappings(aws_ecs.PortMapping(container_port=8765))
# Setup service
serviceSG = aws_ec2.SecurityGroup(self, "ECSEC2ServiceSG", vpc=defaultVPC, allow_all_outbound=True)
serviceSG.connections.allow_from(albSecurityGroup, aws_ec2.Port.all_tcp(), "Application Load Balancer")
service = aws_ecs.Ec2Service(
self,
"Service",
cluster=ecsCluster,
desired_count=1,
task_definition=taskDefinition,
assign_public_ip=False,
security_groups=[serviceSG],
capacity_provider_strategies=[
aws_ecs.CapacityProviderStrategy(
capacity_provider=capacity_provider.capacity_provider_name,
weight=1
)
]
)
service.attach_to_application_target_group(target_group)
autoscaling = service.auto_scale_task_count(
min_capacity=1,
max_capacity=10
)
autoscaling.scale_on_cpu_utilization("AutoscalingCPUService", target_utilization_percent=70)
Can you reach the public internet from within your container?