Skip to content

Instantly share code, notes, and snippets.

@xschildw

xschildw/beanstalkapp.yaml

Created May 14, 2019 18:42
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save xschildw/0256a0e8383eddda1a729e462b339d70 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save xschildw/0256a0e8383eddda1a729e462b339d70 to your computer and use it in GitHub Desktop.
Download ZIP
dwsetup
Raw
beanstalkapp.yaml
AWSTemplateFormatVersion: 2010-09-09
Description: Basics for Beanstalk Datawarehouse
Parameters:
DwEbAppName:
Description: Elastic Beanstalk Datawarehouse Application Name
Type: String
Default: 'beanstalk-dw'
DwAppDeployBucket:
Description: Bucket where beanstalk deploys apps from
Type: String
SnsBounceNotificationEndpoint:
Type: String
Description: Email address for SNS bounce notifications
SnsNotificationEndpoint:
Type: String
Description: Email address for AWS SNS notifications
Resources:
DwBeanstalkApplication:
Type: 'AWS::ElasticBeanstalk::Application'
Properties:
ApplicationName: !Ref DwEbAppName
DwS3AppDeployBucket:
Type: 'AWS::S3::Bucket'
Properties:
BucketName: !Ref DwAppDeployBucket
LoadBalancerAccessLogsBucket:
Type: 'AWS::S3::Bucket'
LoadBalancerAccessLogsBucketPolicy:
Type: "AWS::S3::BucketPolicy"
Properties:
Bucket: !Ref LoadBalancerAccessLogsBucket
PolicyDocument:
Version: "2012-10-17"
Statement:
- Sid: ModAccess
Action:
- 's3:PutObject'
Effect: Allow
Resource:
- !Join
- ''
- - !GetAtt LoadBalancerAccessLogsBucket.Arn
- '/*'
Principal:
AWS:
- 127311923021
DwDeployS3ManagedPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
PolicyDocument:
Version: 2012-10-17
Statement:
- Sid: ListAccess
Action:
- 's3:ListBucket'
- 's3:GetBucketLocation'
Effect: Allow
Resource: !GetAtt DwS3AppDeployBucket.Arn
- Sid: ModAccess
Action:
- 's3:PutObject'
- 's3:PutObjectAcl'
- 's3:GetObject'
- 's3:GetObjectAcl'
- 's3:DeleteObject'
Effect: Allow
Resource: !Join
- ''
- - !GetAtt DwS3AppDeployBucket.Arn
- '/*'
DBSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Security group to grant EC2 instance access to DB instances
VpcId: !ImportValue us-east-1-vpc-VPCId
SecurityGroupIngress:
-
Description: "SynapseDw-VPC"
CidrIp: !ImportValue us-east-1-vpc-VpcCidr
FromPort: 3306
ToPort: 3306
IpProtocol: 'tcp'
EC2SecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Security group to grant EC2 instance access to DB instances
VpcId: !ImportValue us-east-1-vpc-VPCId
SecurityGroupIngress:
-
Description: "SynapseDw-VPC"
CidrIp: !ImportValue us-east-1-vpc-VpcCidr
FromPort: 3306
ToPort: 3306
IpProtocol: 'tcp'
DBSubnetGroup:
Type: AWS::RDS::DBSubnetGroup
Properties:
DBSubnetGroupDescription: Subnets for DB instance
SubnetIds:
- !ImportValue us-east-1-vpc-PrivateSubnet1
- !ImportValue us-east-1-vpc-PrivateSubnet2
DwBeanstalkServiceRole:
Type: 'AWS::IAM::Role'
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- elasticbeanstalk.amazonaws.com
Action:
- 'sts:AssumeRole'
Condition:
StringEquals:
sts:ExternalId: elasticbeanstalk
Path: /
ManagedPolicyArns:
- 'arn:aws:iam::aws:policy/service-role/AWSElasticBeanstalkEnhancedHealth'
- 'arn:aws:iam::aws:policy/service-role/AWSElasticBeanstalkService'
DwEC2ServiceRole:
Type: 'AWS::IAM::Role'
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- 'sts:AssumeRole'
Path: /
ManagedPolicyArns:
- 'arn:aws:iam::aws:policy/AWSElasticBeanstalkWebTier'
- 'arn:aws:iam::aws:policy/AmazonSNSFullAccess'
- 'arn:aws:iam::aws:policy/AmazonSESFullAccess'
# - !Ref SsmManagedPolicy
- !Ref DwDeployS3ManagedPolicy
# - !Ref KmsDecryptManagedPolicy
DwInstanceProfile:
Type: 'AWS::IAM::InstanceProfile'
Properties:
Path: /
Roles:
- !Ref DwEC2ServiceRole
DwSNSBounceTopic:
Type: "AWS::SNS::Topic"
Properties:
Subscription:
-
Endpoint: !Ref SnsBounceNotificationEndpoint
Protocol: "email"
DwSNSTopic:
Type: "AWS::SNS::Topic"
Properties:
Subscription:
-
Endpoint: !Ref SnsNotificationEndpoint
Protocol: "email"
Outputs:
DwBeanstalkAppName:
Value: !Ref DwBeanstalkApplication
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWBeanstalkAppName'
DwAppDeployBucket:
Value: !Ref DwAppDeployBucket
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWAppDeployBucketName'
DwLoadBalancerAccessLogsBucket:
Value: !Ref LoadBalancerAccessLogsBucket
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWLoadBalancerAccessLogsBucketName'
DBSecurityGroup:
Value: !Ref DBSecurityGroup
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DBSecurityGroup'
DBSubnetGroup:
Value: !Ref DBSubnetGroup
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DBSubnetGroup'
DwDeployS3ManagedPolicy:
Value: !Ref DwDeployS3ManagedPolicy
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWS3DeployManagedPolicy'
DwBeanstalkServiceRole:
Value: !Ref DwBeanstalkServiceRole
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWBeanstalkServiceRole'
DwInstanceProfile:
Value: !Ref DwInstanceProfile
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DWInstanceProfile'
Raw
beanstalkdb-restored.yaml
Description: AWS CloudFormation template to create an RDS instance for the warehouse workers build
Parameters:
DBClass:
Default: db.t2.large
Description: Database instance class
Type: String
DBSnapshotArn:
Description: The ARN of the snapshot to restore
Type: String
Subnets:
Description: List of subnetIds spanning 2 availability zones
Type: List<AWS::EC2::Subnet::Id>
VpcId:
Description: VpcId of the VPC to deploy in
Type: AWS::EC2::VPC::Id
Resources:
DBSubnetGroup:
Type: AWS::RDS::DBSubnetGroup
Properties:
DBSubnetGroupDescription: Subnets for DB instance
SubnetIds: !Ref 'Subnets'
DB:
Type: AWS::RDS::DBInstance
Properties:
AllowMajorVersionUpgrade: 'False'
AutoMinorVersionUpgrade: 'False'
DBInstanceClass: !Ref 'DBClass'
DBSnapshotIdentifier: !Ref 'DBSnapshotArn'
DBInstanceIdentifier: !Ref AWS::StackName
DBSubnetGroupName: !Ref 'DBSubnetGroup'
DeletionProtection: 'True'
Engine: MySQL
EngineVersion: 5.7.23
Tags:
- Key: Name
Value: !Ref AWS::StackName
VPCSecurityGroups:
- !ImportValue us-east-1-vpc-VpnSecurityGroup
- !ImportValue us-east-1-dwapp2019-DBSecurityGroup
Outputs:
JDBCConnectionString:
Description: JDBC connection string for database instance
Value: !Join
- ''
- - jdbc:mysql://
- !GetAtt 'DB.Endpoint.Address'
- ':'
- !GetAtt 'DB.Endpoint.Port'
Raw
beanstalkdb.yaml
Description: AWS CloudFormation template to create an RDS instance for the warehouse workers build
Parameters:
DBInstanceIdentifier:
Description: The RDS instance identifier
Type: String
DBAllocatedStorage:
Default: '5'
Description: The size of the database (Gb)
Type: Number
DBClass:
Default: db.t2.micro
Description: Database instance class
Type: String
DBName:
Default: MyDb
Description: The database name
MaxLength: '64'
MinLength: '1'
Type: String
DBPassword:
Description: The database admin account password
NoEcho: true
Type: String
DBUser:
Description: The database admin account username
NoEcho: true
Type: String
Resources:
WarehouseDB:
Type: AWS::RDS::DBInstance
Properties:
DBInstanceIdentifier: !Ref 'DBInstanceIdentifier'
AllocatedStorage: !Ref 'DBAllocatedStorage'
DBInstanceClass: !Ref 'DBClass'
DBName: !Ref 'DBName'
DBSubnetGroupName: !ImportValue us-east-1-dwapp2019-DBSubnetGroup
Engine: MySQL
EngineVersion: 5.7.23
MasterUserPassword: !Ref 'DBPassword'
MasterUsername: !Ref 'DBUser'
VPCSecurityGroups:
- !ImportValue us-east-1-vpc-VpnSecurityGroup
- !ImportValue us-east-1-dwapp2019-DBSecurityGroup
Outputs:
JDBCConnectionString:
Description: JDBC connection string for database instance
Value: !Join
- ''
- - jdbc:mysql://
- !GetAtt 'WarehouseDB.Endpoint.Address'
- ':'
- !GetAtt 'WarehouseDB.Endpoint.Port'
- '/'
- !Ref 'DBName'
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-DwDbEndpoint'
Raw
beanstalkdwinstance.yaml
Description: AWS CloudFormation template to create a beanstalk environment for a datawarehouse instance
Parameters:
AppHealthcheckUrl:
Type: String
Description: The AWS EB health check path
Default: '/health/check'
AutoScalingMaxSize:
Type: String
Default: '2'
AutoScalingMinSize:
Type: String
Default: '1'
EbHealthReportingSystem:
Type: String
Default: basic
AllowedValues:
- basic
- enhanced
ConstraintDescription: must be either basic or enhanced
EbSolutionStackName:
Description: The AWS Solution Stack
Type: String
Default: '64bit Amazon Linux 2018.03 v3.1.0 running Tomcat 8.5 Java 8'
EC2InstanceType:
Type: String
Description: Instance type to use for Elastic Beanstalk Instances
Default: t2.micro
EC2KeyName:
Description: An existing EC2 keypair to allow secure access to instances
Type: String
ArtifactKey:
Description: The S3 file that contains the deployed code
Type: String
UseCase:
Description: Use case for this stack
Type: String
AllowedValues:
- collator
- all-snapshots
- download-report
StackIamId:
Description: DW Stack IAM ID
Type: String
StackIamKey:
Description: DW Stack IAM key
Type: String
StackEnv:
Description: DW Stack environment name
Type: String
JdbcConnUrl:
Description: DW Stack JdbcConnectionUrl
Type: String
JdbcPassword:
Description: DW Stack JdbcConnection password
Type: String
JdbcUserName:
Description: DW Stack JdbcConnection user name
Type: String
SnapshotRecordBucketName:
Description: Name of the S3 bucket that contains the snapshot records
Type: String
AccessRecordBucketName:
Description: Name of the S3 bucket that contains the access records
Type: String
BackfillMonths:
Description: Number of months to backfill
Type: Number
Default: 6
DbSchema:
Description: The schema of the database
Type: String
Default: datawarehouse
Resources:
BeanstalkApplicationVersion:
Type: 'AWS::ElasticBeanstalk::ApplicationVersion'
Properties:
ApplicationName: !ImportValue us-east-1-dwapp2019-DWBeanstalkAppName
Description: Artifact for DW Beanstalk instance
SourceBundle:
S3Bucket: !ImportValue us-east-1-dwapp2019-DWAppDeployBucketName
S3Key: !Ref ArtifactKey
BeanstalkConfigTemplate:
Type: 'AWS::ElasticBeanstalk::ConfigurationTemplate'
Properties:
ApplicationName: !ImportValue us-east-1-dwapp2019-DWBeanstalkAppName
SolutionStackName: !Ref EbSolutionStackName
OptionSettings:
# EB environment options
- Namespace: 'aws:ec2:vpc'
OptionName: VPCId
Value: !ImportValue us-east-1-vpc-VPCId
- Namespace: 'aws:ec2:vpc'
OptionName: ELBScheme
Value: 'internal'
- Namespace: 'aws:ec2:vpc'
OptionName: AssociatePublicIpAddress
Value: 'false'
- Namespace: 'aws:ec2:vpc'
OptionName: Subnets
Value: !Join
- ','
- - !ImportValue us-east-1-vpc-PrivateSubnet1
- !ImportValue us-east-1-vpc-PrivateSubnet2
- Namespace: 'aws:ec2:vpc'
OptionName: ELBSubnets
Value: !Join
- ','
- - !ImportValue us-east-1-vpc-PrivateSubnet1
- !ImportValue us-east-1-vpc-PrivateSubnet2
- Namespace: 'aws:autoscaling:asg'
OptionName: MaxSize
Value: !Ref AutoScalingMaxSize
- Namespace: 'aws:autoscaling:asg'
OptionName: MinSize
Value: !Ref AutoScalingMinSize
- Namespace: 'aws:autoscaling:launchconfiguration'
OptionName: IamInstanceProfile
Value: !ImportValue us-east-1-dwapp2019-DWInstanceProfile
- Namespace: 'aws:autoscaling:launchconfiguration'
OptionName: InstanceType
Value: !Ref EC2InstanceType
- Namespace: 'aws:autoscaling:launchconfiguration'
OptionName: EC2KeyName
Value: !Ref EC2KeyName
- Namespace: 'aws:autoscaling:launchconfiguration'
OptionName: SecurityGroups
Value: !ImportValue us-east-1-vpc-VpnSecurityGroup
- Namespace: 'aws:autoscaling:updatepolicy:rollingupdate'
OptionName: RollingUpdateEnabled
Value: 'true'
# - Namespace: 'aws:autoscaling:updatepolicy:rollingupdate'
# OptionName: RollingUpdateType
# Value: !Ref EbRollingUpdateType
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: HealthCheckInterval
Value: 30
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: HealthCheckPath
Value: !Ref AppHealthcheckUrl
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: HealthCheckTimeout
Value: 5
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: HealthyThresholdCount
Value: 2
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: UnhealthyThresholdCount
Value: 5
- Namespace: 'aws:elasticbeanstalk:cloudwatch:logs'
OptionName: StreamLogs
Value: 'true'
- Namespace: 'aws:elasticbeanstalk:cloudwatch:logs'
OptionName: DeleteOnTerminate
Value: 'true'
- Namespace: 'aws:elasticbeanstalk:cloudwatch:logs'
OptionName: RetentionInDays
Value: '90'
- Namespace: 'aws:elasticbeanstalk:command'
OptionName: DeploymentPolicy
Value: AllAtOnce
- Namespace: 'aws:elasticbeanstalk:environment'
OptionName: ServiceRole
Value: !ImportValue us-east-1-dwapp2019-DWBeanstalkServiceRole
- Namespace: 'aws:elasticbeanstalk:environment'
OptionName: LoadBalancerType
Value: 'application'
- Namespace: 'aws:elbv2:loadbalancer'
OptionName: AccessLogsS3Bucket
Value: !ImportValue us-east-1-dwapp2019-DWLoadBalancerAccessLogsBucketName
- Namespace: 'aws:elbv2:loadbalancer'
OptionName: AccessLogsS3Enabled
Value: 'true'
- Namespace: 'aws:elasticbeanstalk:environment:process:default'
OptionName: Protocol
Value: 'HTTP'
- Namespace: 'aws:elasticbeanstalk:healthreporting:system'
OptionName: SystemType
Value: !Ref EbHealthReportingSystem
- Namespace: 'aws:elasticbeanstalk:hostmanager'
OptionName: LogPublicationControl
Value: 'true'
# Actual parameters to the app
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.stack.iam.key'
Value: !Ref StackIamKey
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.stack.iam.id'
Value: !Ref StackIamId
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.stack'
Value: !Ref StackEnv
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.jdbc.connection.url'
Value: !Ref JdbcConnUrl
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.jdbc.user.password'
Value: !Ref JdbcPassword
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.jdbc.user.username'
Value: !Ref JdbcUserName
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.instance.usecase'
Value: !Ref UseCase
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.bucket.access.record'
Value: !Ref AccessRecordBucketName
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.bucket.snapshot.record'
Value: !Ref SnapshotRecordBucketName
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.backfill.months'
Value: !Ref BackfillMonths
- Namespace: 'aws:elasticbeanstalk:application:environment'
OptionName: 'org.sagebionetworks.warehouse.workers.schema'
Value: !Ref DbSchema
BeanstalkEnvironment:
Type: 'AWS::ElasticBeanstalk::Environment'
Properties:
ApplicationName: !ImportValue us-east-1-dwapp2019-DWBeanstalkAppName
TemplateName: !Ref BeanstalkConfigTemplate
EnvironmentName: !Join
- '-'
- - !Ref 'AWS::StackName'
Tier:
Name: WebServer
Type: Standard
VersionLabel: !Ref BeanstalkApplicationVersion
Outputs:
BeanstalkEnvironmentName:
Value: !Ref BeanstalkEnvironment
Export:
Name: !Sub '${AWS::Region}-${AWS::StackName}-BeanstalkEnvironmentName'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment