neumachen
2/23/2017 - 9:58 PM
continuous deployment of Golang Gin application in AWS Lambda and Amazon API Gateway with CodePipeline/CodeBuild
continuous deployment of Golang Gin application in AWS Lambda and Amazon API Gateway with CodePipeline/CodeBuild
package main
import (
"net/http"
"gopkg.in/gin-gonic/gin.v1"
"github.com/eawsy/aws-lambda-go-net/service/lambda/runtime/net"
"github.com/eawsy/aws-lambda-go-net/service/lambda/runtime/net/apigatewayproxy"
)
var Handle apigatewayproxy.Handler
func init() {
r := gin.Default()
r.GET("/users", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{ "message": "Hello World" })
})
//r.Run("localhost:8080")
ln := net.Listen()
Handle = apigatewayproxy.New(ln, nil).Handle
go http.Serve(ln, r)
}
AWSTemplateFormatVersion: 2010-09-09
Parameters:
ServiceName:
Type: String
Default: RestaurantService
Description: Name for the service, used in the code repository, Lambda function, and pipeline names
CodeBuildEnvironment:
Type: String
Default: "sapessi/aws-lambda-go18-codebuild:latest"
Description: Name of the image to use for the CodeBuild container
Resources:
# Code repository for the service
CodeRepository:
Type: AWS::CodeCommit::Repository
Properties:
RepositoryName: !Sub '${ServiceName}_repo'
RepositoryDescription: !Sub 'Repository for the ${ServiceName} service'
# CodeBuild project and resources (S3 Bucket for build artifacts, Role, Project)
BuildArtifactsBucket:
Type: AWS::S3::Bucket
CodeBuildServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Action:
- 'sts:AssumeRole'
Effect: Allow
Principal:
Service:
- codebuild.amazonaws.com
Path: /
Policies:
- PolicyName: CodeBuildAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Resource:
- !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ServiceName}_build'
- !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ServiceName}_build:*'
Action:
- 'logs:CreateLogGroup'
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
- Effect: Allow
Resource:
- !Sub 'arn:aws:s3:::${BuildArtifactsBucket}/*'
Action:
- 's3:GetObject'
- 's3:GetObjectVersion'
- 's3:PutObject'
CodeBuildProject:
Type: AWS::CodeBuild::Project
Properties:
Name: !Sub '${ServiceName}_build'
Artifacts:
Type: CODEPIPELINE
Environment:
Type: LINUX_CONTAINER
ComputeType: BUILD_GENERAL1_SMALL
Image: !Sub '${CodeBuildEnvironment}'
EnvironmentVariables:
- Name: BUILD_OUTPUT_BUCKET
Value: !Ref BuildArtifactsBucket
ServiceRole: !GetAtt CodeBuildServiceRole.Arn
Source:
Type: CODEPIPELINE
# Integration tests function
StartTestsFunctionExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Action:
- 'sts:AssumeRole'
Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Path: /
Policies:
- PolicyName: InvokeAsync
PolicyDocument:
Version: '2012-10-17'
Statement:
- Action:
- 'lambda:InvokeAsync'
- 'lambda:InvokeFunction'
- 'cloudformation:DescribeStacks'
- 'codepipeline:*'
- 'logs:*'
Effect: Allow
Resource: '*'
StartTestsFunction:
Type: AWS::Lambda::Function
Properties:
FunctionName: !Sub '${ServiceName}_start_tests'
Description: !Sub 'Starts integration tests function ${ServiceName}'
Handler: index.lambda_handler
MemorySize: 128
Role: !GetAtt StartTestsFunctionExecutionRole.Arn
Runtime: python2.7
Timeout: 10
Code:
ZipFile: >
import json
import boto3
import uuid
cp = boto3.client("codepipeline")
cf = boto3.client("cloudformation")
lambda_client = boto3.client("lambda")
def lambda_handler(event, context):
print json.dumps(event, sort_keys=True, indent=4, separators=(',', ': '))
job_id = event["CodePipeline.job"]["id"]
user_parameters = event["CodePipeline.job"]["data"]["actionConfiguration"]["configuration"]["UserParameters"]
stack_name = user_parameters.split("|")[0]
test_stack_name = user_parameters.split("|")[1]
stage = user_parameters.split("|")[2]
print("looking for ApiUrl in {} Stack".format(stack_name))
api_url, error = _get_property_from_stack(stack_name, "ApiUrl")
if not api_url:
_fail_job(job_id, error)
return
print("looking for TestFunction in {} Stack".format(test_stack_name))
test_function, error = _get_property_from_stack(test_stack_name, "TestFunction")
if not test_function:
_fail_job(job_id, error)
return
lambda_client.invoke_async(
FunctionName=test_function,
InvokeArgs=json.dumps({
"job_id": job_id,
"api_url": api_url
})
)
def _get_property_from_stack(stack_name, property_key):
stack_data = cf.describe_stacks(StackName=stack_name)
if not stack_data or len(stack_data["Stacks"]) == 0:
return None, "Could not find stack"
current_stack = stack_data["Stacks"][0]
if "Outputs" not in current_stack:
return None, "Could not find outputs in stack"
for cur_output in current_stack["Outputs"]:
if cur_output["OutputKey"] == property_key:
return cur_output["OutputValue"], None
return None, "Could not find {} output property in stack".format(property_key)
def _complete_job(job_id):
cp.put_job_success_result(jobId=job_id)
def _fail_job(job_id, failure_reason):
cp.put_job_failure_result(
jobId=job_id,
failureDetails={
'type': 'JobFailed',
'message': failure_reason,
'externalExecutionId': uuid.uuid4().hex
}
)
# CodePipeline definition and required roles
CFNPipelinePolicy:
Type: AWS::IAM::ManagedPolicy
Properties:
Description: CloudFormation Pipeline Execution Policy
Path: "/"
PolicyDocument:
Version: '2012-10-17'
Statement:
Effect: Allow
Action:
- 'cloudformation:CreateStack'
- 'cloudformation:DescribeStacks'
- 'cloudformation:DeleteStack'
- 'cloudformation:UpdateStack'
- 'cloudformation:CreateChangeSet'
- 'cloudformation:ExecuteChangeSet'
- 'cloudformation:DeleteChangeSet'
- 'cloudformation:DescribeChangeSet'
- 'cloudformation:SetStackPolicy'
- 'cloudformation:SetStackPolicy'
- 'cloudformation:ValidateTemplate'
- 'codebuild:StartBuild'
- 'codebuild:BatchGetBuilds'
Resource: "*"
CloudFormationExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
Action:
- 'sts:AssumeRole'
Effect: Allow
Principal:
Service:
- cloudformation.amazonaws.com
Path: /
ManagedPolicyArns:
- 'arn:aws:iam::aws:policy/AdministratorAccess'
PipelineExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Action:
- 'sts:AssumeRole'
Effect: Allow
Principal:
Service:
- codepipeline.amazonaws.com
Path: /
ManagedPolicyArns:
- 'arn:aws:iam::aws:policy/AWSCodeCommitFullAccess'
- 'arn:aws:iam::aws:policy/AmazonS3FullAccess'
- !Ref CFNPipelinePolicy
Policies:
- PolicyName: CodePipelineAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Action:
- 'iam:PassRole'
- 'lambda:InvokeFunction'
- 'lambda:ListFunctions'
- 'lambda:InvokeAsyc'
Effect: Allow
Resource: '*'
Pipeline:
Type: AWS::CodePipeline::Pipeline
Properties:
ArtifactStore:
Location: !Ref BuildArtifactsBucket
Type: S3
Name: !Sub ${ServiceName}_pipeline
RoleArn: !GetAtt PipelineExecutionRole.Arn
Stages:
- Name: Source
Actions:
- Name: CodeCommitRepo
ActionTypeId:
Category: Source
Owner: AWS
Provider: CodeCommit
Version: 1
Configuration:
RepositoryName: !Sub '${ServiceName}_repo'
BranchName: master
OutputArtifacts:
- Name: SourceZip
RunOrder: 1
- Name: Build
Actions:
- Name: CodeBuild
ActionTypeId:
Category: Build
Owner: AWS
Provider: CodeBuild
Version: 1
Configuration:
ProjectName: !Ref CodeBuildProject
InputArtifacts:
- Name: SourceZip
OutputArtifacts:
- Name: BuiltZip
- Name: DeployTests
Actions:
- Name: CreateChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_REPLACE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Tests'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Tests'
TemplatePath: BuiltZip::test-output_sam.yaml
Capabilities: CAPABILITY_IAM
InputArtifacts:
- Name: BuiltZip
OutputArtifacts:
- Name: TestChangeSet
RunOrder: 1
- Name: ExecuteChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_EXECUTE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Tests'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Tests'
OutputArtifacts:
- Name: ExecutedTestChangeSet
RunOrder: 2
- Name: Beta
Actions:
- Name: CreateChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_REPLACE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Beta'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Beta'
TemplatePath: BuiltZip::app-output_sam.yaml
Capabilities: CAPABILITY_IAM
InputArtifacts:
- Name: BuiltZip
RunOrder: 1
- Name: ExecuteChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_EXECUTE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Beta'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Beta'
OutputArtifacts:
- Name: !Sub '${ServiceName}BetaChangeSet'
RunOrder: 2
- Name: IntegrationTests
ActionTypeId:
Category: Invoke
Owner: AWS
Provider: Lambda
Version: 1
Configuration:
FunctionName: !Ref StartTestsFunction
UserParameters: !Sub '${ServiceName}-Stack-Beta|${ServiceName}-Stack-Tests|beta'
InputArtifacts:
- Name: !Sub '${ServiceName}BetaChangeSet'
RunOrder: 3
- Name: Gamma
Actions:
- Name: CreateChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_REPLACE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Gamma'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Gamma'
TemplatePath: BuiltZip::app-output_sam.yaml
Capabilities: CAPABILITY_IAM
InputArtifacts:
- Name: BuiltZip
RunOrder: 1
- Name: ExecuteChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_EXECUTE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Gamma'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Gamma'
OutputArtifacts:
- Name: !Sub '${ServiceName}GammaChangeSet'
RunOrder: 2
- Name: IntegrationTests
ActionTypeId:
Category: Invoke
Owner: AWS
Provider: Lambda
Version: 1
Configuration:
FunctionName: !Ref StartTestsFunction
UserParameters: !Sub '${ServiceName}-Stack-Gamma|${ServiceName}-Stack-Tests|gamma'
InputArtifacts:
- Name: !Sub '${ServiceName}GammaChangeSet'
RunOrder: 3
- Name: Prod
Actions:
- Name: DeploymentApproval
ActionTypeId:
Category: Approval
Owner: AWS
Provider: Manual
Version: 1
RunOrder: 1
- Name: CreateChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_REPLACE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Prod'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Prod'
TemplatePath: BuiltZip::app-output_sam.yaml
Capabilities: CAPABILITY_IAM
InputArtifacts:
- Name: BuiltZip
RunOrder: 2
- Name: ExecuteChangeSet
ActionTypeId:
Category: Deploy
Owner: AWS
Provider: CloudFormation
Version: 1
Configuration:
ActionMode: CHANGE_SET_EXECUTE
RoleArn: !GetAtt CloudFormationExecutionRole.Arn
StackName: !Sub '${ServiceName}-Stack-Prod'
ChangeSetName: !Sub '${ServiceName}-ChangeSet-Prod'
OutputArtifacts:
- Name: !Sub '${ServiceName}ProdChangeSet'
RunOrder: 3version: 0.1
phases:
install:
commands:
- go get gopkg.in/gin-gonic/gin.v1
pre_build:
commands:
- rm -rf /tmp/package.zip handler.so
build:
commands:
- mkdir dist
- export GOPATH=$GOPATH:$CODEBUILD_SRC_DIR/models; cd your_service/src; go build -buildmode=plugin -ldflags='-w -s -pluginpath lambda' -o /dist/handler.so
- cd /dist; /shim/pack handler handler.so package.zip
- mv /dist/package.zip /tmp/package.zip
post_build:
commands:
- aws cloudformation package --template-file app-sam.yaml --s3-bucket $BUILD_OUTPUT_BUCKET --output-template-file app-output_sam.yaml
- aws cloudformation package --template-file test-sam.yaml --s3-bucket $BUILD_OUTPUT_BUCKET --output-template-file test-output_sam.yaml
artifacts:
files:
- app-output_sam.yaml
- test-output_sam.yaml
discard-paths: yes
~ AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Resources:
TestFunction:
Type: AWS::Serverless::Function
Properties:
Handler: handler.Handle
CodeUri: /tmp/package.zip
Runtime: python2.7
MemorySize: 128
Timeout: 20
Events:
ProxyResource:
Type: Api
Properties:
Path: /{proxy+}
Method: ANY
Outputs:
ApiUrl:
Description: URL for application
Value: !Sub 'https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod'You can use CodePipeline and CodeBuild to create a continuous deployment/integration pipeline for serverless applications build on AWS Lambda and Amazon API Gateway. This sample application is written in Go with the Gin framework and uses the eawsy API Gateway proxy shim: https://github.com/eawsy/aws-lambda-go-net
We initially detailed our methodology in this blog post: https://aws.amazon.com/blogs/compute/continuous-deployment-for-serverless-applications/
We have used the shim technology created by eawsy to run Golang applications inside AWS Lambda (https://github.com/eawsy/aws-lambda-go-shim) and created a container that can be used with CodeBuild as part of our original pipeline template.
The container is available on DockerHub and is called sapessi/aws-lambda-go18-codebuild:latest. To use this container, simply change the Image property of the CodeBuild project environment.
The pipeline template, sample app, buildspec and SAM files are attached to this gist.