Last active
August 4, 2024 03:03
-
-
Save anniethiessen/188bcaf61a14593a49083f102df7a418 to your computer and use it in GitHub Desktop.
Script file to build and publish AWS EC2 instance start-up package.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env bash | |
| : ' | |
| Script file to build and publish AWS EC2 instance start-up package. | |
| Ensure AWS is installed and configured, | |
| define script variables, then execute script. | |
| Accepts -v (verbose output) and -q (quiet output) arguments. | |
| Prerequisites: | |
| - aws-cli 1.32.6 -> https://pypi.org/project/awscli/ | |
| Script Variables: | |
| - CONDA_ENV: The conda environment that has required packages installed. | |
| Optional, if current environment has them installed. | |
| - PACKAGE_DIRECTORY: Path and name of package to be duplicated, compressed and uploaded to S3. | |
| Required, no default. | |
| - PACKAGE_FILE_NAME: Name of package file to be built. | |
| Required, default "package". | |
| - PACKAGE_ARCHIVE_FILE_NAME: Name of archive package file to be compressed and uploaded to S3. | |
| Required, defaults to "package.zip". | |
| - PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS: Duration of presigned archive package validity in seconds. | |
| Optional, defaults to 300. | |
| - PACKAGE_S3_BUCKET: S3 bucket to which the package file will be uploaded. | |
| The ""check"" function checks if it exists and provides an option to create it if necessary. | |
| Required, no default. | |
| - PACKAGE_S3_PREFIX: S3 bucket prefix to which the package file should be uploaded. | |
| Optional, defaults to "packages". | |
| - PACKAGE_S3_BUCKET_REGION: AWS region of ""PACKAGE_S3_BUCKET"", if created. | |
| Optional, defaults to "us-west-2". | |
| - PACKAGE_S3_BUCKET_TAGS: List of tags to add to ""PACKAGE_S3_BUCKET"", if created. | |
| Optional, defaults to (). | |
| - PERFORM_CHECK: Whether to perform the ""check"" step. | |
| The ""check"" function checks if required pre-existing resources exist. | |
| Currently, only ""PACKAGE_S3_BUCKET"". | |
| Optional, defaults to "true". | |
| - PERFORM_PREPARE: Whether to perform the ""prepare"" step. | |
| The ""prepare"" function removes any previous archive files. | |
| Optional, defaults to "true". | |
| - PERFORM_BUILD: Whether to perform the build step. | |
| The ""build"" function compresses ""PACKAGE_DIRECTORY"" to ""PACKAGE_FILE_NAME"". | |
| Optional, defaults to "true". | |
| - PERFORM_UPLOAD: Whether to perform the upload step. | |
| The ""upload"" function uploads compressed package to S3. | |
| Optional, defaults to "true". | |
| - PERFORM_PRESIGN: Whether to perform the presign step. | |
| The ""presign"" function pre-signs package object URL for temporary access. | |
| Optional, defaults to "true". | |
| - PERFORM_CLEAN: Whether to perform the clean-up step. | |
| The ""clean"" function removes archive file. | |
| Optional, defaults to "true". | |
| Usage Example: | |
| CONDA_ENV=xxx | |
| PACKAGE_S3_BUCKET=xxx | |
| wget -cO - "<this_file_url>" > "temp.sh" | |
| chmod +x "temp.sh" | |
| source "temp.sh" "$@" | |
| rm -rf "temp.sh" | |
| ' | |
| #-------------------------------------------- | |
| #------------------ CONSTANTS --------------- | |
| #-------------------------------------------- | |
| CONDA_ENV="${CONDA_ENV}" | |
| PACKAGE_DIRECTORY="${PACKAGE_DIRECTORY}" | |
| PACKAGE_S3_BUCKET="${PACKAGE_S3_BUCKET}" | |
| PACKAGE_S3_BUCKET_REGION="${PACKAGE_S3_BUCKET_REGION:-us-west-2}" | |
| PACKAGE_S3_BUCKET_TAGS=${PACKAGE_S3_BUCKET_TAGS:-()} | |
| PACKAGE_S3_PREFIX="${PACKAGE_S3_PREFIX:-packages}" | |
| PACKAGE_FILE_NAME="${PACKAGE_FILE_NAME:-package}" | |
| PACKAGE_ARCHIVE_FILE_NAME="${PACKAGE_ARCHIVE_FILE_NAME:-package.zip}" | |
| PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS=${PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS:-300} | |
| PERFORM_CHECK=${PERFORM_CHECK:-true} | |
| PERFORM_PREPARE=${PERFORM_PREPARE:-true} | |
| PERFORM_BUILD=${PERFORM_BUILD:-true} | |
| PERFORM_UPLOAD=${PERFORM_UPLOAD:-true} | |
| PERFORM_PRESIGN=${PERFORM_PRESIGN:-true} | |
| PERFORM_CLEAN=${PERFORM_CLEAN:-true} | |
| #------- DO NOT EDIT BELOW THIS LINE -------- | |
| FORMAT_SCRIPT_URL="https://gist.githubusercontent.com/anniethiessen/efb6bc0e52ccfc8b330aa41364b53e97/raw/0012edc1f009a36d196f03f09fda68e70691860b/shell_script_essentials.sh" | |
| FORMAT_SCRIPT_NAME="shell_script_essentials.sh" | |
| #-------------------------------------------- | |
| #--------------- FUNCTIONS ------------------ | |
| #-------------------------------------------- | |
| function set_opts { | |
| while getopts ":qv" flag; do | |
| case "${flag}" in | |
| q) OUTPUT_FLAG="-q"; shift;; | |
| v) OUTPUT_FLAG="-v"; shift;; | |
| *) | |
| esac | |
| done | |
| OUTPUT_FLAG="${OUTPUT_FLAG:--q}" | |
| } | |
| function activate_env { | |
| local retval | |
| output_info_message "Activating local environment '${CONDA_ENV}'." | |
| if [ -n "${CONDA_ENV}" ]; then | |
| eval "$(conda shell.bash hook)" | |
| conda activate "${CONDA_ENV}" | |
| fi | |
| retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Local environment activated." | |
| else | |
| output_error_message "Local environment activation error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| if [[ "${OUTPUT}" == "${VERBOSE_OUTPUT}" ]]; then | |
| PYTHON_VERSION=$( python --version 2>&1 ) | |
| AWSEBCLI_VERSION=$( eb --version 2>&1 ) | |
| output_warning_message "Using ${PYTHON_VERSION} and ${AWSEBCLI_VERSION}" | |
| fi | |
| } | |
| function run_format_script { | |
| wget -cO - ${FORMAT_SCRIPT_URL} > ${FORMAT_SCRIPT_NAME} | |
| chmod +x ${FORMAT_SCRIPT_NAME} | |
| source ${FORMAT_SCRIPT_NAME} "$@" | |
| rm -rf ${FORMAT_SCRIPT_NAME} | |
| } | |
| function generate_s3_bucket_tags { | |
| local tag_set="" | |
| for tag in "${PACKAGE_S3_BUCKET_TAGS[@]}" ; do | |
| local tag_key="${tag%%:*}" | |
| local tag_value="${tag##*:}" | |
| local tag_dict="{Key=${tag_key},Value=${tag_value}}" | |
| local tag_set="${tag_set}${tag_set:+,}${tag_dict}" | |
| done | |
| local tag_set="[${tag_set}]" | |
| echo "${tag_set}" | |
| } | |
| function check_s3_bucket { | |
| aws s3api head-bucket --bucket "${PACKAGE_S3_BUCKET}" &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "S3 bucket ${PACKAGE_S3_BUCKET} exists." | |
| else | |
| output_error_message "S3 bucket ${PACKAGE_S3_BUCKET} does not exist." | |
| output_warning_message "Do you want to create S3 bucket ${PACKAGE_S3_BUCKET}?" | |
| select response in "Yes" "No"; do | |
| case ${response} in | |
| Yes ) create_s3_bucket; break;; | |
| No ) exit_script;; | |
| esac | |
| done | |
| fi | |
| } | |
| function create_s3_bucket { | |
| aws s3 mb s3://"${PACKAGE_S3_BUCKET}" \ | |
| --region "${PACKAGE_S3_BUCKET_REGION}" \ | |
| &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "S3 bucket ${PACKAGE_S3_BUCKET} created." | |
| else | |
| output_error_message "S3 bucket ${PACKAGE_S3_BUCKET} creation error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| local tag_set | |
| tag_set=$(generate_s3_bucket_tags) | |
| aws s3api put-bucket-tagging \ | |
| --bucket "${PACKAGE_S3_BUCKET}" \ | |
| --tagging "TagSet=${tag_set}" \ | |
| &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "S3 bucket tag set ${tag_set} added." | |
| else | |
| output_error_message "S3 bucket tag set ${tag_set} add error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| } | |
| function check { | |
| check_s3_bucket | |
| } | |
| function prepare () { | |
| rm -rf "${PACKAGE_FILE_NAME}" &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Previous package directory removed." | |
| else | |
| output_error_message "Previous package directory removal error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| rm -f "${PACKAGE_ARCHIVE_FILE_NAME}" &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Previous package archive removed." | |
| else | |
| output_error_message "Previous package archive removal error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| } | |
| function build () { | |
| cp -r "${PACKAGE_DIRECTORY}" "${PACKAGE_FILE_NAME}" &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package built." | |
| else | |
| output_error_message "Package build error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| zip -r "${PACKAGE_ARCHIVE_FILE_NAME}" "${PACKAGE_FILE_NAME}" &> ${OUTPUT} | |
| chmod 777 "${PACKAGE_ARCHIVE_FILE_NAME}" | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package compressed." | |
| else | |
| output_error_message "Package compression error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| } | |
| function upload () { | |
| aws s3 cp \ | |
| "${PACKAGE_ARCHIVE_FILE_NAME}" \ | |
| "s3://${PACKAGE_S3_BUCKET}/${PACKAGE_S3_PREFIX}/${PACKAGE_ARCHIVE_FILE_NAME}" \ | |
| &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package archive uploaded to S3." | |
| else | |
| output_error_message "Package archive upload to S3 error." ${PROMPT_VERBOSE} | |
| fi | |
| } | |
| function presign () { | |
| aws s3 presign \ | |
| "s3://${PACKAGE_S3_BUCKET}/${PACKAGE_S3_PREFIX}/${PACKAGE_ARCHIVE_FILE_NAME}" \ | |
| --expires-in ${PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS} \ | |
| &> ${VERBOSE_OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package object URL pre-signed. Valid for ${PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS} seconds." | |
| else | |
| output_error_message "Package object URL pre-sign error." ${PROMPT_VERBOSE} | |
| fi | |
| } | |
| function clean () { | |
| rm -rf "${PACKAGE_FILE_NAME}" \ | |
| &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package directory removed." | |
| else | |
| output_error_message "Package directory removal error." ${PROMPT_VERBOSE} | |
| fi | |
| rm -f "${PACKAGE_ARCHIVE_FILE_NAME}" &> ${OUTPUT} | |
| local retval=$? | |
| if [[ ${retval} -eq 0 ]]; then | |
| output_success_message "Package archive removed." | |
| else | |
| output_error_message "Package archive removal error." ${PROMPT_VERBOSE} | |
| exit_script | |
| fi | |
| } | |
| #-------------------------------------------- | |
| #------------------ MAIN -------------------- | |
| #-------------------------------------------- | |
| set_opts "$@" | |
| run_format_script ${OUTPUT_FLAG} | |
| activate_env | |
| output_header_message "----------------------------------------" | |
| output_header_message "[1/6] CHECK" | |
| output_header_message "checking required resources exist ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_CHECK}" == true ] ; then | |
| check | |
| else output_warning_message "Checks skipped"; fi | |
| output_header_message "----------------------------------------" | |
| output_header_message "[2/6] PREPARE" | |
| output_header_message "preparing work directory ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_PREPARE}" = true ] ; then | |
| prepare | |
| else output_warning_message "Preparation skipped"; fi | |
| output_header_message "----------------------------------------" | |
| output_header_message "[3/6] BUILD" | |
| output_header_message "building package ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_BUILD}" = true ] ; then | |
| build | |
| else output_warning_message "Build skipped"; fi | |
| output_header_message "----------------------------------------" | |
| output_header_message "[4/6] UPLOAD" | |
| output_header_message "uploading package to S3 ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_UPLOAD}" = true ] ; then | |
| upload | |
| else output_warning_message "Upload skipped"; fi | |
| output_header_message "----------------------------------------" | |
| output_header_message "[5/6] PRESIGN" | |
| output_header_message "presign package URL ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_PRESIGN}" = true ] ; then | |
| presign | |
| else output_warning_message "Presign skipped"; fi | |
| output_header_message "----------------------------------------" | |
| output_header_message "[6/6] CLEAN" | |
| output_header_message "cleaning local package ..." | |
| output_header_message "----------------------------------------" | |
| if [ "${PERFORM_CLEAN}" = true ] ; then | |
| clean | |
| else output_warning_message "Clean-up skipped"; fi |
Author
Author
Change Log:
v1: Initial
v2: add 'PACKAGE_ARCHIVE_SIGNATURE_VALID_SECONDS' variable
Author
Known Issues
v1:
-not parsing -v/-q opts properly
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
TODO: