From 1e1ae9944621b0e8152d7934afa79adf9dcf0250 Mon Sep 17 00:00:00 2001 From: "Collin J. Doering" Date: Mon, 6 Dec 2021 22:25:05 -0500 Subject: [PATCH] Use guix for managing the required deployment environment * infra/variables.tf: Add new variable 'site_statis_files_dir' * infra/manifest.scm: Add guix manifest that captures all tools required for deploying this site. This currently includes terraform, in use terraform providers, as well as awscliv2 which is used directly from a null resource * infra/main.tf: Pin all provider version so they are available from the rekahsoft-guix channel Remove the need for the template provider. It is still included as these changes need to be applied to all environments before it can be removed. Remove TF-UPGRAGE-TODO's Use the new variable 'site_static_files_dir' for the location of the static site files to be deployed * channels.scm (channel): Add symlink to top-level channels file * infra/Makefile (SELECTED_WORKSPACE): Removed the dependency on terraform (clean): Add new PHONY target 'clean' which cleans up terraform temporary files (workspace): Add new PHONY target 'workspace which switches to user provided ENV * channels.scm (channel): Updated rekahsoft-guix channel * README.org (Features): Updated sections on deployment --- README.org | 137 ++++++++++++++++++++++++++++++++++++--------- channels.scm | 2 +- infra/Makefile | 11 +++- infra/channels.scm | 1 + infra/main.tf | 71 ++++++++++++----------- infra/manifest.scm | 37 ++++++++++++ infra/variables.tf | 3 + 7 files changed, 196 insertions(+), 66 deletions(-) create mode 120000 infra/channels.scm create mode 100644 infra/manifest.scm diff --git a/README.org b/README.org index 22ec558..4e4e0b8 100644 --- a/README.org +++ b/README.org @@ -13,8 +13,9 @@ * Features - [[http://en.wikipedia.org/wiki/Single-page_application][Single Page Application (SPA)]] -- Utilizes CSS 3 -- Uses HTML5 Application Cache for offline viewing of website +- Write blog posts and pages in markdown +- Support for math markup via MathJax +- RSS/Atom feed * Tools @@ -26,6 +27,7 @@ libraries: - [[http://www.getskeleton.com/][Skeleton]] is used for CSS boilerplate - [[http://www.mathjax.org/][MathJax]] is used for rendering mathematics - [[http://jquery.com][JQuery]] and [[https://github.com/asual/jquery-address][JQuery-address]] are used for various DOM manipulations +- [[https://guix.gnu.org/][Gnu Guix]] is used to manage development environments and packaging - [[http://inkscape.org/][Inkscape]] and the [[http://www.gimp.org/][Gimp]] were used to create various images/artwork - [[http://www.gnu.org/software/freefont/][Gnu Free Fonts]], specifically *FreeMono* is used as main font - [[http://www.gnu.org/software/emacs/][Gnu Emacs]] because there is no place like home; and no greater editor! @@ -48,10 +50,11 @@ License]]) are deployed dependent on which part of the site is in question. Plea :header-args: :session *vterm blog-rekahsoft-ca* :results none :END: -[[https://guix.gnu.org/][Guix]] is used to manage dependencies for this project. +[[https://guix.gnu.org/][Gnu Guix]] is used to package this project and manage its dependencies, as well as to provide +reproducible development environments. -A simple wrapper script ~site~ is provided that also takes care of building the static site -and offering access to hakyll commands. +A simple wrapper script [[./site][site]] is provided that wraps the hakyll powered site builder to +offer some additional functionality. ** Start Development Environment @@ -61,8 +64,10 @@ The development environment is defined by the following files: - [[./guix.scm][guix.scm]] :: Defines the package for this site, ~blog-rekahsoft-ca~. - [[./manifest.scm][manifest.scm]] :: Defines packages required for development of this site. +To start a development environment, run the following: + #+begin_src sh - guix time-machine -C channels.scm -- shell -CN -E LANG -E TERM -E PS1 -Df guix.scm -m manifest.scm + guix time-machine -C channels.scm -- shell -CN -E '^LANG$' -E '^TERM$' -E '^PS1$' -Df guix.scm -m manifest.scm #+end_src This uses the [[info:guix#Invoking guix time-machine][guix time-machine]] feature to ensure the development environment is reproducible @@ -76,6 +81,30 @@ indicates that development dependencies of the ~blog-rekahsoft-ca~ package shoul in the environment and ~-m manifest.scm~ ensures that extra tools for development are included as well. +*** Deployment Development Environment + +[[https://guix.gnu.org/][Gnu Guix]] is used, similar to in the [[*Start Development Environment][previous section]], to create environments with all tools +necessary for deployments, with a notable difference being a ~guix.scm~ file is not provided +or needed, as the deployment environment is used solely for its side effects. + +- [[./infra/channels.scm][infra/channels.scm]] :: Symlink to [[./channels.scm][../channels.scm]] to make the guix cli workflow nicer when + in the ~infra~ directory. Technically this doesn't need to be a symlink, however this would + complicate [[*Composing Site Development and Deployment Environments][Composing Site Development and Deployment Environments]]. +- [[./infra/manifest.scm][infra/manifest.scm]] :: Defines packages required for deployment of this site + +To start a deployment development environment, run the following: + +#+begin_src sh + cd infra + guix time-machine -C channels.scm -- shell -CN -E '^LANG$' -E '^TERM$' -E '^PS1$' -E '^AWS.*$' +#+end_src + +*** Composing Site Development and Deployment Environments + +#+begin_src sh + guix time-machine -C channels.scm -- shell -CN -E '^LANG$' -E '^TERM$' -E '^PS1$' -E '^AWS.*$' -Df guix.scm -m manifest.scm -m infra/manifest.scm +#+end_src + ** Build Site #+begin_src sh @@ -96,41 +125,95 @@ included as well. ** Deploy Site +Terraform is used to deploy this site. Its configuration files are located in ~./infra~. + +Under normal conditions, all deployments occur from my internal ci/cd system. This ensures +that the deployment process is reliable, repeatable and quick. However, in the case of both +development and emergency deployments, clear documentation surrounding the deployment process +is necessary. + +*** TODO ~site deploy~ command + #+begin_src sh site deploy #+end_src -** Test +*** Start [[*Deployment Development Environment][Deployment Development Environment]] +*** Setup a Particular Environment + +Three environments (terraform workspaces) are currently available, including: + + - default :: unused default terraform workspace + - staging :: https://www.blog.staging.rekahsoft.ca + - production :: https://www.blog.rekahsoft.ca #+begin_src sh - site test + make setup ENV= #+end_src +From this point onward, any ~make~ target run will operate on the selected environment, +unless its switched with the ~workspace~ or ~setup~ targets, or manually with ~terraform~. + +*** See What Infrastructure Will Change + +Run a terraform plan to see how the selected environments infrastructure will change. + +#+begin_src sh + make plan +#+end_src + +*** Deploy the Site + +Run a terraform apply to deploy to the selected environment. + +#+begin_src sh + make deploy +#+end_src + +*** Working with Terraform Directly + +Within a development environment, ~terraform~, its providers and all other dependencies are +available. As such, its possible to directly leverage ~terraform~ and its various operations. +This is particularly useful when debugging or adding make targets. + +** Clean up Guix Store + +#+begin_src sh + guix gc --list-dead | grep -e '^/gnu/store/.*-blog-rekahsoft-ca-.*' | xargs guix gc -D +#+end_src + +* Building a Release + +The software built that itself builds this blog is released as a Guix package. It is +currently not, and is not ever expected to be distributed via a channel, as it provides +little benefit to anyone except myself, and is meant to operate along with stateful data, +including the site templates, content, pages, posts, etc.. + +To build a release, run the following command: + +#+begin_src sh + guix time-machine -C channels.scm -- build -f guix.scm +#+end_src + +This will produce a guix package with the following three outputs: + +- ~out~ :: The ~blog-rekahsoft-ca~ site builder and ~gencss~ css generator binaries, as well + as ~site~ user script +- ~site~ :: A build of the website made with the site builder, etc.. in the ~out~ output of + this package, using the content at the same version +- ~static~ :: License file + +** TODO What is done with the release? + * Writing a Blog Post #+begin_src sh - guix time-machine -C channels.scm -- shell -CN -E LANG -#+end_src - -* Deploying - -Terraform is used to deploy this site. Its configuration files are located in ~./infra~. Three -workspaces are currently available, including: - - - default (unused) - - staging - - production - -For example, this is how to deploy the production version of the site: - -#+begin_src shell - cd infra - terraform workspace select production - terraform plan --var-file=production.tfvars --out local.plan - terraform apply local.plan + guix time-machine -C channels.scm -- shell -CN -E LANG -E TERM -E PS1 -f guix.scm -- site watch #+end_src * Known Issues If you have an issue while browsing [[http://www.blog.rekahsoft.ca][my blog]] please file a issue in the [[https://git.rekahsoft.ca/rekahsoft/blog-rekahsoft-ca/issues][blog-rekahsoft-ca]] issue tracker. + +To see a list of already known issues, see [[./TODO.org][TODO.org]]. diff --git a/channels.scm b/channels.scm index a61bd2d..fc0c24f 100644 --- a/channels.scm +++ b/channels.scm @@ -12,7 +12,7 @@ (name 'rekahsoft-guix) (url "https://git.rekahsoft.ca/rekahsoft/rekahsoft-guix.git") (commit - "792792e9408ac183d36eef43d2a9d943298d42d1") + "9915757715c96e0643bd5bd0dff466a1d9c652a7") (introduction (make-channel-introduction "191cdaa0947657e0c85fe89ebbb8e7b1e7a8e0a4" diff --git a/infra/Makefile b/infra/Makefile index 6a3a868..179afd9 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -5,11 +5,14 @@ .PHONY: default default: deploy -SELECTED_WORKSPACE := $(shell terraform workspace show) +SELECTED_WORKSPACE := $(shell cat .terraform/environment 2>/dev/null || echo default) ENV := $(if $(ENV),$(ENV),$(SELECTED_WORKSPACE)) .PHONY: setup -setup: +setup: init workspace + +.PHONY: workspace +workspace: ifneq ($(SELECTED_WORKSPACE),$(ENV)) ifndef CI @terraform workspace select $(ENV) @@ -38,3 +41,7 @@ destroy: setup @terraform destroy \ $(if $(ENV),--var-file=$(ENV).tfvars) \ $(ARGS) + +.PHONY: clean +clean: + @rm -rf .terraform diff --git a/infra/channels.scm b/infra/channels.scm new file mode 120000 index 0000000..3197c9f --- /dev/null +++ b/infra/channels.scm @@ -0,0 +1 @@ +../channels.scm \ No newline at end of file diff --git a/infra/main.tf b/infra/main.tf index 56ff589..8ba8744 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -11,7 +11,7 @@ terraform { provider "aws" { region = var.region - version = "~> 2.15" + version = "= 2.70.0" assume_role { role_arn = var.workspace_iam_roles[terraform.workspace] @@ -21,7 +21,7 @@ provider "aws" { provider "aws" { alias = "us_east_1" region = "us-east-1" - version = "~> 2.1" + version = "= 2.70.0" assume_role { role_arn = var.workspace_iam_roles[terraform.workspace] @@ -33,11 +33,11 @@ provider "null" { } provider "random" { - version = "~> 2.1" + version = "= 2.1.2" } provider "template" { - version = "~> 2.1" + version = "= 2.2.0" } # @@ -55,18 +55,41 @@ locals { naked_domain = "${local.subdomain}${var.dns_apex}" domain = "${local.www}${local.naked_domain}" project_env = "${var.project}-${terraform.workspace}" + + bucket_arn = aws_s3_bucket.static.arn + user_arn = aws_iam_user.app_deploy.arn + cloudfront_arn = aws_cloudfront_origin_access_identity.origin_access_identity.iam_arn } # # Data Sources -data "template_file" "s3_origin_policy" { - template = file("templates/s3_origin_policy.json") +data "aws_iam_policy_document" "s3_origin_policy" { + statement { + principals { + type = "AWS" + identifiers = [local.cloudfront_arn] + } + actions = ["s3:GetObject"] + resources = ["${local.bucket_arn}/*"] + } - vars = { - bucket_arn = aws_s3_bucket.static.arn - user_arn = aws_iam_user.app_deploy.arn - cloudfront_arn = aws_cloudfront_origin_access_identity.origin_access_identity.iam_arn + statement { + principals { + type = "AWS" + identifiers = [local.user_arn] + } + actions = ["s3:ListBucket"] + resources = ["${local.bucket_arn}"] + } + + statement { + principals { + type = "AWS" + identifiers = [local.user_arn] + } + actions = ["s3:*"] + resources = ["${local.bucket_arn}/*"] } } @@ -93,14 +116,6 @@ resource "aws_route53_record" "cert_validation" { name = aws_acm_certificate.cert.domain_validation_options[count.index]["resource_record_name"] type = aws_acm_certificate.cert.domain_validation_options[count.index]["resource_record_type"] ttl = 60 - # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to - # force an interpolation expression to be interpreted as a list by wrapping it - # in an extra set of list brackets. That form was supported for compatibilty in - # v0.11, but is no longer supported in Terraform v0.12. - # - # If the expression in the following list itself returns a list, remove the - # brackets to avoid interpretation as a list of lists. If the expression - # returns a single list item then leave it as-is and remove this TODO comment. records = [aws_acm_certificate.cert.domain_validation_options[count.index]["resource_record_value"]] } @@ -210,7 +225,7 @@ resource "aws_route53_record" "static_redirect_ipv6" { resource "aws_s3_bucket_policy" "static_policy" { bucket = aws_s3_bucket.static.id - policy = data.template_file.s3_origin_policy.rendered + policy = data.aws_iam_policy_document.s3_origin_policy.json } resource "aws_cloudfront_origin_access_identity" "origin_access_identity" { @@ -246,14 +261,6 @@ resource "aws_cloudfront_distribution" "cdn" { bucket = aws_s3_bucket.static_logs.bucket_domain_name } - # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to - # force an interpolation expression to be interpreted as a list by wrapping it - # in an extra set of list brackets. That form was supported for compatibilty in - # v0.11, but is no longer supported in Terraform v0.12. - # - # If the expression in the following list itself returns a list, remove the - # brackets to avoid interpretation as a list of lists. If the expression - # returns a single list item then leave it as-is and remove this TODO comment. aliases = [local.domain] default_cache_behavior { @@ -340,14 +347,6 @@ resource "aws_cloudfront_distribution" "cdn_redirect" { bucket = aws_s3_bucket.static_logs.bucket_domain_name } - # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to - # force an interpolation expression to be interpreted as a list by wrapping it - # in an extra set of list brackets. That form was supported for compatibilty in - # v0.11, but is no longer supported in Terraform v0.12. - # - # If the expression in the following list itself returns a list, remove the - # brackets to avoid interpretation as a list of lists. If the expression - # returns a single list item then leave it as-is and remove this TODO comment. aliases = [local.naked_domain] default_cache_behavior { @@ -404,7 +403,7 @@ aws configure --profile ${aws_iam_user.app_deploy.name} set aws_secret_access_ke aws configure --profile ${aws_iam_user.app_deploy.name} set region ${var.region}; : Sync latest app build to s3 bucket; -aws --profile ${aws_iam_user.app_deploy.name} s3 sync --delete ../_site s3://${aws_s3_bucket.static.id}/; +aws --profile ${aws_iam_user.app_deploy.name} s3 sync --delete ${var.site_static_files_dir} s3://${aws_s3_bucket.static.id}/; : Cleanup temporary aws config and credentials files rm $${AWS_CONFIG_FILE} $${AWS_SHARED_CREDENTIALS_FILE}; diff --git a/infra/manifest.scm b/infra/manifest.scm new file mode 100644 index 0000000..ded0fe1 --- /dev/null +++ b/infra/manifest.scm @@ -0,0 +1,37 @@ +;; (C) Copyright Collin J. Doering 2021 +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . + +;; File: manifest.scm +;; Author: Collin J. Doering +;; Date: Dec 2, 2021 + +(use-modules + (gnu packages) + (guix packages) + (guix profiles)) + +(setenv "PS1" "\\W [env]\\$ ") +(setenv "AWS_PAGER" "") + +(specifications->manifest + `("coreutils" + "make" + "terraform-wrapper@0.12.31" + "terraform-provider-aws" + "terraform-provider-null@2.1.2" + "terraform-provider-random" + "terraform-provider-template" + "awscliv2" + "nss-certs")) diff --git a/infra/variables.tf b/infra/variables.tf index b38cffa..dc9d1c9 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -25,3 +25,6 @@ variable "enable_naked_domain" { default = false } +variable "site_static_files_dir" { + default = "../_site" +}