Deploy Infrastructure #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Infrastructure | |
| on: | |
| workflow_run: | |
| workflows: [CI] | |
| types: [completed] | |
| branches: [main] | |
| workflow_dispatch: | |
| # Prevent overlapping infrastructure deployments. | |
| # Do not cancel an active deployment, since OpenTofu may be modifying resources. | |
| concurrency: | |
| group: infrastructure-deployment | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| deploy-test: | |
| name: Deploy TEST | |
| # Only deploy after successful CI on a push to main, | |
| # or when manually triggered from main. | |
| if: >- | |
| (github.event_name == 'workflow_run' && | |
| github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.event == 'push' && | |
| github.event.workflow_run.head_branch == 'main') || | |
| (github.event_name == 'workflow_dispatch' && | |
| github.ref == 'refs/heads/main') | |
| runs-on: [self-hosted, linux, assignment2] | |
| timeout-minutes: 45 | |
| environment: TEST | |
| # Authenticate to SkyHiGh using environment-specific credentials. | |
| env: | |
| OS_AUTH_TYPE: v3applicationcredential | |
| OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} | |
| OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} | |
| OS_IDENTITY_API_VERSION: "3" | |
| OS_INTERFACE: public | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | |
| KOPS_STATE_STORE: swift://demo-kops | |
| steps: | |
| # Checkout the exact commit validated by CI. | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} | |
| persist-credentials: false | |
| # Use the same OpenTofu version as the CI workflow. | |
| - name: Setup OpenTofu | |
| uses: opentofu/setup-opentofu@v2 | |
| with: | |
| tofu_version: "1.12.6" | |
| tofu_wrapper: false | |
| - name: Verify jq installation | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if ! command -v jq >/dev/null 2>&1; then | |
| echo "::error::jq is not installed on the runner." | |
| exit 1 | |
| fi | |
| jq --version | |
| # Verify OpenTofu, credentials, and TEST stack configurations. | |
| - name: Preflight checks | |
| shell: bash | |
| run: | | |
| tofu version | |
| test -f environments/test/network/providers.tf | |
| test -f environments/test/frontend/providers.tf | |
| : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" | |
| : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" | |
| echo "TEST deployment prerequisites are configured." | |
| # Install tools into the job's temporary directory. | |
| - name: Install kOps and kubectl | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| BIN_DIR="$RUNNER_TEMP/tofu-tools" | |
| mkdir -p "$BIN_DIR" | |
| # Kubernetes API server | |
| KUBECTL_VERSION="v1.36.4" | |
| curl -fsSL -o "$BIN_DIR/kubectl" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" | |
| curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kubectl.sha256) kubectl" | sha256sum --check | |
| ) | |
| # Download kOps and verify its checksum. | |
| KOPS_VERSION="v1.36.2" | |
| curl -fsSL -o "$BIN_DIR/kops" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" | |
| curl -fsSL -o "$BIN_DIR/kops.sha256" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kops.sha256) kops" | sha256sum --check | |
| ) | |
| chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" | |
| # Make both tools available to subsequent steps. | |
| echo "$BIN_DIR" >> "$GITHUB_PATH" | |
| - name: Verify installed tools | |
| shell: bash | |
| run: | | |
| kops version | |
| kubectl version --client | |
| - name: Verify kOps state store access | |
| shell: bash | |
| env: | |
| # Override TEST deployment credentials for this step only. | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps credential secret}" | |
| : "${KOPS_STATE_STORE:?Missing kOps state store}" | |
| kops get clusters --state="$KOPS_STATE_STORE" | |
| kops get cluster einalien.k8s.local \ | |
| --state="$KOPS_STATE_STORE" > /dev/null | |
| echo "kOps state store access verified." | |
| # Generate short-lived Kubernetes credentials. | |
| - name: Generate Kubernetes kubeconfig | |
| shell: bash | |
| env: | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| KUBECONFIG_FILE="$RUNNER_TEMP/assignment2-kubeconfig" | |
| umask 077 | |
| kops export kubeconfig einalien.k8s.local \ | |
| --admin=1h \ | |
| --state="$KOPS_STATE_STORE" \ | |
| --kubeconfig="$KUBECONFIG_FILE" | |
| test -s "$KUBECONFIG_FILE" | |
| chmod 600 "$KUBECONFIG_FILE" | |
| # Verify that the generated configuration contains a cluster context. | |
| KUBECONFIG="$KUBECONFIG_FILE" kubectl config current-context | |
| # Make the kubeconfig available to subsequent steps. | |
| echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| # Confirm that the runner can communicate with Kubernetes. | |
| - name: Verify Kubernetes state access | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| kubectl --request-timeout=15s get --raw=/readyz | |
| kubectl --request-timeout=15s get namespace iac | |
| # Check the permissions required by the state backend. | |
| kubectl auth can-i get secrets -n iac --quiet | |
| kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet | |
| echo "Kubernetes state backend access verified." | |
| # Verify OpenTofu can access existing TEST remote state. | |
| - name: Verify TEST remote state | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/test/$STACK" | |
| echo "Initializing TEST $STACK backend..." | |
| tofu -chdir="$STACK_DIR" init \ | |
| -reconfigure \ | |
| -input=false \ | |
| -lockfile=readonly \ | |
| -no-color | |
| echo "Reading TEST $STACK state..." | |
| RESOURCES="$(tofu -chdir="$STACK_DIR" state list)" | |
| if [ -z "$RESOURCES" ]; then | |
| echo "ERROR: No managed resources found in TEST $STACK state." | |
| exit 1 | |
| fi | |
| RESOURCE_COUNT="$(printf '%s\n' "$RESOURCES" | wc -l)" | |
| echo "TEST $STACK: $RESOURCE_COUNT managed resources found." | |
| done | |
| echo "TEST remote state access verified." | |
| # Plan and automatically apply safe TEST infrastructure changes. | |
| - name: Deploy TEST infrastructure | |
| shell: bash | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} | |
| TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} | |
| run: | | |
| set -euo pipefail | |
| umask 077 | |
| # Verify required variables and configuration files. | |
| : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR secret}" | |
| : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME variable}" | |
| test -f environments/test/network/deployment.tfvars | |
| test -f environments/test/frontend/deployment.tfvars | |
| # Deploy network before frontend. | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/test/$STACK" | |
| PLAN_FILE="$RUNNER_TEMP/test-$STACK.tfplan" | |
| PLAN_JSON="$RUNNER_TEMP/test-$STACK.plan.json" | |
| PLAN_LOG="$RUNNER_TEMP/test-$STACK.plan.log" | |
| APPLY_LOG="$RUNNER_TEMP/test-$STACK.apply.log" | |
| echo "::group::Deploy TEST $STACK" | |
| # Generate a saved plan. | |
| set +e | |
| tofu -chdir="$STACK_DIR" plan \ | |
| -var-file="deployment.tfvars" \ | |
| -out="$PLAN_FILE" \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| -detailed-exitcode > "$PLAN_LOG" 2>&1 | |
| EXIT_CODE=$? | |
| set -e | |
| # Handle planning errors. | |
| if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 2 ]; then | |
| echo "::error::TEST $STACK planning failed." | |
| exit 1 | |
| fi | |
| # Skip apply when there are no changes. | |
| if [ "$EXIT_CODE" -eq 0 ]; then | |
| echo "TEST $STACK: No changes required." | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| # Convert the saved plan into JSON for inspection. | |
| test -s "$PLAN_FILE" | |
| tofu -chdir="$STACK_DIR" show -json \ | |
| "$PLAN_FILE" > "$PLAN_JSON" | |
| # Only allow known non-destructive resource actions. | |
| if ! jq -e ' | |
| (.format_version | type == "string") and | |
| (.resource_changes | type == "array") and | |
| all(.resource_changes[]?; | |
| (.change.actions == ["no-op"] or | |
| .change.actions == ["read"] or | |
| .change.actions == ["create"] or | |
| .change.actions == ["update"]) | |
| ) | |
| ' "$PLAN_JSON" > /dev/null; then | |
| echo "::error::TEST $STACK contains invalid or destructive changes." | |
| echo "Automatic apply has been blocked." | |
| exit 1 | |
| fi | |
| # Display resource actions without exposing configuration values. | |
| echo "Planned TEST $STACK changes:" | |
| jq -r ' | |
| .resource_changes[]? | |
| | select(.change.actions != ["no-op"]) | |
| | "\(.address): \(.change.actions | join(", "))" | |
| ' "$PLAN_JSON" | |
| # Apply the exact saved plan. | |
| echo "Applying TEST $STACK..." | |
| if ! tofu -chdir="$STACK_DIR" apply \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| "$PLAN_FILE" > "$APPLY_LOG" 2>&1; then | |
| echo "::error::TEST $STACK apply failed." | |
| exit 1 | |
| fi | |
| echo "TEST $STACK deployed successfully." | |
| echo "::endgroup::" | |
| done | |
| echo "TEST infrastructure deployment completed." | |
| # Remove temporary credentials, saved plans and logs. | |
| - name: Clean up temporary files | |
| if: always() | |
| shell: bash | |
| run: | | |
| rm -f "$RUNNER_TEMP/assignment2-kubeconfig" | |
| rm -f "$RUNNER_TEMP"/test-*.tfplan | |
| rm -f "$RUNNER_TEMP"/test-*.plan.json | |
| rm -f "$RUNNER_TEMP"/test-*.plan.log | |
| rm -f "$RUNNER_TEMP"/test-*.apply.log | |
| deploy-stage: | |
| name: Deploy STAGE | |
| needs: deploy-test | |
| runs-on: [self-hosted, linux, assignment2] | |
| timeout-minutes: 45 | |
| environment: STAGE | |
| env: | |
| OS_AUTH_TYPE: v3applicationcredential | |
| OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} | |
| OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} | |
| OS_IDENTITY_API_VERSION: "3" | |
| OS_INTERFACE: public | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | |
| KOPS_STATE_STORE: swift://demo-kops | |
| steps: | |
| # Checkout the same commit deployed to TEST. | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} | |
| persist-credentials: false | |
| - name: Setup OpenTofu | |
| uses: opentofu/setup-opentofu@v2 | |
| with: | |
| tofu_version: "1.12.6" | |
| tofu_wrapper: false | |
| - name: Verify jq installation | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if ! command -v jq >/dev/null 2>&1; then | |
| echo "::error::jq is not installed on the runner." | |
| exit 1 | |
| fi | |
| jq --version | |
| - name: Preflight checks | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| tofu version | |
| test -f environments/stage/network/providers.tf | |
| test -f environments/stage/frontend/providers.tf | |
| test -f environments/stage/network/deployment.tfvars | |
| test -f environments/stage/frontend/deployment.tfvars | |
| : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" | |
| : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" | |
| echo "STAGE deployment prerequisites verified." | |
| # Install tools for this job's runner. | |
| - name: Install kOps and kubectl | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| BIN_DIR="$RUNNER_TEMP/tofu-tools" | |
| mkdir -p "$BIN_DIR" | |
| KUBECTL_VERSION="v1.36.4" | |
| KOPS_VERSION="v1.36.2" | |
| curl -fsSL -o "$BIN_DIR/kubectl" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" | |
| curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kubectl.sha256) kubectl" | sha256sum --check | |
| ) | |
| curl -fsSL -o "$BIN_DIR/kops" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" | |
| curl -fsSL -o "$BIN_DIR/kops.sha256" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kops.sha256) kops" | sha256sum --check | |
| ) | |
| chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" | |
| echo "$BIN_DIR" >> "$GITHUB_PATH" | |
| - name: Verify installed tools | |
| shell: bash | |
| run: | | |
| kops version | |
| kubectl version --client | |
| # Authenticate to the existing kOps state store. | |
| - name: Verify kOps state store access | |
| shell: bash | |
| env: | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps credential secret}" | |
| kops get cluster einalien.k8s.local \ | |
| --state="$KOPS_STATE_STORE" > /dev/null | |
| echo "kOps state store access verified." | |
| - name: Generate Kubernetes kubeconfig | |
| shell: bash | |
| env: | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| KUBECONFIG_FILE="$RUNNER_TEMP/assignment2-stage-kubeconfig" | |
| umask 077 | |
| kops export kubeconfig einalien.k8s.local \ | |
| --admin=1h \ | |
| --state="$KOPS_STATE_STORE" \ | |
| --kubeconfig="$KUBECONFIG_FILE" | |
| test -s "$KUBECONFIG_FILE" | |
| chmod 600 "$KUBECONFIG_FILE" | |
| KUBECONFIG="$KUBECONFIG_FILE" kubectl config current-context | |
| echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| - name: Verify Kubernetes state access | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| kubectl --request-timeout=15s get --raw=/readyz | |
| kubectl --request-timeout=15s get namespace iac | |
| # Check permissions needed by the backend. | |
| for VERB in get list create update patch; do | |
| kubectl auth can-i "$VERB" secrets -n iac --quiet | |
| done | |
| kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet | |
| echo "Kubernetes backend access verified." | |
| # STAGE may have no state yet. Do not require existing resources. | |
| - name: Initialize STAGE remote state | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/stage/$STACK" | |
| echo "Initializing STAGE $STACK backend..." | |
| tofu -chdir="$STACK_DIR" init \ | |
| -reconfigure \ | |
| -input=false \ | |
| -lockfile=readonly \ | |
| -no-color | |
| done | |
| echo "STAGE backends initialized." | |
| # Deploy the network first, then the frontend. | |
| - name: Deploy STAGE infrastructure | |
| shell: bash | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} | |
| TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} | |
| run: | | |
| set -euo pipefail | |
| umask 077 | |
| : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR}" | |
| : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME}" | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/stage/$STACK" | |
| PLAN_FILE="$RUNNER_TEMP/stage-$STACK.tfplan" | |
| PLAN_JSON="$RUNNER_TEMP/stage-$STACK.plan.json" | |
| PLAN_LOG="$RUNNER_TEMP/stage-$STACK.plan.log" | |
| APPLY_LOG="$RUNNER_TEMP/stage-$STACK.apply.log" | |
| echo "::group::Deploy STAGE $STACK" | |
| # Generate a saved plan. | |
| set +e | |
| tofu -chdir="$STACK_DIR" plan \ | |
| -var-file="deployment.tfvars" \ | |
| -out="$PLAN_FILE" \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| -detailed-exitcode > "$PLAN_LOG" 2>&1 | |
| EXIT_CODE=$? | |
| set -e | |
| if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 2 ]; then | |
| echo "::error::STAGE $STACK planning failed." | |
| exit 1 | |
| fi | |
| if [ "$EXIT_CODE" -eq 0 ]; then | |
| echo "STAGE $STACK: No changes required." | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| # Inspect planned changes before applying. | |
| test -s "$PLAN_FILE" | |
| tofu -chdir="$STACK_DIR" show -json \ | |
| "$PLAN_FILE" > "$PLAN_JSON" | |
| # Block destructive changes. | |
| if ! jq -e ' | |
| (.format_version | type == "string") and | |
| (.resource_changes | type == "array") and | |
| all(.resource_changes[]?; | |
| (.change.actions == ["no-op"] or | |
| .change.actions == ["read"] or | |
| .change.actions == ["create"] or | |
| .change.actions == ["update"]) | |
| ) | |
| ' "$PLAN_JSON" > /dev/null; then | |
| echo "::error::STAGE $STACK contains invalid or destructive changes." | |
| echo "Automatic apply has been blocked." | |
| exit 1 | |
| fi | |
| echo "Planned STAGE $STACK changes:" | |
| jq -r ' | |
| .resource_changes[]? | |
| | select(.change.actions != ["no-op"]) | |
| | "\(.address): \(.change.actions | join(", "))" | |
| ' "$PLAN_JSON" | |
| # Apply the exact saved plan. | |
| echo "Applying STAGE $STACK..." | |
| if ! tofu -chdir="$STACK_DIR" apply \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| "$PLAN_FILE" > "$APPLY_LOG" 2>&1; then | |
| echo "::error::STAGE $STACK apply failed." | |
| exit 1 | |
| fi | |
| echo "STAGE $STACK deployed successfully." | |
| echo "::endgroup::" | |
| done | |
| echo "STAGE infrastructure deployment completed." | |
| - name: Clean up temporary files | |
| if: always() | |
| shell: bash | |
| run: | | |
| rm -f "$RUNNER_TEMP/assignment2-stage-kubeconfig" | |
| rm -f "$RUNNER_TEMP"/stage-*.tfplan | |
| rm -f "$RUNNER_TEMP"/stage-*.plan.json | |
| rm -f "$RUNNER_TEMP"/stage-*.plan.log | |
| rm -f "$RUNNER_TEMP"/stage-*.apply.log | |
| deploy-prod: | |
| name: Deploy PROD | |
| needs: deploy-stage | |
| # Only deploy PROD when manually triggered from main | |
| # with the deploy_prod checkbox selected. | |
| if: >- | |
| github.event_name == 'workflow_dispatch' && | |
| github.ref == 'refs/heads/main' | |
| runs-on: [self-hosted, linux, assignment2] | |
| timeout-minutes: 45 | |
| environment: PROD | |
| # Authenticate to SkyHiGh using PROD-specific credentials. | |
| env: | |
| OS_AUTH_TYPE: v3applicationcredential | |
| OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} | |
| OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} | |
| OS_IDENTITY_API_VERSION: "3" | |
| OS_INTERFACE: public | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | |
| KOPS_STATE_STORE: swift://demo-kops | |
| steps: | |
| # Checkout the same commit deployed to TEST and STAGE. | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} | |
| persist-credentials: false | |
| # Use the same OpenTofu version as CI, TEST and STAGE. | |
| - name: Setup OpenTofu | |
| uses: opentofu/setup-opentofu@v2 | |
| with: | |
| tofu_version: "1.12.6" | |
| tofu_wrapper: false | |
| - name: Verify jq installation | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if ! command -v jq >/dev/null 2>&1; then | |
| echo "::error::jq is not installed on the runner." | |
| exit 1 | |
| fi | |
| jq --version | |
| # Verify OpenTofu, credentials and PROD configurations. | |
| - name: Preflight checks | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| tofu version | |
| test -f environments/prod/network/providers.tf | |
| test -f environments/prod/frontend/providers.tf | |
| test -f environments/prod/network/deployment.tfvars | |
| test -f environments/prod/frontend/deployment.tfvars | |
| : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" | |
| : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" | |
| echo "PROD deployment prerequisites verified." | |
| # Install tools into the job's temporary directory. | |
| - name: Install kOps and kubectl | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| BIN_DIR="$RUNNER_TEMP/tofu-tools" | |
| mkdir -p "$BIN_DIR" | |
| # Kubernetes API server | |
| KUBECTL_VERSION="v1.36.4" | |
| curl -fsSL -o "$BIN_DIR/kubectl" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" | |
| curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ | |
| "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kubectl.sha256) kubectl" | sha256sum --check | |
| ) | |
| # Download kOps and verify its checksum. | |
| KOPS_VERSION="v1.36.2" | |
| curl -fsSL -o "$BIN_DIR/kops" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" | |
| curl -fsSL -o "$BIN_DIR/kops.sha256" \ | |
| "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" | |
| ( | |
| cd "$BIN_DIR" | |
| echo "$(cat kops.sha256) kops" | sha256sum --check | |
| ) | |
| chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" | |
| # Make tools available to subsequent steps. | |
| echo "$BIN_DIR" >> "$GITHUB_PATH" | |
| - name: Verify installed tools | |
| shell: bash | |
| run: | | |
| kops version | |
| kubectl version --client | |
| # Authenticate to the existing kOps state store. | |
| - name: Verify kOps state store access | |
| shell: bash | |
| env: | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps credential ID}" | |
| : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps credential secret}" | |
| : "${KOPS_STATE_STORE:?Missing kOps state store}" | |
| kops get cluster einalien.k8s.local \ | |
| --state="$KOPS_STATE_STORE" > /dev/null | |
| echo "kOps state store access verified." | |
| # Generate short-lived Kubernetes credentials. | |
| - name: Generate Kubernetes kubeconfig | |
| shell: bash | |
| env: | |
| OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} | |
| OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} | |
| run: | | |
| set -euo pipefail | |
| KUBECONFIG_FILE="$RUNNER_TEMP/assignment2-prod-kubeconfig" | |
| umask 077 | |
| kops export kubeconfig einalien.k8s.local \ | |
| --admin=1h \ | |
| --state="$KOPS_STATE_STORE" \ | |
| --kubeconfig="$KUBECONFIG_FILE" | |
| test -s "$KUBECONFIG_FILE" | |
| chmod 600 "$KUBECONFIG_FILE" | |
| KUBECONFIG="$KUBECONFIG_FILE" kubectl config current-context | |
| echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" | |
| # Confirm access to the Kubernetes state backend. | |
| - name: Verify Kubernetes state access | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| kubectl --request-timeout=15s get --raw=/readyz | |
| kubectl --request-timeout=15s get namespace iac | |
| # Check permissions needed by the backend. | |
| for VERB in get list create update patch; do | |
| kubectl auth can-i "$VERB" secrets -n iac --quiet | |
| done | |
| kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet | |
| echo "Kubernetes backend access verified." | |
| # PROD may have no state yet. | |
| # Do not require existing managed resources. | |
| - name: Initialize PROD remote state | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/prod/$STACK" | |
| echo "Initializing PROD $STACK backend..." | |
| tofu -chdir="$STACK_DIR" init \ | |
| -reconfigure \ | |
| -input=false \ | |
| -lockfile=readonly \ | |
| -no-color | |
| done | |
| echo "PROD backends initialized." | |
| # Deploy network before frontend. | |
| # Only permit non-destructive infrastructure changes. | |
| - name: Deploy PROD infrastructure | |
| shell: bash | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} | |
| TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} | |
| run: | | |
| set -euo pipefail | |
| umask 077 | |
| # Verify required variables. | |
| : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR}" | |
| : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME}" | |
| for STACK in network frontend; do | |
| STACK_DIR="environments/prod/$STACK" | |
| PLAN_FILE="$RUNNER_TEMP/prod-$STACK.tfplan" | |
| PLAN_JSON="$RUNNER_TEMP/prod-$STACK.plan.json" | |
| PLAN_LOG="$RUNNER_TEMP/prod-$STACK.plan.log" | |
| APPLY_LOG="$RUNNER_TEMP/prod-$STACK.apply.log" | |
| echo "::group::Deploy PROD $STACK" | |
| # Generate a saved plan. | |
| set +e | |
| tofu -chdir="$STACK_DIR" plan \ | |
| -var-file="deployment.tfvars" \ | |
| -out="$PLAN_FILE" \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| -detailed-exitcode > "$PLAN_LOG" 2>&1 | |
| EXIT_CODE=$? | |
| set -e | |
| # Handle planning errors. | |
| if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 2 ]; then | |
| echo "::error::PROD $STACK planning failed." | |
| exit 1 | |
| fi | |
| # Skip apply when there are no changes. | |
| if [ "$EXIT_CODE" -eq 0 ]; then | |
| echo "PROD $STACK: No changes required." | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| # Convert the saved plan into JSON. | |
| test -s "$PLAN_FILE" | |
| tofu -chdir="$STACK_DIR" show -json \ | |
| "$PLAN_FILE" > "$PLAN_JSON" | |
| # Only allow non-destructive resource actions. | |
| if ! jq -e ' | |
| (.format_version | type == "string") and | |
| (.resource_changes | type == "array") and | |
| all(.resource_changes[]?; | |
| (.change.actions == ["no-op"] or | |
| .change.actions == ["read"] or | |
| .change.actions == ["create"] or | |
| .change.actions == ["update"]) | |
| ) | |
| ' "$PLAN_JSON" > /dev/null; then | |
| echo "::error::PROD $STACK contains invalid or destructive changes." | |
| echo "Automatic apply has been blocked." | |
| exit 1 | |
| fi | |
| # Display resource actions without exposing values. | |
| echo "Planned PROD $STACK changes:" | |
| jq -r ' | |
| .resource_changes[]? | |
| | select(.change.actions != ["no-op"]) | |
| | "\(.address): \(.change.actions | join(", "))" | |
| ' "$PLAN_JSON" | |
| # Apply the exact saved plan. | |
| echo "Applying PROD $STACK..." | |
| if ! tofu -chdir="$STACK_DIR" apply \ | |
| -input=false \ | |
| -no-color \ | |
| -lock-timeout=5m \ | |
| "$PLAN_FILE" > "$APPLY_LOG" 2>&1; then | |
| echo "::error::PROD $STACK apply failed." | |
| exit 1 | |
| fi | |
| echo "PROD $STACK deployed successfully." | |
| echo "::endgroup::" | |
| done | |
| echo "PROD infrastructure deployment completed." | |
| # Remove temporary credentials, plans and logs. | |
| - name: Clean up temporary files | |
| if: always() | |
| shell: bash | |
| run: | | |
| rm -f "$RUNNER_TEMP/assignment2-prod-kubeconfig" | |
| rm -f "$RUNNER_TEMP"/prod-*.tfplan | |
| rm -f "$RUNNER_TEMP"/prod-*.plan.json | |
| rm -f "$RUNNER_TEMP"/prod-*.plan.log | |
| rm -f "$RUNNER_TEMP"/prod-*.apply.log |