diff --git a/.github/workflows/destroy-prod.yml b/.github/workflows/destroy-prod.yml new file mode 100644 index 0000000..e0aa6fd --- /dev/null +++ b/.github/workflows/destroy-prod.yml @@ -0,0 +1,209 @@ +name: Destroy Infrastructure (PROD) + +# Destructive PROD operation: manual trigger only; must run from main. +on: + workflow_dispatch: + +concurrency: + group: infrastructure-deployment + cancel-in-progress: false + +permissions: + contents: read + +jobs: + destroy: + name: Destroy PROD + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + runs-on: [self-hosted, linux, assignment2] + timeout-minutes: 60 + + # Reuse the existing PROD environment's required reviewer approval and secrets. + environment: PROD + + env: + OS_AUTH_TYPE: v3applicationcredential + OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} + OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} + OS_IDENTITY_API_VERSION: "3" + OS_INTERFACE: public + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + KOPS_STATE_STORE: swift://demo-kops + TARGET_ENV: prod + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Setup OpenTofu + uses: opentofu/setup-opentofu@v2 + with: + tofu_version_file: .opentofu-version + tofu_wrapper: false + + - name: Verify tools and configuration + shell: bash + run: | + set -euo pipefail + + tofu version + command -v jq + + # Fail closed if the target is ever changed to a non-PROD environment. + test "$TARGET_ENV" = "prod" + + : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" + : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" + + test -f "environments/$TARGET_ENV/frontend/deployment.tfvars" + test -f "environments/$TARGET_ENV/network/deployment.tfvars" + + - name: Install kOps and kubectl + shell: bash + run: | + set -euo pipefail + + BIN_DIR="$RUNNER_TEMP/tofu-destroy-tools" + mkdir -p "$BIN_DIR" + + KUBECTL_VERSION="v1.36.4" + + curl -fsSL -o "$BIN_DIR/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + + curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + + ( + cd "$BIN_DIR" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + ) + + KOPS_VERSION="v1.36.2" + + curl -fsSL -o "$BIN_DIR/kops" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" + + curl -fsSL -o "$BIN_DIR/kops.sha256" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" + + ( + cd "$BIN_DIR" + echo "$(cat kops.sha256) kops" | sha256sum --check + ) + + chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" + echo "$BIN_DIR" >> "$GITHUB_PATH" + + - name: Export temporary Kubernetes kubeconfig + shell: bash + env: + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} + run: | + set -euo pipefail + + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps application credential secret}" + + KUBECONFIG_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + + umask 077 + + kops get cluster einalien.k8s.local \ + --state="$KOPS_STATE_STORE" > /dev/null + + kops export kubeconfig einalien.k8s.local \ + --admin=1h \ + --state="$KOPS_STATE_STORE" \ + --kubeconfig="$KUBECONFIG_FILE" + + test -s "$KUBECONFIG_FILE" + chmod 600 "$KUBECONFIG_FILE" + + echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + + - name: Check Kubernetes backend access + shell: bash + run: | + set -euo pipefail + + kubectl --request-timeout=15s get namespace iac + kubectl auth can-i get secrets -n iac --quiet + kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet + + - name: Destroy frontend then network + shell: bash + env: + TF_IN_AUTOMATION: "true" + TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} + TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} + run: | + set -euo pipefail + umask 077 + + : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR}" + : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME}" + + # Preserve reverse dependency order: frontend must be removed before network. + for STACK in frontend network; do + STACK_DIR="environments/$TARGET_ENV/$STACK" + PLAN_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.tfplan" + PLAN_JSON="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.json" + + # Fail if the backend is not the expected PROD Kubernetes state. + grep -Eq "secret_suffix[[:space:]]*=[[:space:]]*\"assignment2-$TARGET_ENV-$STACK\"" \ + "$STACK_DIR/providers.tf" + + echo "::group::Destroy $TARGET_ENV $STACK" + + # Initialize the existing Kubernetes backend. + tofu -chdir="$STACK_DIR" init \ + -reconfigure \ + -input=false \ + -lockfile=readonly \ + -no-color + + # Generate a destruction plan. + tofu -chdir="$STACK_DIR" plan \ + -destroy \ + -var-file="deployment.tfvars" \ + -out="$PLAN_FILE" \ + -lock-timeout=5m \ + -input=false \ + -no-color + + # Display resource addresses being deleted. + tofu -chdir="$STACK_DIR" show -json "$PLAN_FILE" > "$PLAN_JSON" + + jq -r ' + .resource_changes[]? + | select(.change.actions | index("delete")) + | "Destroy: \(.address)" + ' "$PLAN_JSON" + + # Apply the saved destruction plan. + tofu -chdir="$STACK_DIR" apply \ + -input=false \ + -no-color \ + -lock-timeout=5m \ + "$PLAN_FILE" + + rm -f "$PLAN_FILE" "$PLAN_JSON" + + echo "::endgroup::" + done + + - name: Clean up temporary files + if: always() + shell: bash + run: | + rm -f "$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.tfplan + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.json diff --git a/README.md b/README.md index 117b8c8..d93e362 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ - [Environment Configuration](#environment-configuration) 3. [CI/CD Workflow](#3-cicd-workflow) - [3.1 Developer Workflow and Pull Requests](#31-developer-workflow-and-pull-requests) + - [Protect main Branch Ruleset](#protect-main-branch-ruleset) - [3.2 Continuous Integration](#32-continuous-integration--githubworkflowsciyml) - [3.3 Deployment](#33-deployment--githubworkflowsdeployyml) - - [3.4 TEST destruction workflow](#34-test-destruction-workflow--githubworkflowsdestroyyml) + - [3.4 TEST and STAGE Destruction Workflow](#34-test-and-stage-destruction-workflow--githubworkflowsdestroyyml) 4. [Prerequisites and Credentials](#4-prerequisites-and-credentials) - [GitHub Actions Environments: Required Configuration](#github-actions-environments-required-configuration) 5. [Running the Workflow End to End](#5-running-the-workflow-end-to-end) @@ -22,9 +23,9 @@ - [5.2 Automatic TEST and STAGE Deployment](#52-automatic-test-and-stage-deployment) - [5.3 Manual Deployment through PROD](#53-manual-deployment-through-prod) - [5.4 Local Development, Validation, and Manual Deployment](#54-local-development-validation-and-manual-deployment) - - [5.5 TEST Destruction and Recreation](#55-test-destruction-and-recreation) + - [5.5 TEST and STAGE Destruction and Recreation](#55-test-and-stage-destruction-and-recreation) 6. [Destroying Resources and Cleanup](#6-destroying-resources-and-cleanup) - - [6.1 Automated Destruction of TEST](#61-automated-destruction-of-test) + - [6.1 Automated Destruction of TEST and STAGE](#61-automated-destruction-of-test-and-stage) - [6.2 Manually Destroy an Individual Frontend Stack](#62-manually-destroy-an-individual-frontend-stack) - [6.3 Manually Destroy All Stacks in One Environment](#63-manually-destroy-all-stacks-in-one-environment) - [6.4 Manually Destroy All Three Environments](#64-manually-destroy-all-three-environments) @@ -34,7 +35,7 @@ ## 1. Purpose and scope -This proof of concept addresses the problems described in the TerraNova scenario: unreviewed changes to `main`, inconsistent code quality, deployments from individual workstations, and configuration drift between Git and deployed infrastructure. Infrastructure is defined in version-controlled OpenTofu configurations and validated before changes are automatically deployed to TEST and STAGE. Production deployment requires a separate manual workflow run, with environment approval provided by GitHub environment protection rules **when those rules are configured**. +This proof of concept addresses the problems described in the TerraNova scenario: unreviewed changes to `main`, inconsistent code quality, deployments from individual workstations, and configuration drift between Git and deployed infrastructure. Infrastructure is defined in version-controlled OpenTofu configurations and validated before changes are automatically deployed to TEST and STAGE. Production deployment requires a separate manual workflow run and passes through a **configured human approval gate** in the `PROD` GitHub environment before the production job executes. The implementation provisions **two independently managed stacks per environment**: @@ -47,21 +48,23 @@ The backend and database *subnets* are created as part of the network topology; | Capability | Status | | --- | --- | -| CI formatting, validation, linting, and security scanning | Implemented | -| Automatic TEST deployment after successful CI on `main` | Implemented; successful run observed | -| Automatic STAGE deployment after TEST | Implemented; successful run observed | -| Manual PROD deployment after TEST and STAGE | Implemented; successful run reported | +| CI formatting, validation, linting, and security scanning | Implemented; CI passed | +| Automatic TEST deployment after successful CI on `main` | Implemented and successfully executed | +| Automatic STAGE deployment after TEST | Implemented and successfully executed | +| Manual PROD deployment after TEST and STAGE | Implemented and successfully executed | +| Human approval for PROD deployment | **Configured** through GitHub environment protection | +| `main` branch protection | **Configured:** `Protect main` ruleset; pull requests and passing CI required, 0 required approvals | | Two OpenTofu stacks in each environment | Implemented | | Environment- and stack-specific remote state | Implemented | | Reusable network and VM modules | Implemented (local modules) | -| GitHub Actions TEST destruction | **Implemented and successfully executed**; frontend before network | -| TEST recreation after complete destruction | **Implemented and successfully executed** via `Deploy Infrastructure` | -| STAGE/PROD destruction through GitHub Actions | **Not yet implemented**; manual CLI procedure documented below | -| Manual destruction using OpenTofu CLI | Documented for all three environments | -| Centralized OpenTofu version file | `.opentofu-version` added (1.12.6); used by `destroy.yml` | -| Independent, tagged releases of the reusable modules | **Not implemented**; modules are versioned with repository commits | +| GitHub Actions TEST and STAGE destruction | **Implemented and successfully executed** using a matrix; frontend before network | +| TEST and STAGE recreation after destruction | **Successfully tested** through `Deploy Infrastructure` | +| PROD destruction through GitHub Actions | **Not yet implemented**; manual CLI procedure documented below | +| Manual destruction using OpenTofu CLI | Documented for all three environments and for individual frontend stacks | +| Centralized CI tool versions | `.opentofu-version`, `.tflint-version`, `requirements-checkov.txt` | +| Independently tagged releases of reusable modules | **Not implemented**; modules are versioned with repository commits | -The TEST destroy/recreate functionality was added after the original November 1 submission; the current documented implementation includes that later change. The repository's workflow definitions alone do not prove that repository branch protection, required reviewers, instructor permissions, or runner availability have been enabled. These are administrative prerequisites described below. +The destruction and recreation workflows were developed after the original November 1 submission. This README describes the current repository state rather than claiming that post-submission changes were part of the original ZIP. Runner availability and instructor permissions still require verification through GitHub and OpenStack administration. ## 2. Architecture and repository structure @@ -70,7 +73,7 @@ assigment2-iac/ ├── .github/workflows/ │ ├── ci.yml # Formatting, validation, TFLint, Checkov │ ├── deploy.yml # TEST → STAGE → conditional PROD -│ ├── destroy.yml # Manually triggered TEST-only destruction +│ ├── destroy.yml # Manually triggered TEST + STAGE destruction │ └── runner-test.yml # Manually triggered runner health check ├── bootstrap/ │ ├── cloud_init/runner.yaml # Runner VM operating-system packages @@ -88,7 +91,9 @@ assigment2-iac/ ├── modules/ │ ├── network/ # Reusable OpenStack network resources │ └── vm/ # Reusable VM, SG, IP, optional volume attach -├── .opentofu-version # Version used by destroy workflow (1.12.6) +├── .opentofu-version # Pinned OpenTofu version (1.12.6) +├── .tflint-version # Pinned TFLint version +├── requirements-checkov.txt # Pinned Checkov dependency ├── .tflint.hcl ├── .gitignore └── README.md @@ -119,20 +124,39 @@ The `bootstrap/` root module creates the dedicated self-hosted runner VM and its 5. Open a pull request targeting `main`. Review the OpenTofu changes and verify the CI job passes before merging. 6. Merge the reviewed change into `main`. The successful push CI run triggers the deployment workflow. -Recommended repository settings: protect `main`, require pull requests, require successful CI checks, restrict direct pushes, and require review by another collaborator. **These policies must be verified in repository settings; they are not enforced by the YAML files themselves.** +#### Protect main branch ruleset + +The repository has an active GitHub branch ruleset named **`Protect main`**. It enforces pull request-based changes and successful CI checks before merging into `main`. + +| Rule | Configuration | +| --- | --- | +| Ruleset | `Protect main` | +| Enforcement status | Active | +| Target branch | `main` | +| Require a pull request before merging | Enabled | +| Required approvals | **0** | +| Require status checks to pass | Enabled | +| Required status check | `Format and validate infrastructure` | +| Block force pushes | Enabled | +| Restrict deletions | Enabled | +| Require deployments before merging | Disabled | + +Zero required approvals allows contributions to be merged without another reviewer, which is practical for this individually developed assignment, but **does not enforce independent code review**. Pull requests and the required CI check still provide a controlled path into `main`. These repository rules are configured in GitHub settings rather than in the workflow YAML files. + +This branch ruleset is separate from the **human approval gate for PROD deployment**, which is configured through GitHub environment protection rules. ### 3.2 Continuous integration — `.github/workflows/ci.yml` CI is triggered on **pull requests targeting `main`**, **pushes to `main`**, or manual **Run workflow**. It runs on the self-hosted runner labeled `[self-hosted, linux, assignment2]` and performs: 1. Checkout with `persist-credentials: false`. -2. Install OpenTofu **1.12.6**. -3. `tofu fmt -check -recursive` for consistent formatting. -4. `tofu init -backend=false -input=false -lockfile=readonly` and `tofu validate -no-color` for the bootstrap and all six environment stacks. Using `-backend=false` avoids accessing deployment state during static validation. -5. TFLint **v0.64.0** on the repository's OpenTofu configurations. -6. Checkov **3.4.0**, scanning both Terraform/OpenTofu and GitHub Actions definitions. +2. Install the version of OpenTofu specified in **`.opentofu-version`** (currently `1.12.6`). +3. Execute `tofu fmt -check -recursive` for consistent formatting. +4. Run `tofu init -backend=false -input=false -lockfile=readonly` and `tofu validate -no-color` for the bootstrap and all six environment stacks. Using `-backend=false` avoids accessing deployment state during static validation. +5. Install the TFLint version specified in **`.tflint-version`** and lint OpenTofu configurations using `.tflint.hcl`. +6. Create an isolated Python virtual environment and install the pinned Checkov dependency from **`requirements-checkov.txt`**; scan Terraform/OpenTofu and GitHub Actions definitions. -A failed check causes CI to fail. Only a successful **push-originated** CI run on `main` meets the automatic deployment workflow's trigger condition; a pull-request CI run does not deploy infrastructure. +The tool-version files are tracked in Git. They make deliberate upgrades easy to review, but do **not** automatically update themselves. A failed check causes CI to fail. Only a successful **push-originated** CI run on `main` meets the automatic deployment workflow's trigger condition; a pull-request CI run does not deploy infrastructure. ### 3.3 Deployment — `.github/workflows/deploy.yml` @@ -153,26 +177,38 @@ Manual "Deploy Infrastructure" run from main │ └──> TEST ──> STAGE ──> [PROD environment approval*] ──> PROD -* Approval is required only if configured in the GitHub PROD environment. +* The PROD GitHub environment currently has human approval enabled. ``` `deploy.yml` is triggered by a completed CI workflow on `main` or by `workflow_dispatch`. The TEST job checks that CI succeeded and was triggered by a push to `main`, or that the deployment was manually dispatched from `main`. STAGE has `needs: deploy-test`, so it runs only after TEST succeeds. PROD has `needs: deploy-stage` and an additional condition requiring a **manual `workflow_dispatch` from `main`**. There is no `deploy_prod` checkbox in this version: **every manual run of Deploy Infrastructure from `main` attempts PROD after TEST and STAGE**. For each environment, the deployment job checks OpenStack credentials, installs checksum-verified `kubectl` and `kOps`, obtains a short-lived **1-hour Kubernetes admin kubeconfig**, verifies access to the `iac` namespace, initializes the backends, then processes **network before frontend**. OpenTofu saves each plan and inspects the plan JSON with `jq`. The pipeline only allows `no-op`, `read`, `create`, and `update` resource actions; it rejects planned deletions and replacements. It then applies the exact saved plan, or skips application when there are no changes. Temporary plans, logs, and kubeconfig files are removed in cleanup steps. -The TEST `Initialize TEST remote state` step no longer requires pre-existing managed resources. It initializes the network and frontend Kubernetes backends even when the state is empty, allowing TEST to be provisioned initially or recreated following a complete destroy. **The TEST destroy → recreate sequence has been successfully tested through GitHub Actions.** STAGE and PROD already supported initialization from empty state. +The TEST `Initialize TEST remote state` step no longer requires pre-existing managed resources. It initializes the network and frontend Kubernetes backends even when the state is empty, allowing TEST to be provisioned initially or recreated following a complete destroy. **Both TEST and STAGE destroy → recreate sequences have now been successfully tested through GitHub Actions.** PROD already supports initialization from empty state, but its destruction/recreation cycle has not yet been tested. + +### 3.4 TEST and STAGE destruction workflow — `.github/workflows/destroy.yml` -### 3.4 TEST destruction workflow — `.github/workflows/destroy.yml` +The separate **Destroy Infrastructure (TEST, STAGE)** workflow follows the Lab 7 pattern and is **manually triggered** with `workflow_dispatch` from `main`. A GitHub Actions **matrix** creates one job for each of these environment mappings: -The separate **Destroy Infrastructure (TEST)** workflow follows the Lab 7 pattern and is **manually triggered** using `workflow_dispatch` from `main`. It is intentionally restricted to TEST: the job uses `environment: TEST`, sets `TARGET_ENV: test`. This avoids accidentally targeting STAGE or PROD during the initial destruction test. +| GitHub environment | Configuration path | State suffixes | +| --- | --- | --- | +| `TEST` | `environments/test/` | `assignment2-test-frontend`, `assignment2-test-network` | +| `STAGE` | `environments/stage/` | `assignment2-stage-frontend`, `assignment2-stage-network` | + +The matrix currently **does not include PROD**. Each job selects the corresponding GitHub environment for its credentials, verifies its configuration and backend state suffixes, and operates only on that environment's two stacks. + +The workflow runs on the existing self-hosted runner. It installs OpenTofu from `.opentofu-version`, installs checksum-verified `kubectl` and kOps, exports a temporary one-hour Kubernetes admin kubeconfig, and checks access to the `iac` namespace. For **each environment**, it performs: -The workflow runs on the same self-hosted runner and uses the same Kubernetes state backend and TEST-scoped OpenStack credentials as the deployment workflow. It uses `opentofu/setup-opentofu@v2` with `tofu_version_file: .opentofu-version` (currently **1.12.6**). After installing and verifying `kubectl` and kOps, it exports a temporary, one-hour Kubernetes kubeconfig and checks access to the `iac` namespace. +```text +Initialize frontend backend → plan -destroy → apply saved plan +Initialize network backend → plan -destroy → apply saved plan +``` -For each stack, it initializes the existing backend, produces a saved `tofu plan -destroy`, displays resource addresses selected for deletion (without printing attribute values), and applies that saved plan. The order is **frontend → network**, respecting the dependency between the stacks. It checks the expected `assignment2-test-frontend` and `assignment2-test-network` state suffixes and removes temporary credentials and plans on completion. +Planned resource addresses are displayed without dumping configuration attributes. Temporary kubeconfig, plan, and JSON files are cleaned up. Both deployment and destruction workflows use the `infrastructure-deployment` concurrency group to avoid overlapping infrastructure changes. -Both deployment and destruction workflows share the `infrastructure-deployment` concurrency group, preventing overlapping runs of these workflows. **TEST destruction has been successfully executed**, and TEST has subsequently been recreated using the deployment pipeline. +The matrix uses `max-parallel: 1`, so only one environment job executes at a time, but **GitHub does not guarantee TEST executes before STAGE**. With `fail-fast: false`, a failure in one environment does not automatically cancel the other. **A single manual run attempts destruction of both TEST and STAGE**, not an operator-selected subset. Once triggered (and after any configured environment protection checks), the destruction is noninteractive and applies saved destroy plans. -The trigger is manual, but the actual destruction steps are automated after the workflow starts. If human authorization is required before deletion, GitHub environment protection must be configured separately. Adding required reviewers to the existing `TEST` environment would also affect ordinary automatic TEST deployments, so a dedicated protected destruction environment would be preferable for an additional approval gate. +**Verified:** The matrix-based workflow successfully destroyed the TEST and STAGE infrastructure, and the subsequent deployment jobs successfully recreated both environments. PROD infrastructure was outside the matrix and was not targeted. The current approach reduces duplicated YAML; independent environment triggers would provide finer-grained destructive-operation control in a future iteration. ## 4. Prerequisites and credentials @@ -184,7 +220,7 @@ The workflow assumes the following pre-existing infrastructure and permissions: - Kubernetes namespace **`iac`**, plus permission to read/update Secrets and use Lease objects needed for backend locking. - A running, registered GitHub Actions self-hosted Linux runner labeled `self-hosted`, `linux`, `assignment2`, with outbound access to the OpenStack APIs, the cluster, GitHub Actions, and the relevant download endpoints. - `jq`, Git, Python 3 with venv, curl, and the libraries required by the runner. `bootstrap/cloud_init/runner.yaml` installs these OS packages, **but does not register the GitHub runner agent**. Agent registration and long-running service setup must be completed separately. -- In GitHub's repository settings, environment names **TEST**, **STAGE**, and **PROD**, with appropriate environment-specific credentials. Configure **required reviewers for PROD** if a human approval gate is required; the YAML's manual trigger alone is not a reviewer approval. The TEST destruction workflow also uses the existing `TEST` environment; adding reviewer protection there would affect automatic TEST deployment. +- In GitHub's repository settings, environment names **TEST**, **STAGE**, and **PROD**, with appropriate environment-specific credentials. **Required human review is configured for PROD deployment**; the YAML's manual trigger alone would not constitute reviewer approval. Destruction currently uses the existing TEST/STAGE environments; adding reviewer protection to either would also affect automatic deployments to that environment. ### GitHub Actions environments: required configuration @@ -229,7 +265,7 @@ The bootstrap configuration for the runner is in `bootstrap/`. An administrator 1. Open the repository on `git.ntnu.no` and select **Actions → Deploy Infrastructure**. 2. Click **Run workflow** and select **`main`**. This manually invokes the pipeline even if no new commit has been pushed. 3. Wait for **Deploy TEST** and **Deploy STAGE** to succeed. -4. If the PROD environment has required reviewers, an authorized reviewer selects **Review deployments → PROD → Approve and deploy**. +4. The configured PROD approval gate pauses the production job. An authorized reviewer selects **Review deployments → PROD → Approve and deploy**. 5. Verify **Deploy PROD** succeeds, and inspect the created resources and floating IP in OpenStack. Do **not** select **Re-run all jobs** on an automatically triggered run to deploy PROD: it retains the original event and the PROD condition remains false. Start a **new manual workflow run** instead. @@ -295,31 +331,33 @@ done OpenTofu prompts for confirmation of each apply. Choose the appropriate credentials **before** running this sequence, especially if the environments use different OpenStack application credentials. Local plans may include destructive changes; unlike `deploy.yml`, this command sequence does not implement the CI pipeline's JSON action filter. Review each plan carefully before approval. -### 5.5 TEST destruction and recreation +### 5.5 TEST and STAGE destruction and recreation -A complete TEST lifecycle has been verified: **deploy → destroy → recreate**. From the repository's GitHub Actions page: +The complete **deploy → destroy → recreate** lifecycle has been successfully verified for **both TEST and STAGE** through GitHub Actions. -1. Confirm you are using the current code on `main`, including the updated TEST backend initialization. -2. Open **Actions → Destroy Infrastructure (TEST) → Run workflow**, select `main`, and start the run. This removes the **TEST frontend** first, then the **TEST network**. Confirm that the job succeeds and the targeted resources are absent from OpenStack. -3. Recreate TEST by re-running a successful **CI-triggered** `Deploy Infrastructure` run that used the updated `main` commit (select **Re-run all jobs**). It deploys TEST network and frontend, then processes STAGE; PROD remains skipped because the original event was `workflow_run`, not `workflow_dispatch`. -4. Check that TEST resources exist again and that STAGE and PROD were not destroyed. +1. Confirm that the updated `deploy.yml` and `destroy.yml` are merged to `main`, CI has passed, and an automatic deployment from the updated `main` revision has succeeded. Capture deployment evidence before deletion. +2. Open **Actions → Destroy Infrastructure (TEST, STAGE) → Run workflow**, select `main`, and trigger the run. This starts **both** matrix jobs; it does not allow selection of only one environment. +3. Verify that **Destroy TEST** and **Destroy STAGE** succeed. Within each environment, the frontend is destroyed **before** its network. Confirm the corresponding resources are absent from OpenStack and PROD remains intact. +4. Recreate both environments by opening a successful **CI-triggered** `Deploy Infrastructure` run created after the updated code reached `main`, then selecting **Re-run all jobs**. It recreates TEST network/frontend, then STAGE network/frontend. PROD is skipped because the original event is `workflow_run`, not `workflow_dispatch`. +5. Confirm that all intended resources exist again and capture screenshots of the successful destruction and recreation runs. -**Do not use a new manual `Run workflow` of `Deploy Infrastructure` solely to recreate TEST** unless you intend to run PROD as well: a manually dispatched deployment from `main` also attempts PROD after TEST and STAGE. +**Do not start a new manual `Run workflow` of `Deploy Infrastructure` solely to recreate TEST/STAGE** unless you also intend to attempt PROD: a manual deployment run from `main` executes TEST → STAGE → the protected PROD job. The TEST and STAGE lifecycle checks reported success; this is not yet evidence that PROD destruction has been tested. ## 6. Destroying resources and cleanup -**Current status:** `.github/workflows/destroy.yml` provides an **implemented and successfully tested TEST-only destruction workflow**. The normal `deploy.yml` still rejects deletions and replacements. **STAGE and PROD destruction through GitHub Actions is not yet implemented**; the CLI procedures below cover manual cleanup for those environments. +**Current status:** `.github/workflows/destroy.yml` provides an **implemented and successfully tested TEST + STAGE matrix destruction workflow**. Both environments were subsequently recreated successfully. Normal `deploy.yml` still rejects deletions and replacements. **PROD destruction through GitHub Actions is not yet implemented**; the CLI procedures below also cover manually selected stack/environment cleanup. **Warning:** Destruction deletes real OpenStack resources. Obtain authorization, capture the required deployment evidence, protect all remote state, and confirm the target environment before proceeding. Never destroy `bootstrap/` while it hosts the required runner. Never destroy the external kOps cluster because the six stack states depend on it. -### 6.1 Automated destruction of TEST +### 6.1 Automated destruction of TEST and STAGE -1. Open **Actions → Destroy Infrastructure (TEST)** in the repository. -2. Choose **Run workflow**, select **`main`**, and confirm that TEST is the intended target. -3. Start the workflow. It uses the TEST GitHub environment credentials, initializes each Kubernetes remote state, plans and destroys the **frontend** stack, and then plans and destroys the **network** stack. -4. Verify the run succeeds and that the corresponding TEST OpenStack resources were removed. The runner VM and kOps cluster are **not** part of this workflow. +1. Open **Actions → Destroy Infrastructure (TEST, STAGE)** in the repository. +2. Choose **Run workflow**, select **`main`**, and confirm you intend to remove infrastructure in **both TEST and STAGE**. +3. Start the workflow. Each matrix job uses its matching GitHub environment credentials and Kubernetes remote-state suffixes. +4. Within **each** environment, the workflow initializes state, creates and applies a saved destroy plan for **frontend**, and then repeats the procedure for **network**. +5. Verify that **Destroy TEST** and **Destroy STAGE** succeed and their resources have been removed. The runner VM, external kOps cluster, and PROD resources are **not** part of this workflow. -The workflow does **not** offer a stack selector or operate on STAGE/PROD. It applies saved destroy plans noninteractively after a manual trigger; it does not pause for a separate plan approval unless an environment approval gate has been configured. +**Scope and safety:** There is no environment or stack selector in the current GitHub Actions destroy workflow. `max-parallel: 1` serializes the matrix jobs but does not guarantee which environment executes first; `fail-fast: false` means one job's failure does not stop the other from being attempted. These jobs apply destroy plans automatically once dispatched. For an individual stack or a single environment, use the carefully reviewed CLI procedures below or introduce an explicitly scoped workflow entry point. The human approval gate for **deploying PROD** is separate from, and does not authorize, any future PROD destruction workflow. ### 6.2 Manually destroy an individual frontend stack @@ -355,41 +393,47 @@ done Run the **section 6.3 sequence separately for PROD, STAGE, and TEST**, in that order. **Switch to the correct OpenStack deployment credentials before each environment**. This deliberate, environment-by-environment process avoids accidentally using PROD permissions or configuration for another environment. Collect a successful OpenTofu destroy log and resource inventory screenshot for each environment if destruction evidence is required. -The updated `deploy-test` job initializes an empty TEST remote state successfully. The TEST workflow has been used to recreate the network and frontend following successful automated destruction. See section 5.5 for the verified TEST lifecycle procedure. +The updated deployment jobs initialize empty TEST and STAGE remote state successfully. Both environments were recreated after their matrix-based destruction. See section 5.5 for the verified lifecycle procedure. PROD destruction must be separately authorized and tested before being added to the automated workflow. ## 7. Security, state management, and design decisions -- **Git as the intended source of truth:** changes are reviewed and merged before automatic application, reducing configuration drift and unreviewed local changes. -- **Environment isolation:** TEST/STAGE/PROD have separate configuration folders, CIDRs, GitHub Environments, and Kubernetes state keys. Frontend reads only the corresponding network state. +- **Git as the intended source of truth:** changes are proposed through pull requests and merged before automatic application, reducing configuration drift and unreviewed local changes. The active `Protect main` ruleset requires pull requests and successful `Format and validate infrastructure` CI checks, blocks force pushes and branch deletion, and permits merging with 0 required approvals. Human approval for PROD deployment is enforced separately by GitHub environment protection. +- **Environment isolation:** TEST/STAGE/PROD have separate configuration folders, CIDRs, GitHub environments, and Kubernetes state keys. Frontend reads only the corresponding network state. - **State and locking:** the Kubernetes backend stores OpenTofu state in namespace `iac`. The pipeline checks Kubernetes API and Lease permissions before deploying. Protect access to the cluster and its Secrets, as state may include sensitive data. -- **Limited authentication exposure:** OpenStack uses application credentials and GitHub environment secrets; kOps generates a short-lived kubeconfig. The deployment workflow removes temporary kubeconfig, plans, and log files in cleanup steps. -- **Static quality gates:** formatting, validation, TFLint, and Checkov execute before automatic deployment. Provider selections are constrained by `.terraform.lock.hcl` and version requirements. The TEST destruction workflow uses `.opentofu-version` to select OpenTofu 1.12.6. -- **Conservative deployment and controlled destruction:** normal deployments reject destructive plan actions. TEST destruction is isolated in a separate manually triggered workflow, uses saved destroy plans and the TEST backend, and shares the deployment concurrency group. STAGE and PROD destruction remains manual via OpenTofu CLI. +- **Limited authentication exposure:** OpenStack uses application credentials and GitHub environment secrets; kOps generates short-lived kubeconfig files. Deployment and destruction workflows remove temporary kubeconfig and plan files during cleanup. +- **Static quality gates and pinned tools:** formatting, validation, TFLint, and Checkov execute before automatic deployment. `.opentofu-version`, `.tflint-version`, and `requirements-checkov.txt` centralize CI tool version selection; `.terraform.lock.hcl` pins provider selections. Upgrades should be reviewed and validated in CI. +- **Non-destructive deployment:** normal deployments reject planned deletion and replacement actions and apply only the saved, checked plan. +- **Controlled destruction:** a separate manually triggered workflow uses a matrix for TEST and STAGE. Each job verifies the matching Kubernetes backend, uses environment-scoped credentials, destroys frontend before network, and applies saved destroy plans. Jobs are serialized by `max-parallel: 1` but have no guaranteed environment order. One trigger affects both TEST and STAGE; independent triggers are an improvement for selective cleanup. +- **PROD authorization:** the production deployment job is manually triggered and protected by configured human reviewer approval. **PROD destruction is not yet implemented** and should receive its own explicit authorization controls before use. - **Network exposure:** frontend permits HTTP on port 80 from the configured `http_cidr` (currently `0.0.0.0/0`), and SSH on port 22 only from `admin_cidr`. This is a demonstration Nginx service, not a hardened application delivery platform. -- **Reuse and versioning:** `modules/network` and `modules/vm` are reused across environments via relative source paths. Module code is therefore versioned at the **repository commit level**, with the same module revision used by each environment at a given commit. There is **no independent module release/tag or registry version pinning**. Provider versions are constrained in code and resolved in committed lock files. +- **Reuse and versioning:** `modules/network` and `modules/vm` are reused across environments via relative source paths. Module code is versioned at the **repository commit level**, with the same module revision used by each environment at a given commit. There is **no independent module release/tag or registry version pinning**. -For reuse across multiple customers, a future iteration could move the modules into independently tagged releases, adopt reusable deployment workflows with customer-specific configuration, use separate customer state backends and accounts, add policy-as-code checks, and extend approved automated destruction to STAGE and PROD. +For reuse across customers, a future iteration could publish independently tagged module releases, adopt reusable deployment/destruction workflows with explicitly scoped entry points, use separate customer state backends and accounts, and add policy-as-code and approved PROD cleanup. ## 8. Verification and evidence -Verify a successful end-to-end deployment in **Actions → Deploy Infrastructure**: +The following workflow outcomes have been reported as successful and can be reviewed in **GitHub Actions**: -- Automatic CI-triggered run: **TEST succeeded**, **STAGE succeeded**, **PROD skipped** (expected). -- Manually dispatched run from `main`: **TEST succeeded**, **STAGE succeeded**, **PROD succeeded**. -- OpenStack dashboard: expected private networks, subnets, routers, and at least one frontend VM per environment. -- TEST destroy run: **Destroy Infrastructure (TEST) succeeded**, removing the frontend and network stacks. -- TEST recreation run: **Deploy TEST succeeded after the destroy**, recreating the network and frontend from empty state; STAGE succeeded and PROD was skipped on the CI-triggered rerun. -- Optional workload test: identify a frontend floating IP from the OpenTofu outputs or OpenStack dashboard and verify the default Nginx HTTP page is reachable on port 80. +| Workflow / operation | Verified outcome | +| --- | --- | +| CI checks | Passed | +| Automatic deployment on `main` | TEST and STAGE succeeded; PROD skipped as expected | +| Manually approved production deployment | TEST, STAGE, and PROD succeeded | +| Earlier TEST-only destroy/recreate test | Destroy TEST succeeded; Deploy TEST recreated both stacks | +| Matrix-based destruction | **Destroy TEST and Destroy STAGE succeeded** | +| Deployment rerun after matrix destruction | **TEST and STAGE recreated successfully**; PROD skipped | +| PROD destruction and recreation | **Not yet tested** | A screenshot of a full successful run end to end with destruction will be added once STAGE and PROD destruction has been completed. +For an optional workload check, identifying a frontend floating IP from OpenTofu outputs or OpenStack and verify the Nginx default page responds over HTTP port 80 will be given. The assignment permits application-level testing to be omitted. ## 9. Known limitations and improvements -1. **GitHub Actions destruction currently covers TEST only:** STAGE/PROD destruction is documented as a manual CLI process and remains to be implemented and verified in GitHub Actions. -2. **TEST lifecycle verified:** TEST successfully completed deployment, destruction, and recreation; STAGE and PROD destruction/recreation have not yet been demonstrated. -3. **PROD approval depends on GitHub settings:** manual triggering is implemented in YAML, but required reviewers must be enabled administratively. -4. **Module versions are tied to the repository commit:** independently published and pinned module versions would improve customer reuse and rollback. -5. **Runner and kOps dependencies are external:** the self-hosted runner agent registration, continuous availability, kOps cluster, Kubernetes namespace, access control, and instructor permissions require separate setup and maintenance. +1. **PROD destruction is not yet implemented:** the current GitHub Actions destruction matrix covers TEST and STAGE only. Before including PROD, add an explicitly authorized destruction path, verify state isolation, and test deletion followed by recreation. +2. **Matrix scope:** one manual `Destroy Infrastructure` run targets **both TEST and STAGE**; selecting an individual environment or stack is not supported through this workflow. Manual OpenTofu CLI recipes provide targeted cleanup in the meantime. +3. **Matrix sequencing:** `max-parallel: 1` prevents concurrent destruction jobs but does not guarantee TEST-before-STAGE ordering; `fail-fast: false` permits another matrix job to proceed if one fails. +4. **Module versions are tied to the repository commit:** separately published and pinned module releases would improve customer reuse and rollback. +5. **Runner and kOps dependencies are external:** self-hosted runner service availability, kOps cluster, Kubernetes namespace, access control, and instructor permissions require separate setup and ongoing maintenance. 6. **Bootstrap state is not remotely configured in the provided code:** its state needs separate backup and lifecycle protection. 7. **CI/CD tests are infrastructure-focused:** end-to-end application or Nginx availability tests are not implemented; the assignment permits workload-specific tests to be omitted.