diff --git a/.github/workflows/destroy-stage.yml b/.github/workflows/destroy-stage.yml new file mode 100644 index 0000000..b173bbb --- /dev/null +++ b/.github/workflows/destroy-stage.yml @@ -0,0 +1,208 @@ +name: Destroy Infrastructure (STAGE) + +# Destructive operation: manually triggered on main; targets only STAGE. +on: + workflow_dispatch: + +concurrency: + group: infrastructure-deployment + cancel-in-progress: false + +permissions: + contents: read + +jobs: + destroy: + name: Destroy STAGE + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + runs-on: [self-hosted, linux, assignment2] + timeout-minutes: 60 + + # Use STAGE-scoped credentials and any configured environment protections. + environment: STAGE + + env: + OS_AUTH_TYPE: v3applicationcredential + OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} + OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} + OS_IDENTITY_API_VERSION: "3" + OS_INTERFACE: public + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + KOPS_STATE_STORE: swift://demo-kops + TARGET_ENV: stage + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Setup OpenTofu + uses: opentofu/setup-opentofu@v2 + with: + tofu_version_file: .opentofu-version + tofu_wrapper: false + + - name: Verify tools and configuration + shell: bash + run: | + set -euo pipefail + + tofu version + command -v jq + + # Fail closed if the target is ever changed. + test "$TARGET_ENV" = "stage" + + : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" + : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" + + test -f "environments/$TARGET_ENV/frontend/deployment.tfvars" + test -f "environments/$TARGET_ENV/network/deployment.tfvars" + + - name: Install kOps and kubectl + shell: bash + run: | + set -euo pipefail + + BIN_DIR="$RUNNER_TEMP/tofu-destroy-tools" + mkdir -p "$BIN_DIR" + + KUBECTL_VERSION="v1.36.4" + + curl -fsSL -o "$BIN_DIR/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + + curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + + ( + cd "$BIN_DIR" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + ) + + KOPS_VERSION="v1.36.2" + + curl -fsSL -o "$BIN_DIR/kops" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" + + curl -fsSL -o "$BIN_DIR/kops.sha256" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" + + ( + cd "$BIN_DIR" + echo "$(cat kops.sha256) kops" | sha256sum --check + ) + + chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" + echo "$BIN_DIR" >> "$GITHUB_PATH" + + - name: Export temporary Kubernetes kubeconfig + shell: bash + env: + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} + run: | + set -euo pipefail + + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps application credential secret}" + + KUBECONFIG_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + + umask 077 + + kops get cluster einalien.k8s.local \ + --state="$KOPS_STATE_STORE" > /dev/null + + kops export kubeconfig einalien.k8s.local \ + --admin=1h \ + --state="$KOPS_STATE_STORE" \ + --kubeconfig="$KUBECONFIG_FILE" + + test -s "$KUBECONFIG_FILE" + chmod 600 "$KUBECONFIG_FILE" + + echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + + - name: Check Kubernetes backend access + shell: bash + run: | + set -euo pipefail + + kubectl --request-timeout=15s get namespace iac + kubectl auth can-i get secrets -n iac --quiet + kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet + + - name: Destroy frontend then network + shell: bash + env: + TF_IN_AUTOMATION: "true" + TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} + TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} + run: | + set -euo pipefail + umask 077 + + : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR}" + : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME}" + + for STACK in frontend network; do + STACK_DIR="environments/$TARGET_ENV/$STACK" + PLAN_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.tfplan" + PLAN_JSON="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.json" + + # Verify that the backend belongs to the selected environment. + grep -Eq "secret_suffix[[:space:]]*=[[:space:]]*\"assignment2-$TARGET_ENV-$STACK\"" \ + "$STACK_DIR/providers.tf" + + echo "::group::Destroy $TARGET_ENV $STACK" + + # Initialize the existing Kubernetes backend. + tofu -chdir="$STACK_DIR" init \ + -reconfigure \ + -input=false \ + -lockfile=readonly \ + -no-color + + # Generate a destruction plan. + tofu -chdir="$STACK_DIR" plan \ + -destroy \ + -var-file="deployment.tfvars" \ + -out="$PLAN_FILE" \ + -lock-timeout=5m \ + -input=false \ + -no-color + + # Display resource addresses being deleted. + tofu -chdir="$STACK_DIR" show -json "$PLAN_FILE" > "$PLAN_JSON" + + jq -r ' + .resource_changes[]? + | select(.change.actions | index("delete")) + | "Destroy: \(.address)" + ' "$PLAN_JSON" + + # Apply the saved destruction plan. + tofu -chdir="$STACK_DIR" apply \ + -input=false \ + -no-color \ + -lock-timeout=5m \ + "$PLAN_FILE" + + rm -f "$PLAN_FILE" "$PLAN_JSON" + + echo "::endgroup::" + done + + - name: Clean up temporary files + if: always() + shell: bash + run: | + rm -f "$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.tfplan + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.json diff --git a/.github/workflows/destroy-test.yml b/.github/workflows/destroy-test.yml new file mode 100644 index 0000000..e5249d1 --- /dev/null +++ b/.github/workflows/destroy-test.yml @@ -0,0 +1,168 @@ +name: Destroy Infrastructure (TEST) + +on: + workflow_dispatch: + +concurrency: + group: infrastructure-deployment + cancel-in-progress: false + +permissions: + contents: read + +jobs: + destroy: + name: Destroy TEST + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + runs-on: [self-hosted, linux, assignment2] + timeout-minutes: 60 + + environment: TEST + + env: + OS_AUTH_TYPE: v3applicationcredential + OS_AUTH_URL: ${{ vars.OS_AUTH_URL }} + OS_REGION_NAME: ${{ vars.OS_REGION_NAME }} + OS_IDENTITY_API_VERSION: "3" + OS_INTERFACE: public + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + KOPS_STATE_STORE: swift://demo-kops + TARGET_ENV: test + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + # Use the same OpenTofu version as the CI workflow. + - name: Setup OpenTofu + uses: opentofu/setup-opentofu@v2 + with: + tofu_version_file: .opentofu-version + tofu_wrapper: false + + - name: Verify tools and configuration + shell: bash + run: | + set -euo pipefail + tofu version + command -v jq + test "$TARGET_ENV" = "test" + : "${OS_AUTH_URL:?Missing OS_AUTH_URL}" + : "${OS_REGION_NAME:?Missing OS_REGION_NAME}" + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing application credential secret}" + test -f "environments/$TARGET_ENV/frontend/deployment.tfvars" + test -f "environments/$TARGET_ENV/network/deployment.tfvars" + + # Reuse the pinned, checksum-verified installation approach from deploy.yml. + - name: Install kOps and kubectl + shell: bash + run: | + set -euo pipefail + BIN_DIR="$RUNNER_TEMP/tofu-destroy-tools" + mkdir -p "$BIN_DIR" + + KUBECTL_VERSION="v1.36.4" + curl -fsSL -o "$BIN_DIR/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl -fsSL -o "$BIN_DIR/kubectl.sha256" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + (cd "$BIN_DIR" && echo "$(cat kubectl.sha256) kubectl" | sha256sum --check) + + KOPS_VERSION="v1.36.2" + curl -fsSL -o "$BIN_DIR/kops" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64" + curl -fsSL -o "$BIN_DIR/kops.sha256" \ + "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-linux-amd64.sha256" + (cd "$BIN_DIR" && echo "$(cat kops.sha256) kops" | sha256sum --check) + + chmod +x "$BIN_DIR/kubectl" "$BIN_DIR/kops" + echo "$BIN_DIR" >> "$GITHUB_PATH" + + - name: Export temporary Kubernetes kubeconfig + shell: bash + env: + # kOps state in Swift uses different credentials from resource deletion. + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.KOPS_APPLICATION_CREDENTIAL_SECRET }} + run: | + set -euo pipefail + : "${OS_APPLICATION_CREDENTIAL_ID:?Missing kOps application credential ID}" + : "${OS_APPLICATION_CREDENTIAL_SECRET:?Missing kOps application credential secret}" + + KUBECONFIG_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + umask 077 + kops get cluster einalien.k8s.local --state="$KOPS_STATE_STORE" > /dev/null + kops export kubeconfig einalien.k8s.local \ + --admin=1h \ + --state="$KOPS_STATE_STORE" \ + --kubeconfig="$KUBECONFIG_FILE" + + test -s "$KUBECONFIG_FILE" + chmod 600 "$KUBECONFIG_FILE" + echo "KUBECONFIG=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + echo "KUBE_CONFIG_PATH=$KUBECONFIG_FILE" >> "$GITHUB_ENV" + + - name: Check Kubernetes backend access + shell: bash + run: | + set -euo pipefail + kubectl --request-timeout=15s get namespace iac + kubectl auth can-i get secrets -n iac --quiet + kubectl auth can-i create leases.coordination.k8s.io -n iac --quiet + + # A saved destroy plan is more auditable than direct destroy -auto-approve. + - name: Destroy frontend then network + shell: bash + env: + TF_IN_AUTOMATION: "true" + TF_VAR_admin_cidr: ${{ secrets.ADMIN_CIDR }} + TF_VAR_ssh_key_name: ${{ vars.SSH_KEY_NAME }} + run: | + set -euo pipefail + umask 077 + : "${TF_VAR_admin_cidr:?Missing ADMIN_CIDR}" + : "${TF_VAR_ssh_key_name:?Missing SSH_KEY_NAME}" + + for STACK in frontend network; do + STACK_DIR="environments/$TARGET_ENV/$STACK" + PLAN_FILE="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.tfplan" + PLAN_JSON="$RUNNER_TEMP/destroy-$TARGET_ENV-$STACK.json" + + # Fail closed if a stack's backend points outside TEST. + grep -Eq "secret_suffix[[:space:]]*=[[:space:]]*\"assignment2-test-$STACK\"" \ + "$STACK_DIR/providers.tf" + + echo "::group::Destroy $TARGET_ENV $STACK" + tofu -chdir="$STACK_DIR" init \ + -reconfigure -input=false -lockfile=readonly -no-color + + tofu -chdir="$STACK_DIR" plan \ + -destroy \ + -var-file="deployment.tfvars" \ + -out="$PLAN_FILE" \ + -lock-timeout=5m \ + -input=false -no-color + + # Show only resource addresses/actions, not sensitive attribute values. + tofu -chdir="$STACK_DIR" show -json "$PLAN_FILE" > "$PLAN_JSON" + jq -r '.resource_changes[]? | select(.change.actions | index("delete")) | "Destroy: \(.address)"' "$PLAN_JSON" + + # Applying the saved plan is noninteractive, just like -auto-approve. + tofu -chdir="$STACK_DIR" apply \ + -input=false -no-color -lock-timeout=5m "$PLAN_FILE" + + rm -f "$PLAN_FILE" "$PLAN_JSON" + echo "::endgroup::" + done + + - name: Clean up temporary files + if: always() + shell: bash + run: | + rm -f "$RUNNER_TEMP/destroy-$TARGET_ENV-kubeconfig" + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.tfplan + rm -f "$RUNNER_TEMP"/destroy-"$TARGET_ENV"-*.json \ No newline at end of file diff --git a/README.md b/README.md index d93e362..f4b49b2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ - [Protect main Branch Ruleset](#protect-main-branch-ruleset) - [3.2 Continuous Integration](#32-continuous-integration--githubworkflowsciyml) - [3.3 Deployment](#33-deployment--githubworkflowsdeployyml) - - [3.4 TEST and STAGE Destruction Workflow](#34-test-and-stage-destruction-workflow--githubworkflowsdestroyyml) + - [3.4 Combined TEST and STAGE Destruction](#34-combined-test-and-stage-destruction--githubworkflowsdestroyyml) + - [3.5 Dedicated PROD Destruction](#35-dedicated-prod-destruction--githubworkflowsdestroy-prodyml) + - [3.6 Planned Individual TEST and STAGE Destruction](#36-planned-individual-test-and-stage-destruction) 4. [Prerequisites and Credentials](#4-prerequisites-and-credentials) - [GitHub Actions Environments: Required Configuration](#github-actions-environments-required-configuration) 5. [Running the Workflow End to End](#5-running-the-workflow-end-to-end) @@ -24,11 +26,14 @@ - [5.3 Manual Deployment through PROD](#53-manual-deployment-through-prod) - [5.4 Local Development, Validation, and Manual Deployment](#54-local-development-validation-and-manual-deployment) - [5.5 TEST and STAGE Destruction and Recreation](#55-test-and-stage-destruction-and-recreation) + - [5.6 PROD Destruction and Recreation](#56-prod-destruction-and-recreation) 6. [Destroying Resources and Cleanup](#6-destroying-resources-and-cleanup) - [6.1 Automated Destruction of TEST and STAGE](#61-automated-destruction-of-test-and-stage) - - [6.2 Manually Destroy an Individual Frontend Stack](#62-manually-destroy-an-individual-frontend-stack) - - [6.3 Manually Destroy All Stacks in One Environment](#63-manually-destroy-all-stacks-in-one-environment) - - [6.4 Manually Destroy All Three Environments](#64-manually-destroy-all-three-environments) + - [6.2 Automated Destruction of PROD](#62-automated-destruction-of-prod) + - [6.3 Planned Single-Environment Destruction](#63-planned-single-environment-destruction) + - [6.4 Manually Destroy an Individual Frontend Stack](#64-manually-destroy-an-individual-frontend-stack) + - [6.5 Manually Destroy All Stacks in One Environment](#65-manually-destroy-all-stacks-in-one-environment) + - [6.6 Manually Destroy All Three Environments](#66-manually-destroy-all-three-environments) 7. [Security, State Management, and Design Decisions](#7-security-state-management-and-design-decisions) 8. [Verification and Evidence](#8-verification-and-evidence) 9. [Known Limitations and Improvements](#9-known-limitations-and-improvements) @@ -46,23 +51,25 @@ The backend and database *subnets* are created as part of the network topology; ### Implementation status -| Capability | Status | -| --- | --- | -| CI formatting, validation, linting, and security scanning | Implemented; CI passed | -| Automatic TEST deployment after successful CI on `main` | Implemented and successfully executed | -| Automatic STAGE deployment after TEST | Implemented and successfully executed | -| Manual PROD deployment after TEST and STAGE | Implemented and successfully executed | -| Human approval for PROD deployment | **Configured** through GitHub environment protection | -| `main` branch protection | **Configured:** `Protect main` ruleset; pull requests and passing CI required, 0 required approvals | -| Two OpenTofu stacks in each environment | Implemented | -| Environment- and stack-specific remote state | Implemented | -| Reusable network and VM modules | Implemented (local modules) | -| GitHub Actions TEST and STAGE destruction | **Implemented and successfully executed** using a matrix; frontend before network | -| TEST and STAGE recreation after destruction | **Successfully tested** through `Deploy Infrastructure` | -| PROD destruction through GitHub Actions | **Not yet implemented**; manual CLI procedure documented below | -| Manual destruction using OpenTofu CLI | Documented for all three environments and for individual frontend stacks | -| Centralized CI tool versions | `.opentofu-version`, `.tflint-version`, `requirements-checkov.txt` | -| Independently tagged releases of reusable modules | **Not implemented**; modules are versioned with repository commits | +| Capability | Status | +| --- |-------------------------------------------------------------------------------------------------------------| +| CI formatting, validation, linting, and security scanning | Implemented; CI passed | +| Automatic TEST deployment after successful CI on `main` | Implemented and successfully executed | +| Automatic STAGE deployment after TEST | Implemented and successfully executed | +| Manual PROD deployment after TEST and STAGE | Implemented and successfully executed | +| Human approval for PROD deployment and destruction | **Configured** through GitHub environment protection; authorized reviewers: Einar, Jørn, and Pau | +| `main` branch protection | **Configured:** `Protect main` ruleset; pull requests and passing CI required, 0 required approvals | +| Two OpenTofu stacks in each environment | Implemented | +| Environment- and stack-specific remote state | Implemented | +| Reusable network and VM modules | Implemented (local modules) | +| GitHub Actions TEST and STAGE destruction | **Implemented and successfully executed** using a matrix; frontend before network | +| TEST and STAGE recreation after destruction | **Successfully tested** through `Deploy Infrastructure` | +| Dedicated PROD destruction through GitHub Actions | **Implemented and successfully executed** through `destroy-prod.yml` with approval; frontend before network | +| PROD recreation after complete destruction | **Successfully tested** through a new manual `Deploy Infrastructure` run | +| Independent TEST-only and STAGE-only destroy workflows | **Planned; not yet implemented or tested**; combined matrix remains available | +| Manual destruction using OpenTofu CLI | Documented for all three environments and for individual frontend stacks | +| Centralized CI tool versions | `.opentofu-version`, `.tflint-version`, `requirements-checkov.txt` | +| Independently tagged releases of reusable modules | **Not implemented**; modules are versioned with repository commits | The destruction and recreation workflows were developed after the original November 1 submission. This README describes the current repository state rather than claiming that post-submission changes were part of the original ZIP. Runner availability and instructor permissions still require verification through GitHub and OpenStack administration. @@ -74,6 +81,7 @@ assigment2-iac/ │ ├── ci.yml # Formatting, validation, TFLint, Checkov │ ├── deploy.yml # TEST → STAGE → conditional PROD │ ├── destroy.yml # Manually triggered TEST + STAGE destruction +│ ├── destroy-prod.yml # Manually triggered, approval-gated PROD destruction │ └── runner-test.yml # Manually triggered runner health check ├── bootstrap/ │ ├── cloud_init/runner.yaml # Runner VM operating-system packages @@ -99,6 +107,8 @@ assigment2-iac/ └── README.md ``` +**Planned additions (not yet present in the repository):** `.github/workflows/destroy-test.yml` and `.github/workflows/destroy-stage.yml` will provide independently triggered cleanup while retaining `destroy.yml` for combined TEST/STAGE cleanup. + Every environment has its own network and frontend root module, its own `deployment.tfvars`, and its own `.terraform.lock.hcl`. The frontend stack uses `terraform_remote_state` to read the network ID and frontend subnet ID from the corresponding environment's network state. Therefore **network must be created before frontend, and frontend must be destroyed before network**. ### Environment configuration @@ -143,7 +153,7 @@ The repository has an active GitHub branch ruleset named **`Protect main`**. It Zero required approvals allows contributions to be merged without another reviewer, which is practical for this individually developed assignment, but **does not enforce independent code review**. Pull requests and the required CI check still provide a controlled path into `main`. These repository rules are configured in GitHub settings rather than in the workflow YAML files. -This branch ruleset is separate from the **human approval gate for PROD deployment**, which is configured through GitHub environment protection rules. +This branch ruleset is separate from the **human approval gate for PROD deployment and destruction**, which is configured through GitHub environment protection rules. The authorized environment reviewers are Einar, Jørn, and Pau. Reviewers' availability does not replace the assignment's separate requirement that Jørn and Pau have repository access and permission to execute the workflows. ### 3.2 Continuous integration — `.github/workflows/ci.yml` @@ -184,9 +194,9 @@ Manual "Deploy Infrastructure" run from main For each environment, the deployment job checks OpenStack credentials, installs checksum-verified `kubectl` and `kOps`, obtains a short-lived **1-hour Kubernetes admin kubeconfig**, verifies access to the `iac` namespace, initializes the backends, then processes **network before frontend**. OpenTofu saves each plan and inspects the plan JSON with `jq`. The pipeline only allows `no-op`, `read`, `create`, and `update` resource actions; it rejects planned deletions and replacements. It then applies the exact saved plan, or skips application when there are no changes. Temporary plans, logs, and kubeconfig files are removed in cleanup steps. -The TEST `Initialize TEST remote state` step no longer requires pre-existing managed resources. It initializes the network and frontend Kubernetes backends even when the state is empty, allowing TEST to be provisioned initially or recreated following a complete destroy. **Both TEST and STAGE destroy → recreate sequences have now been successfully tested through GitHub Actions.** PROD already supports initialization from empty state, but its destruction/recreation cycle has not yet been tested. +The TEST `Initialize TEST remote state` step no longer requires pre-existing managed resources. It initializes the network and frontend Kubernetes backends even when the state is empty, allowing TEST to be provisioned initially or recreated following a complete destroy. **All three environments have now successfully completed deploy → destroy → recreate tests through GitHub Actions.** PROD was recreated through a fresh manual deployment run after the dedicated PROD destruction succeeded. -### 3.4 TEST and STAGE destruction workflow — `.github/workflows/destroy.yml` +### 3.4 Combined TEST and STAGE destruction — `.github/workflows/destroy.yml` The separate **Destroy Infrastructure (TEST, STAGE)** workflow follows the Lab 7 pattern and is **manually triggered** with `workflow_dispatch` from `main`. A GitHub Actions **matrix** creates one job for each of these environment mappings: @@ -208,7 +218,26 @@ Planned resource addresses are displayed without dumping configuration attribute The matrix uses `max-parallel: 1`, so only one environment job executes at a time, but **GitHub does not guarantee TEST executes before STAGE**. With `fail-fast: false`, a failure in one environment does not automatically cancel the other. **A single manual run attempts destruction of both TEST and STAGE**, not an operator-selected subset. Once triggered (and after any configured environment protection checks), the destruction is noninteractive and applies saved destroy plans. -**Verified:** The matrix-based workflow successfully destroyed the TEST and STAGE infrastructure, and the subsequent deployment jobs successfully recreated both environments. PROD infrastructure was outside the matrix and was not targeted. The current approach reduces duplicated YAML; independent environment triggers would provide finer-grained destructive-operation control in a future iteration. +**Verified:** The matrix-based workflow successfully destroyed the TEST and STAGE infrastructure, and the subsequent deployment jobs successfully recreated both environments. PROD infrastructure was outside the matrix and was not targeted. The current matrix remains useful for complete non-production cleanup. + +### 3.5 Dedicated PROD destruction — `.github/workflows/destroy-prod.yml` + +The **Destroy Infrastructure (PROD)** workflow is a separate, manually dispatched entry point available from `main`. It does **not** use the TEST/STAGE matrix. Its job uses `environment: PROD`, so the **existing required human approval gate** must be satisfied before any destruction steps execute. Authorized reviewers are the repository owner, Jørn, and Pau; self-approval depends on the configured environment rules. + +Once approved, the job uses PROD-scoped OpenStack credentials, validates Kubernetes remote-state suffixes `assignment2-prod-frontend` and `assignment2-prod-network`, initializes the backends, and applies saved `tofu plan -destroy` plans **frontend first, then network**. It shares `infrastructure-deployment` concurrency with deployment and other destruction workflows. It does not target TEST, STAGE, the self-hosted runner, or the Kubernetes state cluster. + +**Verified:** The PROD destruction run succeeded after environment approval. A subsequent **new, manually dispatched** `Deploy Infrastructure` run successfully recreated PROD through the TEST → STAGE → approved PROD sequence. + +### 3.6 Planned individual TEST and STAGE destruction + +To enable selective cleanup while retaining the tested combined matrix workflow, two additional manual workflow entry points are planned: + +| Planned workflow | Target | Approval/credentials | Status | +| --- | --- | --- | --- | +| `.github/workflows/destroy-test.yml` | TEST only | `TEST` GitHub environment | **Planned; not yet added** | +| `.github/workflows/destroy-stage.yml` | STAGE only | `STAGE` GitHub environment | **Planned; not yet added** | + +Each will reuse the validated operational pattern (correct environment-scoped state and credentials; saved destroy plans; frontend → network; shared concurrency and cleanup). The combined `destroy.yml` and dedicated `destroy-prod.yml` will remain intact. Separate manual entry points avoid adding `workflow_dispatch` inputs and reduce the risk of destroying both non-production environments when only one needs cleanup. Duplicating the workflow body is an interim trade-off; a future `workflow_call` reusable workflow would centralize the shared steps. ## 4. Prerequisites and credentials @@ -220,7 +249,7 @@ The workflow assumes the following pre-existing infrastructure and permissions: - Kubernetes namespace **`iac`**, plus permission to read/update Secrets and use Lease objects needed for backend locking. - A running, registered GitHub Actions self-hosted Linux runner labeled `self-hosted`, `linux`, `assignment2`, with outbound access to the OpenStack APIs, the cluster, GitHub Actions, and the relevant download endpoints. - `jq`, Git, Python 3 with venv, curl, and the libraries required by the runner. `bootstrap/cloud_init/runner.yaml` installs these OS packages, **but does not register the GitHub runner agent**. Agent registration and long-running service setup must be completed separately. -- In GitHub's repository settings, environment names **TEST**, **STAGE**, and **PROD**, with appropriate environment-specific credentials. **Required human review is configured for PROD deployment**; the YAML's manual trigger alone would not constitute reviewer approval. Destruction currently uses the existing TEST/STAGE environments; adding reviewer protection to either would also affect automatic deployments to that environment. +- In GitHub's repository settings, environment names **TEST**, **STAGE**, and **PROD**, with appropriate environment-specific credentials. **Required human review is configured for both PROD deployment and destruction** through the shared `PROD` environment; the YAML's manual trigger alone would not constitute reviewer approval. Destruction currently uses the existing TEST/STAGE environments; adding reviewer protection to either would also affect automatic deployments to that environment. ### GitHub Actions environments: required configuration @@ -341,11 +370,23 @@ The complete **deploy → destroy → recreate** lifecycle has been successfully 4. Recreate both environments by opening a successful **CI-triggered** `Deploy Infrastructure` run created after the updated code reached `main`, then selecting **Re-run all jobs**. It recreates TEST network/frontend, then STAGE network/frontend. PROD is skipped because the original event is `workflow_run`, not `workflow_dispatch`. 5. Confirm that all intended resources exist again and capture screenshots of the successful destruction and recreation runs. -**Do not start a new manual `Run workflow` of `Deploy Infrastructure` solely to recreate TEST/STAGE** unless you also intend to attempt PROD: a manual deployment run from `main` executes TEST → STAGE → the protected PROD job. The TEST and STAGE lifecycle checks reported success; this is not yet evidence that PROD destruction has been tested. +**Do not start a new manual `Run workflow` of `Deploy Infrastructure` solely to recreate TEST/STAGE** unless you also intend to attempt PROD: a manual deployment run from `main` executes TEST → STAGE → the protected PROD job. + +### 5.6 PROD destruction and recreation + +A complete **deploy → destroy → recreate** lifecycle has been successfully verified for PROD using a separate, approval-protected destruction workflow: + +1. From **Actions → Deploy Infrastructure**, manually run the workflow on `main`. After TEST and STAGE complete, approve the waiting PROD job and verify that PROD network and frontend are provisioned. +2. Capture evidence of the deployed PROD resources before cleanup. +3. Open **Actions → Destroy Infrastructure (PROD) → Run workflow**, select `main`, and trigger it. The protected `PROD` environment pauses the job for an authorized reviewer (repository owner, Jørn, or Pau). +4. After approval, verify **Destroy PROD** successfully removes the **frontend first, then the network**. Confirm the TEST and STAGE environments, self-hosted runner, and Kubernetes state cluster are not targeted. +5. To recreate PROD, start a **new manual** **Actions → Deploy Infrastructure → Run workflow** execution on `main`. TEST and STAGE run before PROD, and the PROD job again requires human approval. Verify the recreated frontend and network resources in OpenStack. + +**Verified:** PROD destruction succeeded, and a fresh manually triggered deployment of TEST → STAGE → PROD subsequently succeeded. A rerun of a previous *manual* deployment would also retain PROD eligibility, but a new manual run was used to produce a clearer audit trail. ## 6. Destroying resources and cleanup -**Current status:** `.github/workflows/destroy.yml` provides an **implemented and successfully tested TEST + STAGE matrix destruction workflow**. Both environments were subsequently recreated successfully. Normal `deploy.yml` still rejects deletions and replacements. **PROD destruction through GitHub Actions is not yet implemented**; the CLI procedures below also cover manually selected stack/environment cleanup. +**Current status:** Both `.github/workflows/destroy.yml` (**combined TEST + STAGE**) and `.github/workflows/destroy-prod.yml` (**approved PROD-only**) have been successfully executed, and all three environments have been recreated through GitHub Actions. Normal `deploy.yml` still rejects deletions and replacements. Individually triggered `destroy-test.yml` and `destroy-stage.yml` workflows are planned but **not yet implemented**; CLI procedures below permit selective stack cleanup in the meantime. **Warning:** Destruction deletes real OpenStack resources. Obtain authorization, capture the required deployment evidence, protect all remote state, and confirm the target environment before proceeding. Never destroy `bootstrap/` while it hosts the required runner. Never destroy the external kOps cluster because the six stack states depend on it. @@ -357,9 +398,32 @@ The complete **deploy → destroy → recreate** lifecycle has been successfully 4. Within **each** environment, the workflow initializes state, creates and applies a saved destroy plan for **frontend**, and then repeats the procedure for **network**. 5. Verify that **Destroy TEST** and **Destroy STAGE** succeed and their resources have been removed. The runner VM, external kOps cluster, and PROD resources are **not** part of this workflow. -**Scope and safety:** There is no environment or stack selector in the current GitHub Actions destroy workflow. `max-parallel: 1` serializes the matrix jobs but does not guarantee which environment executes first; `fail-fast: false` means one job's failure does not stop the other from being attempted. These jobs apply destroy plans automatically once dispatched. For an individual stack or a single environment, use the carefully reviewed CLI procedures below or introduce an explicitly scoped workflow entry point. The human approval gate for **deploying PROD** is separate from, and does not authorize, any future PROD destruction workflow. +**Scope and safety:** There is no environment or stack selector in the current GitHub Actions destroy workflow. `max-parallel: 1` serializes the matrix jobs but does not guarantee which environment executes first; `fail-fast: false` means one job's failure does not stop the other from being attempted. These jobs apply destroy plans automatically once dispatched. For an individual stack or a single environment, use the carefully reviewed CLI procedures below or introduce an explicitly scoped workflow entry point. The separate **PROD destruction** workflow is already protected by the `PROD` environment approval gate; it is not part of this matrix. + +### 6.2 Automated destruction of PROD + +1. Open **Actions → Destroy Infrastructure (PROD)** and choose **Run workflow** on **`main`**. +2. The job waits for required approval in the `PROD` GitHub environment. An authorized reviewer (repository owner, Jørn, or Pau) uses **Review deployments → PROD → Approve and deploy**. This generic GitHub label approves the **destruction job**, not a deployment. +3. After approval, the job checks PROD credentials, Kubernetes access, and the expected `assignment2-prod-*` state suffixes before applying saved destroy plans. +4. It destroys the **PROD frontend** followed by the **PROD network**, without targeting other environments or the runner/state infrastructure. +5. Verify **Destroy PROD** succeeded, then recreate the infrastructure using the manual deployment recipe in section 5.6. + +**Verified:** The approval-protected PROD destroy workflow ran successfully, and PROD was successfully recreated afterward. The approval gate applies before the job starts; there is no additional pause to approve each individual destroy plan. + +### 6.3 Planned single-environment destruction + +The intended extensions `.github/workflows/destroy-test.yml` and `.github/workflows/destroy-stage.yml` will be manually triggered, with **no `workflow_dispatch` inputs**, and will target only the named environment. They are **not yet implemented or verified**. Once created and tested, operators will have these choices: + +| Workflow | Intended destruction scope | Current availability | +| --- | --- | --- | +| `destroy-test.yml` | TEST frontend → TEST network | Planned | +| `destroy-stage.yml` | STAGE frontend → STAGE network | Planned | +| `destroy.yml` | TEST and STAGE, each frontend → network | Implemented and verified | +| `destroy-prod.yml` | PROD frontend → PROD network, with human approval | Implemented and verified | + +All destructive workflows should share `infrastructure-deployment` concurrency and validate environment-specific credentials/state. Keep the combined workflow for intentional whole-non-production cleanup, but use individual entry points for routine single-environment cleanup once available. -### 6.2 Manually destroy an individual frontend stack +### 6.4 Manually destroy an individual frontend stack Ensure the local Kubernetes kubeconfig and correct OpenStack credentials are active as described in section 5.4. For example, for STAGE: @@ -374,7 +438,7 @@ tofu -chdir="$DIR" destroy -var-file=deployment.tfvars This removes the frontend VM and its managed dependencies, while leaving the network stack intact. **Do not destroy the network first** while any frontend VM still uses its subnet and network remote-state outputs. -### 6.3 Manually destroy all stacks in one environment +### 6.5 Manually destroy all stacks in one environment Change the selected environment as needed (`test`, `stage`, `prod`). Use the **correct environment-specific** deployment credentials before running. Both stacks are processed in reverse dependency order: @@ -389,11 +453,11 @@ for STACK in frontend network; do done ``` -### 6.4 Manually destroy all three environments +### 6.6 Manually destroy all three environments -Run the **section 6.3 sequence separately for PROD, STAGE, and TEST**, in that order. **Switch to the correct OpenStack deployment credentials before each environment**. This deliberate, environment-by-environment process avoids accidentally using PROD permissions or configuration for another environment. Collect a successful OpenTofu destroy log and resource inventory screenshot for each environment if destruction evidence is required. +Run the **section 6.5 sequence separately for PROD, STAGE, and TEST**, for example in that order. There is no dependency requiring a particular order **between** environments. **Switch to the correct OpenStack deployment credentials before each environment**. This deliberate, environment-by-environment process avoids accidentally using PROD permissions or configuration for another environment. Collect a successful OpenTofu destroy log and resource inventory screenshot for each environment if destruction evidence is required. -The updated deployment jobs initialize empty TEST and STAGE remote state successfully. Both environments were recreated after their matrix-based destruction. See section 5.5 for the verified lifecycle procedure. PROD destruction must be separately authorized and tested before being added to the automated workflow. +The deployment jobs successfully recreated TEST and STAGE after their combined destruction, and PROD after its separately approved destruction. See sections 5.5 and 5.6 for verified lifecycle procedures. The two individual TEST/STAGE workflow entry points remain planned. ## 7. Security, state management, and design decisions @@ -403,12 +467,13 @@ The updated deployment jobs initialize empty TEST and STAGE remote state success - **Limited authentication exposure:** OpenStack uses application credentials and GitHub environment secrets; kOps generates short-lived kubeconfig files. Deployment and destruction workflows remove temporary kubeconfig and plan files during cleanup. - **Static quality gates and pinned tools:** formatting, validation, TFLint, and Checkov execute before automatic deployment. `.opentofu-version`, `.tflint-version`, and `requirements-checkov.txt` centralize CI tool version selection; `.terraform.lock.hcl` pins provider selections. Upgrades should be reviewed and validated in CI. - **Non-destructive deployment:** normal deployments reject planned deletion and replacement actions and apply only the saved, checked plan. -- **Controlled destruction:** a separate manually triggered workflow uses a matrix for TEST and STAGE. Each job verifies the matching Kubernetes backend, uses environment-scoped credentials, destroys frontend before network, and applies saved destroy plans. Jobs are serialized by `max-parallel: 1` but have no guaranteed environment order. One trigger affects both TEST and STAGE; independent triggers are an improvement for selective cleanup. -- **PROD authorization:** the production deployment job is manually triggered and protected by configured human reviewer approval. **PROD destruction is not yet implemented** and should receive its own explicit authorization controls before use. +- **Controlled destruction and operator scope:** `destroy.yml` uses a manually triggered matrix for combined TEST/STAGE cleanup, with `max-parallel: 1` and no guaranteed environment order. `destroy-prod.yml` independently targets PROD behind approval. Every job verifies its environment-specific Kubernetes state and credentials, destroys frontend before network, and uses saved destroy plans. Separate TEST-only and STAGE-only entry points are planned for safer selective cleanup, while retaining combined non-production cleanup as an explicit option. +- **Workflow reuse trade-off:** dedicated entry points are easier to operate safely without dispatch inputs, but repeating the same shell steps across workflows increases maintenance. Extracting the shared logic into a reusable workflow is a potential post-assignment refactor; the verified workflows remain unchanged during the addition of TEST-only/STAGE-only entry points. +- **PROD authorization:** both production deployment and the dedicated production destruction workflow require manual initiation and `PROD` environment approval. The repository owner, Jørn, and Pau are configured as authorized reviewers; they must also have the necessary repository permissions to execute workflows for grading. - **Network exposure:** frontend permits HTTP on port 80 from the configured `http_cidr` (currently `0.0.0.0/0`), and SSH on port 22 only from `admin_cidr`. This is a demonstration Nginx service, not a hardened application delivery platform. - **Reuse and versioning:** `modules/network` and `modules/vm` are reused across environments via relative source paths. Module code is versioned at the **repository commit level**, with the same module revision used by each environment at a given commit. There is **no independent module release/tag or registry version pinning**. -For reuse across customers, a future iteration could publish independently tagged module releases, adopt reusable deployment/destruction workflows with explicitly scoped entry points, use separate customer state backends and accounts, and add policy-as-code and approved PROD cleanup. +For reuse across customers, a future iteration could publish independently tagged module releases, centralize duplicated deployment/destruction logic through reusable workflows with explicitly scoped entry points, use separate customer state backends and accounts, and add further policy-as-code checks. ## 8. Verification and evidence @@ -422,7 +487,9 @@ The following workflow outcomes have been reported as successful and can be revi | Earlier TEST-only destroy/recreate test | Destroy TEST succeeded; Deploy TEST recreated both stacks | | Matrix-based destruction | **Destroy TEST and Destroy STAGE succeeded** | | Deployment rerun after matrix destruction | **TEST and STAGE recreated successfully**; PROD skipped | -| PROD destruction and recreation | **Not yet tested** | +| Dedicated PROD destruction | **Destroy PROD succeeded** after approval; frontend and network removed | +| New manually triggered deployment after PROD destruction | **TEST, STAGE, and PROD succeeded**; PROD recreated after approval | +| Individual TEST-only and STAGE-only destruction entry points | Planned; not yet implemented or tested | A screenshot of a full successful run end to end with destruction will be added once STAGE and PROD destruction has been completed. @@ -430,8 +497,8 @@ For an optional workload check, identifying a frontend floating IP from OpenTofu ## 9. Known limitations and improvements -1. **PROD destruction is not yet implemented:** the current GitHub Actions destruction matrix covers TEST and STAGE only. Before including PROD, add an explicitly authorized destruction path, verify state isolation, and test deletion followed by recreation. -2. **Matrix scope:** one manual `Destroy Infrastructure` run targets **both TEST and STAGE**; selecting an individual environment or stack is not supported through this workflow. Manual OpenTofu CLI recipes provide targeted cleanup in the meantime. +1. **Individual TEST/STAGE destruction entry points pending:** `.github/workflows/destroy-test.yml` and `.github/workflows/destroy-stage.yml` are planned but have not yet been added or tested. Existing automation supports combined TEST/STAGE destruction and separately approved PROD destruction; individual stack cleanup is documented through the OpenTofu CLI. +2. **Combined matrix scope:** one manual `Destroy Infrastructure (TEST, STAGE)` run targets **both** environments, so operators must not use it when only one should be removed. Dedicated TEST/STAGE workflows will address this once implemented. 3. **Matrix sequencing:** `max-parallel: 1` prevents concurrent destruction jobs but does not guarantee TEST-before-STAGE ordering; `fail-fast: false` permits another matrix job to proceed if one fails. 4. **Module versions are tied to the repository commit:** separately published and pinned module releases would improve customer reuse and rollback. 5. **Runner and kOps dependencies are external:** self-hosted runner service availability, kOps cluster, Kubernetes namespace, access control, and instructor permissions require separate setup and ongoing maintenance.