IDATG2900-Quicksand · majame · May 5, 2025 · May 5, 2025 · May 5, 2025 · May 5, 2025
diff --git a/.github/workflows/deploy_capev2.yml b/.github/workflows/deploy_capev2.yml
@@ -7,8 +7,8 @@ on:
     paths:
       - 'capev2/**'
       - 'modules/cape_vm/**'
-      - 'playbooks/capev2ansible.yaml'
       - 'playbooks/openstack.yaml'
+      - 'playbooks/capev2_install.yaml'
   workflow_dispatch:
 
 env:
@@ -43,7 +43,7 @@ jobs:
         working-directory: ${{ env.WORKING_DIR }}
 
       - name: Run capev2 configuration ansible playbook
-        run: ansible-playbook -i openstack.yaml capev2ansible.yaml -u ansible
+        run: ansible-playbook -i openstack.yaml capev2_install.yaml -u ansible
         working-directory: ./playbooks
 
   staging:
@@ -74,5 +74,5 @@ jobs:
         working-directory: ${{ env.WORKING_DIR }}
 
       - name: Run capev2 configuration ansible playbook
-        run: ansible-playbook -i openstack.yaml capev2ansible.yaml -u ansible
+        run: ansible-playbook -i openstack.yaml capev2_install.yaml -u ansible
         working-directory: ./playbooks
diff --git a/.github/workflows/deploy_kubernetes.yml b/.github/workflows/deploy_kubernetes.yml
@@ -0,0 +1,82 @@
+name: Deploy Kubernetes module
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'kubernetes/**'
+      - 'modules/cluster_network/**'
+      - 'modules/control_plane/**'
+      - 'modules/load_balancer/**'
+      - 'modules/security/**'
+      - 'modules/shared_router/**'
+      - 'modules/ssh_key/**'
+      - 'modules/storage/**'
+      - 'modules/talos_config/**'
+      - 'modules/workers/**'
+  workflow_dispatch:
+
+env:
+  WORKING_DIR: ./kubernetes
+
+jobs:
+
+  dev:
+    runs-on: dev
+    env:
+      PG_CONN_STR: ${{ secrets.PG_CONN_STR_DEV }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Terraform init
+        run: terraform init
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform workspace select or create kubernetes
+        run: terraform workspace select -or-create kubernetes
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Create tfvars file
+        run: echo ${{ secrets.KUBERNETES_TFVARS }} > terraform.tfvars
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform plan
+        run: terraform plan -var "os_auth_url=${{ secrets.OS_AUTH_URL_DEV }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_DEV }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_DEV }}"
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform apply
+        run: terraform apply -auto-approve -var "os_auth_url=${{ secrets.OS_AUTH_URL_DEV }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_DEV }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_DEV }}"
+        working-directory: ${{ env.WORKING_DIR }}
+
+  staging:
+    needs: dev
+    runs-on: stage
+    env:
+      PG_CONN_STR: ${{ secrets.PG_CONN_STR_STAGE }}
+
+    steps:
+      - name: Chekcout
+        uses: actions/checkout@v4
+
+      - name: Terraform init
+        run: terraform init
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform workspace select or create kubernetes
+        run: terraform workspace select -or-create kubernetes
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Create tfvars file
+        run: echo ${{ secrets.KUBERNETES_TFVARS }} > terraform.tfvars
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform plan
+        run: terraform plan -var "os_auth_url=${{ secrets.OS_AUTH_URL_STAGE }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_STAGE }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_STAGE }}"
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform apply
+        run: terraform apply -auto-approve -var "os_auth_url=${{ secrets.OS_AUTH_URL_STAGE }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_STAGE }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_STAGE }}"
+        working-directory: ${{ env.WORKING_DIR }}
diff --git a/.github/workflows/validate_kubernetes.yml b/.github/workflows/validate_kubernetes.yml
@@ -0,0 +1,46 @@
+name: Validate Kubernetes module
+
+on:
+  push:
+    branches-ignore:
+      - 'main'
+    paths:
+      - 'kubernetes/**'
+      - 'modules/cluster_network/**'
+      - 'modules/control_plane/**'
+      - 'modules/load_balancer/**'
+      - 'modules/security/**'
+      - 'modules/shared_router/**'
+      - 'modules/ssh_key/**'
+      - 'modules/storage/**'
+      - 'modules/talos_config/**'
+      - 'modules/workers/**'
+  workflow_dispatch:
+
+jobs:
+
+  validate:
+    runs-on: dev
+    env:
+      WORKING_DIR: ./kubernetes
+      PG_CONN_STR: ${{ secrets.PG_CONN_STR_DEV }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Terraform init
+        run: terraform init
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform fmt
+        run: terraform fmt
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform validate
+        run: terraform validate
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Tfsec
+        run: tfsec
+        working-directory: ${{ env.WORKING_DIR }}
diff --git a/.github/workflows/validate_misp.yml b/.github/workflows/validate_misp.yml
@@ -0,0 +1,39 @@
+name: Validate MISP module
+
+on:
+  push:
+    branches-ignore:
+      - 'main'
+    paths:
+      - 'misp/**'
+      - 'modules/misp/**'
+      - 'modules/security/**'
+  workflow_dispatch:
+
+jobs:
+
+  validate:
+    runs-on: dev
+    env:
+      WORKING_DIR: ./misp
+      PG_CONN_STR: ${{ secrets.PG_CONN_STR_DEV }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Terraform init
+        run: terraform init
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform fmt
+        run: terraform fmt
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Terraform validate
+        run: terraform validate
+        working-directory: ${{ env.WORKING_DIR }}
+
+      - name: Tfsec
+        run: tfsec
+        working-directory: ${{ env.WORKING_DIR }}
diff --git a/kubernetes/locals.tf b/kubernetes/locals.tf
@@ -1,4 +1,4 @@
 locals {
-  kubernetes_internal_endpoint  = "https://${module.control_plane.control_plane_internal_ips[0]}:6443"
+  kubernetes_internal_endpoint = "https://${module.control_plane.control_plane_internal_ips[0]}:6443"
 }
 
diff --git a/kubernetes/main.tf b/kubernetes/main.tf
@@ -28,7 +28,7 @@ module "shared_router" {
   source = "../modules/shared_router"
 
   existing_router_name = var.main_router_name
-  cluster_subnet_id   = module.cluster_network.subnet_id
+  cluster_subnet_id    = module.cluster_network.subnet_id
 }
 
 ##############################
@@ -50,55 +50,55 @@ module "ssh_key" {
 # 2. Control Plane (Master Nodes) Setup
 ##############################
 module "control_plane" {
-  source                    = "../modules/control_plane"
-  talos_image_name          = var.talos_image_name
-  internal_network_id       = module.cluster_network.network_id
-  control_plane_subnet_id   = module.cluster_network.subnet_id
-  control_plane_flavor_name = var.control_plane_flavor_name
-  control_plane_node_count  = var.control_plane_node_count
-  kubernetes_cluster_name   = var.kubernetes_cluster_name
-  control_plane_sg_id       = module.security.control_plane_sg_id
-  allow_all_egress_sg_id    = module.security.allow_all_egress_id
+  source                           = "../modules/control_plane"
+  talos_image_name                 = var.talos_image_name
+  internal_network_id              = module.cluster_network.network_id
+  control_plane_subnet_id          = module.cluster_network.subnet_id
+  control_plane_flavor_name        = var.control_plane_flavor_name
+  control_plane_node_count         = var.control_plane_node_count
+  kubernetes_cluster_name          = var.kubernetes_cluster_name
+  control_plane_sg_id              = module.security.control_plane_sg_id
+  allow_all_egress_sg_id           = module.security.allow_all_egress_id
   allowed_management_ingress_sg_id = module.security.allowed_management_ingress_sg_id
 }
 
 ##############################
 # 3. Worker Nodes Setup
 ##############################
 module "workers" {
-  source                   = "../modules/workers"
-  talos_image_name         = var.talos_image_name
-  internal_network_id      = module.cluster_network.network_id
-  worker_nodes_subnet_id   = module.cluster_network.subnet_id
-  worker_nodes_flavor_name = var.worker_nodes_flavor_name
-  worker_nodes_node_count  = var.worker_nodes_node_count
-  kubernetes_cluster_name  = var.kubernetes_cluster_name
-  allow_all_egress_sg_id          = module.security.allow_all_egress_id
+  source                           = "../modules/workers"
+  talos_image_name                 = var.talos_image_name
+  internal_network_id              = module.cluster_network.network_id
+  worker_nodes_subnet_id           = module.cluster_network.subnet_id
+  worker_nodes_flavor_name         = var.worker_nodes_flavor_name
+  worker_nodes_node_count          = var.worker_nodes_node_count
+  kubernetes_cluster_name          = var.kubernetes_cluster_name
+  allow_all_egress_sg_id           = module.security.allow_all_egress_id
   allowed_management_ingress_sg_id = module.security.allowed_management_ingress_sg_id
-  worker_nodes_sg_id       = module.security.worker_nodes_sg_id
+  worker_nodes_sg_id               = module.security.worker_nodes_sg_id
 }
 
 ##############################
 # 4. Load Balancer Setup
 ##############################
 module "load_balancer" {
-  source                          = "../modules/load_balancer"
-  internal_network_id             = module.cluster_network.network_id
-  control_plane_subnet_id         = module.cluster_network.subnet_id
-  kubernetes_name                 = var.kubernetes_cluster_name
-  instance_image_name             = var.haproxy_image_name
-  instance_flavor_name            = var.haproxy_flavor_name
-  ssh_key_name                    = module.ssh_key.ssh_key_name
-  ssh_public_key                  = module.ssh_key.public_key
-  load_balancer_ip                = var.load_balancer_ip
-  floating_ip_pool                = var.public_network_name
-  control_plane_ips               = module.control_plane.control_plane_internal_ips
-  worker_node_ips                 = module.workers.worker_node_internal_ips
-  control_plane_sg_id             = module.security.control_plane_sg_id
-  allow_all_egress_sg_id          = module.security.allow_all_egress_id
+  source                           = "../modules/load_balancer"
+  internal_network_id              = module.cluster_network.network_id
+  control_plane_subnet_id          = module.cluster_network.subnet_id
+  kubernetes_name                  = var.kubernetes_cluster_name
+  instance_image_name              = var.haproxy_image_name
+  instance_flavor_name             = var.haproxy_flavor_name
+  ssh_key_name                     = module.ssh_key.ssh_key_name
+  ssh_public_key                   = module.ssh_key.public_key
+  load_balancer_ip                 = var.load_balancer_ip
+  floating_ip_pool                 = var.public_network_name
+  control_plane_ips                = module.control_plane.control_plane_internal_ips
+  worker_node_ips                  = module.workers.worker_node_internal_ips
+  control_plane_sg_id              = module.security.control_plane_sg_id
+  allow_all_egress_sg_id           = module.security.allow_all_egress_id
   allowed_management_ingress_sg_id = module.security.allowed_management_ingress_sg_id
-  allow_bastion                   = module.security.bastion_sg_id
-  load_balancer_sg_id             = module.security.load_balancer_sg_id
+  allow_bastion                    = module.security.bastion_sg_id
+  load_balancer_sg_id              = module.security.load_balancer_sg_id
 }
 
 ##############################
@@ -108,52 +108,52 @@ module "talos_config" {
   source = "../modules/talos_config"
 
   # Basic cluster information
-  cluster_name          = var.kubernetes_cluster_name
-  control_plane_lb_ip   = module.load_balancer.control_plane_lb_ip
-  control_plane_endpoints = module.control_plane.control_plane_ips
-  worker_endpoints      = module.workers.worker_nodes_ips
+  cluster_name                  = var.kubernetes_cluster_name
+  control_plane_lb_ip           = module.load_balancer.control_plane_lb_ip
+  control_plane_endpoints       = module.control_plane.control_plane_ips
+  worker_endpoints              = module.workers.worker_nodes_ips
   control_plane_dns_nameservers = var.control_plane_dns_nameservers
 
   # Versions
-  talos_version         = var.talos_version
-  kubernetes_version    = var.kubernetes_cluster_version
+  talos_version      = var.talos_version
+  kubernetes_version = var.kubernetes_cluster_version
 
   # Dependencies on compute resources
-  control_plane_nodes   = module.control_plane.control_plane_ips
-  worker_nodes          = module.workers.worker_node_internal_ips
+  control_plane_nodes = module.control_plane.control_plane_ips
+  worker_nodes        = module.workers.worker_node_internal_ips
 
   # Output configuration
-  output_dir            = "./talos-config"
-  save_configs_locally  = true
-  bootstrap_cluster     = true
+  output_dir           = "./talos-config"
+  save_configs_locally = true
+  bootstrap_cluster    = true
 }
 
 ##############################
 # 6. Storage Setup
 ##############################
 module "storage" {
-  source                 = "../modules/storage"
+  source                  = "../modules/storage"
   kubernetes_cluster_name = var.kubernetes_cluster_name
 
   # Worker node information for volume attachment
-  worker_instance_ids    = module.workers.worker_nodes_instance_ids
+  worker_instance_ids = module.workers.worker_nodes_instance_ids
 
   # Volume configuration
-  volume_count           = var.storage_volume_count
-  volume_size            = var.storage_volume_size
-  volume_type            = var.storage_volume_type
-  availability_zone      = var.storage_availability_zone
+  volume_count      = var.storage_volume_count
+  volume_size       = var.storage_volume_size
+  volume_type       = var.storage_volume_type
+  availability_zone = var.storage_availability_zone
 
   # Mount configuration
-  mount_path             = var.storage_mount_path
+  mount_path = var.storage_mount_path
 
   # StorageClass configuration
-  create_storage_class   = true
+  create_storage_class       = true
   make_default_storage_class = true
-  storage_reclaim_policy = "Delete"
-  fs_type                = "ext4"
+  storage_reclaim_policy     = "Delete"
+  fs_type                    = "ext4"
 
   # Dependency to ensure Kubernetes is available
-  kubernetes_dependency  = module.talos_config
+  kubernetes_dependency = module.talos_config
 }
 ##############################
diff --git a/kubernetes/provider.tf b/kubernetes/provider.tf
@@ -17,12 +17,16 @@ terraform {
       version = "~> 2.24.0"
     }
   }
+
+  backend "pg" {}
 }
 
 provider "talos" {}
 
 provider "openstack" {
-  cloud = "openstack"
+  auth_url                      = var.os_auth_url
+  application_credential_id     = var.os_application_credential_id
+  application_credential_secret = var.os_application_credential_secret
 }
 
 provider "kubernetes" {