From 36833c06e8149615b65328741778ffdd0197bbfe Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 14:50:34 +0200 Subject: [PATCH 1/8] Fixed a bug regarding the Kubernetes storage --- .github/workflows/validate_misp.yml | 1 + misp/main.tf | 14 ++++++++++++-- misp/provider.tf | 6 +++++- misp/variabels.tf | 27 +++++++++++++++++++++++---- modules/misp/provider.tf | 8 +------- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.github/workflows/validate_misp.yml b/.github/workflows/validate_misp.yml index f755921..e277032 100644 --- a/.github/workflows/validate_misp.yml +++ b/.github/workflows/validate_misp.yml @@ -8,6 +8,7 @@ on: - 'misp/**' - 'modules/misp/**' - 'modules/security/**' + - 'modules/ssh_key/**' workflow_dispatch: jobs: diff --git a/misp/main.tf b/misp/main.tf index 2b39396..b1f0e93 100644 --- a/misp/main.tf +++ b/misp/main.tf @@ -1,4 +1,14 @@ # test +# Look up the cluster network directly by ID +data "openstack_networking_network_v2" "cluster_network" { + name = var.cluster_network_name +} + +# Look up the cluster subnet directly by ID +data "openstack_networking_subnet_v2" "cluster_subnet" { + name = var.cluster_subnet_name +} + ############################## # 1. Security and SSH Key Setup ############################## @@ -22,9 +32,9 @@ module "misp" { kubernetes_cluster_name = var.kubernetes_cluster_name misp_flavor_name = var.misp_flavor_name misp_image_name = var.misp_image_name - internal_network_id = var.cluster_network_id + internal_network_id = data.openstack_networking_network_v2.cluster_network.id public_network_name = var.public_network_name - cluster_subnet_id = var.cluster_subnet_id + cluster_subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id ssh_key_name = module.ssh_key.ssh_key_name ssh_public_key = module.ssh_key.public_key allow_all_egress_sg_id = module.security.allow_all_egress_id diff --git a/misp/provider.tf b/misp/provider.tf index 6df9800..dac4518 100644 --- a/misp/provider.tf +++ b/misp/provider.tf @@ -9,10 +9,14 @@ terraform { version = "~> 2.1.0" } } + + backend "pg" {} } provider "openstack" { - cloud = "openstack" + auth_url = var.os_auth_url + application_credential_id = var.os_application_credential_id + application_credential_secret = var.os_application_credential_secret } diff --git a/misp/variabels.tf b/misp/variabels.tf index 6dfa09d..15d84f4 100644 --- a/misp/variabels.tf +++ b/misp/variabels.tf @@ -1,3 +1,22 @@ + +# Credentials +variable "os_auth_url" { + type = string + sensitive = true +} + +variable "os_application_credential_id" { + type = string + sensitive = true +} + +variable "os_application_credential_secret" { + type = string + sensitive = true +} + + + ############################## # Common Variables ############################## @@ -6,13 +25,13 @@ variable "kubernetes_cluster_name" { type = string } -variable "cluster_network_id" { - description = "ID of the internal cluster network" +variable "cluster_network_name" { + description = "The name of the internal cluster network" type = string } -variable "cluster_subnet_id" { - description = "ID of the internal cluster subnet" +variable "cluster_subnet_name" { + description = "The name of the internal cluster subnet" type = string } diff --git a/modules/misp/provider.tf b/modules/misp/provider.tf index 6df9800..ef5ad47 100644 --- a/modules/misp/provider.tf +++ b/modules/misp/provider.tf @@ -9,10 +9,4 @@ terraform { version = "~> 2.1.0" } } -} - -provider "openstack" { - cloud = "openstack" -} - - +} \ No newline at end of file From 70664e990a4075f29a12c222c402a4f53fedd926 Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 14:55:47 +0200 Subject: [PATCH 2/8] Fixed a bug regarding the Kubernetes storage --- .github/workflows/deploy_misp.yml | 83 +++++++++++++++++++++++++++++++ misp/main.tf | 1 - 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy_misp.yml diff --git a/.github/workflows/deploy_misp.yml b/.github/workflows/deploy_misp.yml new file mode 100644 index 0000000..ddf166e --- /dev/null +++ b/.github/workflows/deploy_misp.yml @@ -0,0 +1,83 @@ +name: Deploy MISP module + +on: + push: + branches: + - 'main' + - 'maja' + paths: + - 'misp/**' + - 'modules/misp/**' + - 'modules/security/**' + - 'modules/ssh_key/**' + workflow_dispatch: + +env: + WORKING_DIR: ./misp + +jobs: + + dev: + runs-on: dev + env: + PG_CONN_STR: ${{ secrets.PG_CONN_STR_DEV }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Terraform init + run: terraform init + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform workspace select or create misp + run: terraform workspace select -or-create misp + working-directory: ${{ env.WORKING_DIR }} + + - name: Create tfvars file + run: | + cat < terraform.tfvars + ${{ secrets.MISP_TFVARS_DEV }}" + EOF + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform plan + run: terraform plan -var "os_auth_url=${{ secrets.OS_AUTH_URL_DEV }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_DEV }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_DEV }}" + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform apply + run: terraform apply -auto-approve -var "os_auth_url=${{ secrets.OS_AUTH_URL_DEV }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_DEV }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_DEV }}" + working-directory: ${{ env.WORKING_DIR }} + + staging: + needs: dev + runs-on: stage + env: + PG_CONN_STR: ${{ secrets.PG_CONN_STR_STAGE }} + + steps: + - name: Chekcout + uses: actions/checkout@v4 + + - name: Terraform init + run: terraform init + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform workspace select or create misp + run: terraform workspace select -or-create misp + working-directory: ${{ env.WORKING_DIR }} + + - name: Create tfvars file + run: | + cat < terraform.tfvars + ${{ secrets.MISP_TFVARS_STAGE }}" + EOF + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform plan + run: terraform plan -var "os_auth_url=${{ secrets.OS_AUTH_URL_STAGE }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_STAGE }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_STAGE }}" + working-directory: ${{ env.WORKING_DIR }} + + - name: Terraform apply + run: terraform apply -auto-approve -var "os_auth_url=${{ secrets.OS_AUTH_URL_STAGE }}" -var "os_application_credential_id=${{ secrets.OS_APPLICATION_CREDENTIAL_ID_STAGE }}" -var "os_application_credential_secret=${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_STAGE }}" + working-directory: ${{ env.WORKING_DIR }} \ No newline at end of file diff --git a/misp/main.tf b/misp/main.tf index b1f0e93..31fb909 100644 --- a/misp/main.tf +++ b/misp/main.tf @@ -1,4 +1,3 @@ -# test # Look up the cluster network directly by ID data "openstack_networking_network_v2" "cluster_network" { name = var.cluster_network_name From 46f81a22411df79ca095fb6357ec9e410b3cb470 Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 15:01:47 +0200 Subject: [PATCH 3/8] Fixed a bug regarding the creation of the tfvars files in deploy_kubernetes.yml and deploy_misp.yml --- .github/workflows/deploy_kubernetes.yml | 4 ++-- .github/workflows/deploy_misp.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_kubernetes.yml b/.github/workflows/deploy_kubernetes.yml index 1cbfca5..36ca7b7 100644 --- a/.github/workflows/deploy_kubernetes.yml +++ b/.github/workflows/deploy_kubernetes.yml @@ -42,7 +42,7 @@ jobs: - name: Create tfvars file run: | cat < terraform.tfvars - ${{ secrets.KUBERNETES_TFVARS_DEV }}" + ${{ secrets.KUBERNETES_TFVARS_DEV }} EOF working-directory: ${{ env.WORKING_DIR }} @@ -101,7 +101,7 @@ jobs: - name: Create tfvars file run: | cat < terraform.tfvars - ${{ secrets.KUBERNETES_TFVARS_STAGE }}" + ${{ secrets.KUBERNETES_TFVARS_STAGE }} EOF working-directory: ${{ env.WORKING_DIR }} diff --git a/.github/workflows/deploy_misp.yml b/.github/workflows/deploy_misp.yml index ddf166e..0da0230 100644 --- a/.github/workflows/deploy_misp.yml +++ b/.github/workflows/deploy_misp.yml @@ -37,7 +37,7 @@ jobs: - name: Create tfvars file run: | cat < terraform.tfvars - ${{ secrets.MISP_TFVARS_DEV }}" + ${{ secrets.MISP_TFVARS_DEV }} EOF working-directory: ${{ env.WORKING_DIR }} @@ -70,7 +70,7 @@ jobs: - name: Create tfvars file run: | cat < terraform.tfvars - ${{ secrets.MISP_TFVARS_STAGE }}" + ${{ secrets.MISP_TFVARS_STAGE }} EOF working-directory: ${{ env.WORKING_DIR }} From ec7059f34186f18b495c567c47789edc2491667c Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 15:16:01 +0200 Subject: [PATCH 4/8] Testing automatic deployment of the MISP module --- misp/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/misp/main.tf b/misp/main.tf index 31fb909..b1f0e93 100644 --- a/misp/main.tf +++ b/misp/main.tf @@ -1,3 +1,4 @@ +# test # Look up the cluster network directly by ID data "openstack_networking_network_v2" "cluster_network" { name = var.cluster_network_name From 8e30cd9e46f5f324eb569c20362bcd3a4e658af5 Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 16:08:03 +0200 Subject: [PATCH 5/8] Testing automatic deployment of the MISP module --- misp/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/misp/main.tf b/misp/main.tf index b1f0e93..31fb909 100644 --- a/misp/main.tf +++ b/misp/main.tf @@ -1,4 +1,3 @@ -# test # Look up the cluster network directly by ID data "openstack_networking_network_v2" "cluster_network" { name = var.cluster_network_name From fbfc270126c106919c9e07de551d09bdbc2e81be Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 16:41:03 +0200 Subject: [PATCH 6/8] Storing the KUBECONFIG file from the Kubernetes cluster deployment --- modules/storage/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/storage/main.tf b/modules/storage/main.tf index cef39ab..745bfb9 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -32,6 +32,9 @@ resource "null_resource" "kubernetes_readiness" { sleep 10 done + # Store the KUBECONFIG file + cat $KUBECONFIG &>/home/runner_user/test1 + # Wait for API server to be responsive echo "Waiting for Kubernetes API to be available..." max_attempts=30 From bd23ec7ac381158b8d99f0569c67799a0c2ce436 Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 16:56:45 +0200 Subject: [PATCH 7/8] Testing automatic deployment of the MISP module --- misp/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/misp/main.tf b/misp/main.tf index 31fb909..b1f0e93 100644 --- a/misp/main.tf +++ b/misp/main.tf @@ -1,3 +1,4 @@ +# test # Look up the cluster network directly by ID data "openstack_networking_network_v2" "cluster_network" { name = var.cluster_network_name From 8f7688c34742c195de4db9a54745192d0c20588b Mon Sep 17 00:00:00 2001 From: Maja Date: Wed, 7 May 2025 21:47:02 +0200 Subject: [PATCH 8/8] Testing automatic deployment of the MISP module --- .github/workflows/deploy_kubernetes.yml | 27 +++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_kubernetes.yml b/.github/workflows/deploy_kubernetes.yml index 36ca7b7..44dac09 100644 --- a/.github/workflows/deploy_kubernetes.yml +++ b/.github/workflows/deploy_kubernetes.yml @@ -125,14 +125,29 @@ jobs: env: KUBECONFIG: ./talos-config/kubeconfig - # Sometimes there might be DNS problems. This step will stop - # and restart the DNS services to ensure it works as expected + # Sometimes there might be DNS problems. This step + # will try to fix the problems if there are any - name: Fix potential DNS problems run: | - sleep 10 - kubectl delete pod -n kube-system -l k8s-app=kube-dns - sleep 20 - kubectl get pods -n strelka + echo "Waiting initial 90 seconds before first DNS test..." + sleep 90 + + for i in {1..15}; do + + POD_NAME=$(kubectl get pods -n strelka -l app=strelka-frontend -o jsonpath='{.items[0].metadata.name}') + + if kubectl exec -n strelka "$POD_NAME" -- nslookup strelka-postgresdb.strelka.svc.cluster.local; then + echo "DNS test $i succeeded. Exiting loop." + exit 0 + fi + + echo "DNS test $i failed. Restarting kube-dns pods and retrying in 90 seconds..." + kubectl delete pod -n kube-system -l k8s-app=kube-dns + sleep 90 + done + + echo "All DNS tests failed." + exit 1 working-directory: ${{ env.WORKING_DIR }} env: KUBECONFIG: ./talos-config/kubeconfig \ No newline at end of file