From 960311ebd4e4616e74d6d123b6849e135e8f2256 Mon Sep 17 00:00:00 2001 From: phaer Date: Mon, 7 Feb 2022 12:56:13 +0100 Subject: [PATCH 1/6] use yamlencode for k3s configs... ...and remove the now, hopefully unneeded workaround for agent.conf, all values are in config.yaml now --- agents.tf | 26 +++++++------------------- master.tf | 15 ++++++++++----- servers.tf | 18 ++++++++++++------ templates/agent.conf.tpl | 3 --- templates/agent_config.yaml.tpl | 4 ---- templates/master_config.yaml.tpl | 16 ---------------- templates/server_config.yaml.tpl | 16 ---------------- 7 files changed, 29 insertions(+), 69 deletions(-) delete mode 100644 templates/agent.conf.tpl delete mode 100644 templates/agent_config.yaml.tpl delete mode 100644 templates/master_config.yaml.tpl delete mode 100644 templates/server_config.yaml.tpl diff --git a/agents.tf b/agents.tf index ad65ec8..ffba9a5 100644 --- a/agents.tf +++ b/agents.tf @@ -59,27 +59,15 @@ resource "hcloud_server" "agents" { EOT } - # Generating and uploading the angent.conf file - provisioner "file" { - content = templatefile("${path.module}/templates/agent.conf.tpl", { - server_url = "https://${local.first_control_plane_network_ip}:6443" - node_token = random_password.k3s_token.result - }) - destination = "/etc/rancher/k3s/agent.conf" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - # Generating k3s server config file provisioner "file" { - content = templatefile("${path.module}/templates/agent_config.yaml.tpl", { - node_ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) - node_name = self.name + content = yamlencode({ + node-name = self.name + server = "https://${local.first_control_plane_network_ip}:6443" + kubelet-arg = "cloud-provider=external" + flannel-iface = "eth1" + node-ip = local.first_control_plane_network_ip + token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/master.tf b/master.tf index 33fbf57..98d2fc1 100644 --- a/master.tf +++ b/master.tf @@ -59,11 +59,16 @@ resource "hcloud_server" "first_control_plane" { # Generating k3s master config file provisioner "file" { - content = templatefile("${path.module}/templates/master_config.yaml.tpl", { - node_ip = local.first_control_plane_network_ip - token = random_password.k3s_token.result - node_name = self.name - allow_scheduling_on_control_plane = var.allow_scheduling_on_control_plane + content = yamlencode({ + node-name = self.name + cluster-init = true + disable-cloud-controller = true + disable = "servicelb, local-storage" + flannel-iface = "eth1" + kubelet-arg = "cloud-provider=external" + node-ip = local.first_control_plane_network_ip + advertise-address = local.first_control_plane_network_ip + token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/servers.tf b/servers.tf index 3cbdc68..b933793 100644 --- a/servers.tf +++ b/servers.tf @@ -60,12 +60,18 @@ resource "hcloud_server" "control_planes" { # Generating k3s server config file provisioner "file" { - content = templatefile("${path.module}/templates/server_config.yaml.tpl", { - first_control_plane_url = "https://${local.first_control_plane_network_ip}:6443" - node_ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) - token = random_password.k3s_token.result - node_name = self.name - allow_scheduling_on_control_plane = var.allow_scheduling_on_control_plane + content = yamlencode({ + node-name = self.name + server = "https://${local.first_control_plane_network_ip}:6443" + cluster-init = true + disable-cloud-controller = true + disable = "servicelb, local-storage" + flannel-iface = "eth1" + kubelet-arg = "cloud-provider=external" + node-ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + advertise-address = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + tls-san = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/templates/agent.conf.tpl b/templates/agent.conf.tpl deleted file mode 100644 index 047b9cf..0000000 --- a/templates/agent.conf.tpl +++ /dev/null @@ -1,3 +0,0 @@ -SERVER_URL="${server_url}" -NODE_TOKEN="${node_token}" -AGENT_OPTS= \ No newline at end of file diff --git a/templates/agent_config.yaml.tpl b/templates/agent_config.yaml.tpl deleted file mode 100644 index 8f13611..0000000 --- a/templates/agent_config.yaml.tpl +++ /dev/null @@ -1,4 +0,0 @@ -node-ip: ${node_ip} -kubelet-arg: "cloud-provider=external" -flannel-iface: eth1 -node-name: ${node_name} \ No newline at end of file diff --git a/templates/master_config.yaml.tpl b/templates/master_config.yaml.tpl deleted file mode 100644 index 5da9450..0000000 --- a/templates/master_config.yaml.tpl +++ /dev/null @@ -1,16 +0,0 @@ -cluster-init: true -disable-cloud-controller: true -disable: -- servicelb -- local-storage -flannel-iface: eth1 -node-ip: ${node_ip} -advertise-address: ${node_ip} -tls-san: ${node_ip} -kubelet-arg: "cloud-provider=external" -token: ${token} -node-name: ${node_name} -%{ if !allow_scheduling_on_control_plane } -node-taint: -- node-role.kubernetes.io/master:NoSchedule -%{ endif } diff --git a/templates/server_config.yaml.tpl b/templates/server_config.yaml.tpl deleted file mode 100644 index 60bd076..0000000 --- a/templates/server_config.yaml.tpl +++ /dev/null @@ -1,16 +0,0 @@ -server: ${first_control_plane_url} -disable-cloud-controller: true -disable: -- servicelb -- local-storage -flannel-iface: eth1 -node-ip: ${node_ip} -advertise-address: ${node_ip} -tls-san: ${node_ip} -kubelet-arg: "cloud-provider=external" -token: ${token} -node-name: ${node_name} -%{ if !allow_scheduling_on_control_plane } -node-taint: -- node-role.kubernetes.io/master:NoSchedule -%{ endif } From cd63a5bc18839cba59cbb8c9a73eff20f9030e4d Mon Sep 17 00:00:00 2001 From: phaer Date: Mon, 7 Feb 2022 14:43:40 +0100 Subject: [PATCH 2/6] re-add agent.conf... ...because it's not worth the necessary overrides/hacks in the base image to remove it. --- agents.tf | 21 ++++++++++++++++++--- templates/agent.conf.tpl | 3 +++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 templates/agent.conf.tpl diff --git a/agents.tf b/agents.tf index ffba9a5..04bb1e0 100644 --- a/agents.tf +++ b/agents.tf @@ -59,15 +59,30 @@ resource "hcloud_server" "agents" { EOT } - # Generating k3s server config file + + # Generating and uploading the agent.conf file + provisioner "file" { + content = templatefile("${path.module}/templates/agent.conf.tpl", { + server = "https://${local.first_control_plane_network_ip}:6443" + token = random_password.k3s_token.result + }) + destination = "/etc/rancher/k3s/agent.conf" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Generating k3s agent config file provisioner "file" { content = yamlencode({ node-name = self.name - server = "https://${local.first_control_plane_network_ip}:6443" kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" node-ip = local.first_control_plane_network_ip - token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/templates/agent.conf.tpl b/templates/agent.conf.tpl new file mode 100644 index 0000000..fad0449 --- /dev/null +++ b/templates/agent.conf.tpl @@ -0,0 +1,3 @@ +SERVER_URL="${server}" +NODE_TOKEN="${token}" +AGENT_OPTS="" From 797af82081621536d0fa89e220194bf9b12d2ce8 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 8 Feb 2022 08:55:04 +0100 Subject: [PATCH 3/6] fix agent ip --- agents.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents.tf b/agents.tf index 04bb1e0..a0dbd07 100644 --- a/agents.tf +++ b/agents.tf @@ -82,7 +82,7 @@ resource "hcloud_server" "agents" { node-name = self.name kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = local.first_control_plane_network_ip + node-ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) }) destination = "/etc/rancher/k3s/config.yaml" From d6fe4152c96dd408f8ebe4fb52dceddad0d7e215 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 8 Feb 2022 09:05:36 +0100 Subject: [PATCH 4/6] re-add node-taints after rebase --- master.tf | 1 + servers.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/master.tf b/master.tf index 98d2fc1..11c9557 100644 --- a/master.tf +++ b/master.tf @@ -69,6 +69,7 @@ resource "hcloud_server" "first_control_plane" { node-ip = local.first_control_plane_network_ip advertise-address = local.first_control_plane_network_ip token = random_password.k3s_token.result + node-taint = var.allow_scheduling_on_control_plane ? [] : [ "node-role.kubernetes.io/master:NoSchedule" ] }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/servers.tf b/servers.tf index b933793..1545b9a 100644 --- a/servers.tf +++ b/servers.tf @@ -72,6 +72,7 @@ resource "hcloud_server" "control_planes" { advertise-address = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) tls-san = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) token = random_password.k3s_token.result + node-taint = var.allow_scheduling_on_control_plane ? [] : [ "node-role.kubernetes.io/master:NoSchedule" ] }) destination = "/etc/rancher/k3s/config.yaml" From af78d8fc86aa8e68bf994af105b9fe3de7b380ac Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 8 Feb 2022 09:12:16 +0100 Subject: [PATCH 5/6] terraform fmt --- agents.tf | 2 +- master.tf | 2 +- servers.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agents.tf b/agents.tf index a0dbd07..61ace48 100644 --- a/agents.tf +++ b/agents.tf @@ -64,7 +64,7 @@ resource "hcloud_server" "agents" { provisioner "file" { content = templatefile("${path.module}/templates/agent.conf.tpl", { server = "https://${local.first_control_plane_network_ip}:6443" - token = random_password.k3s_token.result + token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/agent.conf" diff --git a/master.tf b/master.tf index 11c9557..e8eed4a 100644 --- a/master.tf +++ b/master.tf @@ -69,7 +69,7 @@ resource "hcloud_server" "first_control_plane" { node-ip = local.first_control_plane_network_ip advertise-address = local.first_control_plane_network_ip token = random_password.k3s_token.result - node-taint = var.allow_scheduling_on_control_plane ? [] : [ "node-role.kubernetes.io/master:NoSchedule" ] + node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" diff --git a/servers.tf b/servers.tf index 1545b9a..9d2cbbc 100644 --- a/servers.tf +++ b/servers.tf @@ -72,7 +72,7 @@ resource "hcloud_server" "control_planes" { advertise-address = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) tls-san = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) token = random_password.k3s_token.result - node-taint = var.allow_scheduling_on_control_plane ? [] : [ "node-role.kubernetes.io/master:NoSchedule" ] + node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" From 4f6812a59a47f20826f1fcedf3ff7863108fe11c Mon Sep 17 00:00:00 2001 From: Paul Haerle Date: Tue, 8 Feb 2022 14:14:23 +0100 Subject: [PATCH 6/6] Use yaml list for disabled k3s features Co-authored-by: Marco Nenciarini --- master.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master.tf b/master.tf index e8eed4a..3d64c9b 100644 --- a/master.tf +++ b/master.tf @@ -63,7 +63,7 @@ resource "hcloud_server" "first_control_plane" { node-name = self.name cluster-init = true disable-cloud-controller = true - disable = "servicelb, local-storage" + disable = ["servicelb", "local-storage"] flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" node-ip = local.first_control_plane_network_ip