From 117812954553cd14864a7e7dbedda3fae55e8522 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Sun, 3 Apr 2022 09:46:55 +0200
Subject: [PATCH 01/19] replaced gofish by brew

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d2d30e7..1b625a3 100644
--- a/README.md
+++ b/README.md
@@ -52,12 +52,12 @@ Follow those simple steps, and your world's cheapest Kube cluster will be up and
 
 First and foremost, you need to have a Hetzner Cloud account. You can sign up for free [here](https://hetzner.com/cloud/).
 
-Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli),  [kubectl](https://kubernetes.io/docs/tasks/tools/) cli, and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [gofish](https://gofi.sh/#install) package manager to install them.
+Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli),  [kubectl](https://kubernetes.io/docs/tasks/tools/) cli, and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem).
 
 ```sh
-gofish install terraform
-gofish install kubectl
-gofish install hcloud
+brew install terraform
+brew install kubectl
+brew install hcloud
 ```
 
 ### 💡 [Do not skip] Creating the terraform.tfvars file

From 4d6afe7d04e8c1cb2a827c3ea4db745f5e9464e9 Mon Sep 17 00:00:00 2001
From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com>
Date: Mon, 4 Apr 2022 21:06:08 +0200
Subject: [PATCH 02/19] feat(agents): add location, label and taint options to
 agent-nodepool

---
 agents.tf                |  5 +++--
 locals.tf                |  6 ++++++
 terraform.tfvars.example | 31 +++++++++++++++++++++++++++++--
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/agents.tf b/agents.tf
index b7f45cc..003fc82 100644
--- a/agents.tf
+++ b/agents.tf
@@ -10,7 +10,7 @@ module "agents" {
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
-  location               = var.location
+  location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2].id
 
@@ -51,7 +51,8 @@ resource "null_resource" "agents" {
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
       node-ip       = module.agents[each.key].private_ipv4_address
-      node-label    = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
+      node-label    = each.value.labels
+      node-taint    = each.value.taints
     })
     destination = "/tmp/config.yaml"
   }
diff --git a/locals.tf b/locals.tf
index 95daf55..439b1b1 100644
--- a/locals.tf
+++ b/locals.tf
@@ -175,6 +175,9 @@ locals {
       format("%s-%s", nodepool_obj.name, index) => {
         nodepool_name : nodepool_obj.name,
         server_type : nodepool_obj.server_type,
+        location : nodepool_obj.location,
+        labels : concat(local.default_labels, nodepool_obj.labels),
+        taints : nodepool_obj.taints,
         index : index
       }
     }
@@ -190,4 +193,7 @@ locals {
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
 
+
+  # Default k3s node labels
+  default_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
 }
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 4340743..8641bd3 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -14,7 +14,6 @@ private_key  = "/home/username/.ssh/id_ed25519"
 
 # These can be customized, or left with the default values
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
-# For Hetzner server types see https://www.hetzner.com/cloud
 location         = "fsn1"       # change to `ash` for us-east Ashburn, Virginia location
 network_region   = "eu-central" # change to `us-east` if location is ash
 
@@ -31,15 +30,35 @@ control_plane_server_type = "cpx11"
 # For single node clusters set this equal to [] or just set the counts to 0.
 # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
 # You can add others at the end of the list if you want.
+# For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
+# For Hetzner server types see https://www.hetzner.com/cloud
 agent_nodepools = [
   {
     name        = "agent-small",
     server_type = "cpx11",
+    location    = "fsn1",
+    labels      = [],
+    taints      = [],
     count       = 2
   },
   {
     name        = "agent-large",
     server_type = "cpx21",
+    location    = "fsn1",
+    labels      = [],
+    taints      = [],
+    count       = 1
+  },
+  {
+    name        = "storage",
+    server_type = "cpx21",
+    location    = "fsn1",
+    labels      = [
+      "node.kubernetes.io/server-usage=storage"
+    ],
+    taints      = [
+      "server-usage=storage:NoSchedule"
+    ],
     count       = 1
   }
 ]
@@ -81,7 +100,7 @@ load_balancer_type        = "lb11"
 # use_cluster_name_in_node_name = false
 
 # Adding extra firewall rules, like opening a port
-# In this example with allow port TCP 5432 for a Postgres service we will open via a nodeport
+# In this example with allow port TCP 5432 for a Postgres service we will open via a nodeport and allow outgoing SMTP traffic on port TCP 465
 # More info on the format here https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall
 # extra_firewall_rules = [
 #   {
@@ -92,6 +111,14 @@ load_balancer_type        = "lb11"
 #       "0.0.0.0/0"
 #     ]
 #   },
+#   {
+#     direction = "out"
+#     protocol  = "tcp"
+#     port      = "465"
+#     destination_ips = [
+#       "0.0.0.0/0"
+#     ]
+#   },
 # ]
 
 # If you want to configure additional Arguments for traefik, enter them here as a list and in the form of traefik CLI arguments; see https://doc.traefik.io/traefik/reference/static-configuration/cli/

From 5e4d82fd95d2e15cc361e2cc2195cb99bc07d13c Mon Sep 17 00:00:00 2001
From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com>
Date: Wed, 6 Apr 2022 20:38:24 +0200
Subject: [PATCH 03/19] wip

---
 control_planes.tf | 27 ++++++++++++++-------------
 init.tf           | 12 ++++++------
 kubeconfig.tf     |  4 ++--
 locals.tf         | 34 +++++++++++++++++++++++++---------
 output.tf         |  4 +++-
 variables.tf      | 20 ++++++++------------
 6 files changed, 58 insertions(+), 43 deletions(-)

diff --git a/control_planes.tf b/control_planes.tf
index d39047f..c24590a 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -1,7 +1,8 @@
 module "control_planes" {
   source = "./modules/host"
 
-  count                  = var.control_plane_count
+  for_each = local.control_plane_nodepools
+
   name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}control-plane"
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
@@ -9,13 +10,13 @@ module "control_planes" {
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
-  location               = var.location
-  server_type            = var.control_plane_server_type
+  location               = each.value.location
+  server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], count.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], length([for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name]) + each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -28,33 +29,33 @@ module "control_planes" {
 }
 
 resource "null_resource" "control_planes" {
-  count = var.control_plane_count
+  for_each = local.control_plane_nodepools
 
   triggers = {
-    control_plane_id = module.control_planes[count.index].id
+    control_plane_id = module.control_planes[each.key].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[count.index].ipv4_address
+    host           = module.control_planes[each.key].ipv4_address
   }
 
   # Generating k3s server config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = module.control_planes[count.index].name
-      server                   = "https://${element(module.control_planes.*.private_ipv4_address, count.index > 0 ? 0 : 1)}:6443"
+      node-name                = module.control_planes[each.key].name
+      server                   = "https://${local.first_control_plane.private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = module.control_planes[count.index].private_ipv4_address
-      advertise-address        = module.control_planes[count.index].private_ipv4_address
-      node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
-      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
+      node-ip                  = module.control_planes[each.key].private_ipv4_address
+      advertise-address        = module.control_planes[each.key].private_ipv4_address
+      node-label               = each.value.labels
+      node-taint               = each.value.taints
     })
     destination = "/tmp/config.yaml"
   }
diff --git a/init.tf b/init.tf
index 1e8eb7e..884fbe6 100644
--- a/init.tf
+++ b/init.tf
@@ -3,21 +3,21 @@ resource "null_resource" "first_control_plane" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[0].ipv4_address
+    host           = local.first_control_plane.ipv4_address
   }
 
   # Generating k3s master config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = module.control_planes[0].name
+      node-name                = local.first_control_plane.name
       token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = module.control_planes[0].private_ipv4_address
-      advertise-address        = module.control_planes[0].private_ipv4_address
+      node-ip                  = local.first_control_plane.private_ipv4_address
+      advertise-address        = local.first_control_plane.private_ipv4_address
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
       node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
@@ -66,7 +66,7 @@ resource "null_resource" "kustomization" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[0].ipv4_address
+    host           = local.first_control_plane.ipv4_address
   }
 
   # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
@@ -97,7 +97,7 @@ resource "null_resource" "kustomization" {
         name                       = "${var.cluster_name}-traefik"
         load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6
         load_balancer_type         = var.load_balancer_type
-        location                   = var.location
+        location                   = var.load_balancer_location
         traefik_acme_tls           = var.traefik_acme_tls
         traefik_acme_email         = var.traefik_acme_email
         traefik_additional_options = var.traefik_additional_options
diff --git a/kubeconfig.tf b/kubeconfig.tf
index 1c5cee5..b4969b5 100644
--- a/kubeconfig.tf
+++ b/kubeconfig.tf
@@ -1,7 +1,7 @@
 
 data "remote_file" "kubeconfig" {
   conn {
-    host        = module.control_planes[0].ipv4_address
+    host        = local.first_control_plane.ipv4_address
     port        = 22
     user        = "root"
     private_key = local.ssh_private_key
@@ -13,7 +13,7 @@ data "remote_file" "kubeconfig" {
 }
 
 locals {
-  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.control_planes[0].ipv4_address)
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", local.first_control_plane.ipv4_address)
   kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
   kubeconfig_data = {
     host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
diff --git a/locals.tf b/locals.tf
index 439b1b1..101e9a6 100644
--- a/locals.tf
+++ b/locals.tf
@@ -1,6 +1,6 @@
 locals {
   # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB
-  is_single_node_cluster = var.control_plane_count + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
+  is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
   ssh_public_key         = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
@@ -169,16 +169,30 @@ locals {
   install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.apply_k3s_selinux)
   install_k3s_agent  = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.apply_k3s_selinux)
 
-  agent_nodepools = merge([
-    for nodepool_obj in var.agent_nodepools : {
-      for index in range(nodepool_obj.count) :
-      format("%s-%s", nodepool_obj.name, index) => {
+  control_plane_nodepools = merge([
+    for pool_index, nodepool_obj in var.control_plane_nodepools : {
+      for node_index in range(nodepool_obj.count) :
+      format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => {
         nodepool_name : nodepool_obj.name,
         server_type : nodepool_obj.server_type,
         location : nodepool_obj.location,
-        labels : concat(local.default_labels, nodepool_obj.labels),
+        labels : concat(local.default_control_plane_labels, nodepool_obj.labels),
         taints : nodepool_obj.taints,
-        index : index
+        index : node_index
+      }
+    }
+  ]...)
+
+  agent_nodepools = merge([
+    for pool_index, nodepool_obj in var.agent_nodepools : {
+      for node_index in range(nodepool_obj.count) :
+      format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => {
+        nodepool_name : nodepool_obj.name,
+        server_type : nodepool_obj.server_type,
+        location : nodepool_obj.location,
+        labels : concat(local.default_agent_labels, nodepool_obj.labels),
+        taints : nodepool_obj.taints,
+        index : node_index
       }
     }
   ]...)
@@ -193,7 +207,9 @@ locals {
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
 
-
   # Default k3s node labels
-  default_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_agent_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+
+  first_control_plane = module.control_planes[keys(module.control_planes)[0]]
 }
diff --git a/output.tf b/output.tf
index 89514ef..798f81f 100644
--- a/output.tf
+++ b/output.tf
@@ -4,7 +4,9 @@ output "cluster_name" {
 }
 
 output "control_planes_public_ipv4" {
-  value       = module.control_planes.*.ipv4_address
+  value = [
+    for obj in module.control_planes : obj.ipv4_address
+  ]
   description = "The public IPv4 addresses of the controlplane server."
 }
 
diff --git a/variables.tf b/variables.tf
index 2b5a694..df7aaf8 100644
--- a/variables.tf
+++ b/variables.tf
@@ -20,26 +20,16 @@ variable "additional_public_keys" {
   default     = []
 }
 
-variable "location" {
-  description = "Default server location"
-  type        = string
-}
-
 variable "network_region" {
   description = "Default region for network"
   type        = string
 }
 
-variable "control_plane_server_type" {
-  description = "Default control plane server type"
+variable "load_balancer_location" {
+  description = "Default load balancer location"
   type        = string
 }
 
-variable "control_plane_count" {
-  description = "Number of control plane nodes."
-  type        = number
-}
-
 variable "load_balancer_type" {
   description = "Default load balancer server type"
   type        = string
@@ -51,6 +41,12 @@ variable "load_balancer_disable_ipv6" {
   default     = false
 }
 
+variable "control_plane_nodepools" {
+  description = "Number of control plane nodes."
+  type        = list(any)
+  default     = []
+}
+
 variable "agent_nodepools" {
   description = "Number of agent nodes."
   type        = list(any)

From 65297f7ded7cce3a05e2f31b091c4f5a80938544 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Sat, 9 Apr 2022 08:40:36 +0200
Subject: [PATCH 04/19] initial success for one control plane nodepool

---
 agents.tf                |  2 +-
 locals.tf                |  4 ++--
 terraform.tfvars.example | 38 +++++++++++++++++++++++++++-----------
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/agents.tf b/agents.tf
index 003fc82..82aa17c 100644
--- a/agents.tf
+++ b/agents.tf
@@ -46,7 +46,7 @@ resource "null_resource" "agents" {
   provisioner "file" {
     content = yamlencode({
       node-name     = module.agents[each.key].name
-      server        = "https://${module.control_planes[0].private_ipv4_address}:6443"
+      server        = "https://${local.first_control_plane.private_ipv4_address}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
diff --git a/locals.tf b/locals.tf
index 101e9a6..84cc38b 100644
--- a/locals.tf
+++ b/locals.tf
@@ -202,13 +202,13 @@ locals {
 
   # The first two subnets are respectively the default subnet 10.0.0.0/16 use for potientially anything and 10.1.0.0/16 used for control plane nodes.
   # the rest of the subnets are for agent nodes in each nodepools.
-  network_ipv4_subnets = [for index in range(length(var.agent_nodepools) + 2) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
+  network_ipv4_subnets = [for index in range(length(var.control_plane_nodepools) + length(var.agent_nodepools) + 1) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
 
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
 
   # Default k3s node labels
-  default_agent_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 
   first_control_plane = module.control_planes[keys(module.control_planes)[0]]
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 8641bd3..99199f0 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -14,23 +14,38 @@ private_key  = "/home/username/.ssh/id_ed25519"
 
 # These can be customized, or left with the default values
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
-location         = "fsn1"       # change to `ash` for us-east Ashburn, Virginia location
 network_region   = "eu-central" # change to `us-east` if location is ash
 
 # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
 # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
 # For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA).
-control_plane_count = 3
+# Of course, you can choose any number of node pools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
+# For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
+control_plane_nodepools = [
+  {
+    name        = "control-plane-fsn1",
+    server_type = "cpx11",
+    location    = "fsn1",
+    labels      = [],
+    taints      = [],
+    count       = 2
+  },
+  {
+    name        = "control-plane-nbg1",
+    server_type = "cpx11",
+    location    = "nbg1",
+    labels      = [],
+    taints      = [],
+    count       = 1
+  }
+]
 
-# The type of control plane nodes, see https://www.hetzner.com/cloud, the minimum instance supported is cpx11 (just a few cents more than cx11)
-control_plane_server_type = "cpx11"
 
 # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
 # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
 # For single node clusters set this equal to [] or just set the counts to 0.
 # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want.
-# For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
+# You can add others at the end of the list if you want. For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
 # For Hetzner server types see https://www.hetzner.com/cloud
 agent_nodepools = [
   {
@@ -44,7 +59,7 @@ agent_nodepools = [
   {
     name        = "agent-large",
     server_type = "cpx21",
-    location    = "fsn1",
+    location    = "nbg1",
     labels      = [],
     taints      = [],
     count       = 1
@@ -53,18 +68,19 @@ agent_nodepools = [
     name        = "storage",
     server_type = "cpx21",
     location    = "fsn1",
-    labels      = [
+    labels = [
       "node.kubernetes.io/server-usage=storage"
     ],
-    taints      = [
+    taints = [
       "server-usage=storage:NoSchedule"
     ],
-    count       = 1
+    count = 1
   }
 ]
 
 # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
-load_balancer_type        = "lb11"
+load_balancer_type     = "lb11"
+load_balancer_location = "fsn1"
 
 ### The following values are fully optional
 

From 41177769942921336eaac9a2fae2711af347ac45 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Sat, 9 Apr 2022 09:51:11 +0200
Subject: [PATCH 05/19] ip problem solved

---
 agents.tf         |  6 +++---
 control_planes.tf |  6 +++---
 init.tf           | 10 +++++-----
 kubeconfig.tf     |  4 ++--
 locals.tf         |  2 --
 5 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/agents.tf b/agents.tf
index 82aa17c..538f1e5 100644
--- a/agents.tf
+++ b/agents.tf
@@ -12,11 +12,11 @@ module "agents" {
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2].id
+  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2], each.value.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1], each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -46,7 +46,7 @@ resource "null_resource" "agents" {
   provisioner "file" {
     content = yamlencode({
       node-name     = module.agents[each.key].name
-      server        = "https://${local.first_control_plane.private_ipv4_address}:6443"
+      server        = "https://${module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
diff --git a/control_planes.tf b/control_planes.tf
index c24590a..e2e9349 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -12,11 +12,11 @@ module "control_planes" {
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[1].id
+  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], length([for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name]) + each.value.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1], each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${local.first_control_plane.private_ipv4_address}:6443"
+      server                   = "https://${module.control_planes[each.key].private_ipv4_address == "10.1.0.101" ? "10.1.0.102" : "10.1.0.101"}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
diff --git a/init.tf b/init.tf
index 884fbe6..279f837 100644
--- a/init.tf
+++ b/init.tf
@@ -3,21 +3,21 @@ resource "null_resource" "first_control_plane" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = local.first_control_plane.ipv4_address
+    host           = module.control_planes[keys(module.control_planes)[0]].ipv4_address
   }
 
   # Generating k3s master config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = local.first_control_plane.name
+      node-name                = module.control_planes[keys(module.control_planes)[0]].name
       token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = local.first_control_plane.private_ipv4_address
-      advertise-address        = local.first_control_plane.private_ipv4_address
+      node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
+      advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
       node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
@@ -66,7 +66,7 @@ resource "null_resource" "kustomization" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = local.first_control_plane.ipv4_address
+    host           = module.control_planes[keys(module.control_planes)[0]].ipv4_address
   }
 
   # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
diff --git a/kubeconfig.tf b/kubeconfig.tf
index b4969b5..5089da6 100644
--- a/kubeconfig.tf
+++ b/kubeconfig.tf
@@ -1,7 +1,7 @@
 
 data "remote_file" "kubeconfig" {
   conn {
-    host        = local.first_control_plane.ipv4_address
+    host        = module.control_planes[keys(module.control_planes)[0]].ipv4_address
     port        = 22
     user        = "root"
     private_key = local.ssh_private_key
@@ -13,7 +13,7 @@ data "remote_file" "kubeconfig" {
 }
 
 locals {
-  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", local.first_control_plane.ipv4_address)
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.control_planes[keys(module.control_planes)[0]].ipv4_address)
   kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
   kubeconfig_data = {
     host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
diff --git a/locals.tf b/locals.tf
index 84cc38b..42ef5ee 100644
--- a/locals.tf
+++ b/locals.tf
@@ -210,6 +210,4 @@ locals {
   # Default k3s node labels
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
-
-  first_control_plane = module.control_planes[keys(module.control_planes)[0]]
 }

From b142be2d8416b372a759fb528b596f37ba984870 Mon Sep 17 00:00:00 2001
From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com>
Date: Mon, 11 Apr 2022 17:44:50 +0200
Subject: [PATCH 06/19] fix(control-plane): use correct taints and labels for
 first control plane node and fix existing default taints

---
 init.tf   | 4 ++--
 locals.tf | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/init.tf b/init.tf
index 279f837..fd6ff68 100644
--- a/init.tf
+++ b/init.tf
@@ -18,8 +18,8 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
-      node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
-      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
+      node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
+      node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })
     destination = "/tmp/config.yaml"
   }
diff --git a/locals.tf b/locals.tf
index 42ef5ee..e3feeb0 100644
--- a/locals.tf
+++ b/locals.tf
@@ -177,7 +177,7 @@ locals {
         server_type : nodepool_obj.server_type,
         location : nodepool_obj.location,
         labels : concat(local.default_control_plane_labels, nodepool_obj.labels),
-        taints : nodepool_obj.taints,
+        taints : concat(local.default_control_plane_taints, nodepool_obj.taints),
         index : node_index
       }
     }
@@ -209,5 +209,8 @@ locals {
 
   # Default k3s node labels
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
-  default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+  default_control_plane_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+
+  # Default k3s node taints
+  default_control_plane_taints = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 }

From d7405e92e42b254bcf8740a0d5afced7b15b9df6 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 07:12:42 +0200
Subject: [PATCH 07/19] clarified ssh options

---
 README.md   |  2 +-
 docs/ssh.md | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 docs/ssh.md

diff --git a/README.md b/README.md
index 1b625a3..f7a9970 100644
--- a/README.md
+++ b/README.md
@@ -63,7 +63,7 @@ brew install hcloud
 ### 💡 [Do not skip] Creating the terraform.tfvars file
 
 1. Create a project in your [Hetzner Cloud Console](https://console.hetzner.cloud/), and go to **Security > API Tokens** of that project to grab the API key. Take note of the key! ✅
-2. Either, generate a passphrase-less ed25519 SSH key-pair for your cluster, unless you already have one that you'd like to use. Take note of the respective paths of your private and public keys. Or, for a key-pair with passphrase or a device like a Yubikey, make sure you have have an SSH agent running and your key is loaded (`ssh-add -L` to verify) and set `private_key = null` in the following step. ✅
+2. Generate a passphrase-less ed25519 SSH key-pair for your cluster, take note of the respective paths of your private and public keys. Or, see our detailed [SSH options](https://github.com/kube-hetzner/kube-hetzner/blob/master/docs/ssh.md). ✅
 3. Copy `terraform.tfvars.example` to `terraform.tfvars`, and replace the values from steps 1 and 2. ✅
 4. Make sure you have the latest Terraform version, ideally at least 1.1.0. You can check with `terraform -v`. ✅
 5. (Optional) There are other variables in `terraform.tfvars` that could be customized, like Hetzner region, and the node counts and sizes.
diff --git a/docs/ssh.md b/docs/ssh.md
new file mode 100644
index 0000000..7228b77
--- /dev/null
+++ b/docs/ssh.md
@@ -0,0 +1,24 @@
+Kube-Hetzner requires you to have a recent version of OpenSSH (>=6.5) installed on your client, and the use of a key-pair generated with either of the following algorithms:
+
+- ssh-ed25519 (preferred, and most simple to use without passphrase)
+- rsa-sha2-512
+- rsa-sha2-256
+
+If your key-pair is of the `ssh-ed25519` sort, and without of passphrase, you do not need to do anything else. Just set `public_key` and `private_key` to their respective path values in your terraform.tfvars.
+
+---
+
+Otherwise, for a key-pair with passphrase or a device like a Yubikey, make sure you have have an SSH agent running and your key is loaded with:
+
+```bash
+eval ssh-agent $SHELL
+ssh-add ~/.ssh/my_private-key_id
+```
+
+Verify it is loaded with:
+
+```bash
+ssh-add -l
+```
+
+Then set `private_key = null` in your terraform.tfvars, as it will be read from the ssh-agent automatically.
\ No newline at end of file

From fa2d933e563a36ea0cbbf74a62ddc96f0ad6479f Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 07:40:54 +0200
Subject: [PATCH 08/19] tweaked readme

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index f7a9970..8cd1060 100644
--- a/README.md
+++ b/README.md
@@ -33,9 +33,10 @@ _Please note that we are not affiliated to Hetzner, this is just an open source
 
 - Maintenance free with auto-upgrade to the latest version of MicroOS and k3s.
 - Proper use of the Hetzner private network to minimize latency and remove the need for encryption.
-- Automatic HA with the default setting of three control-plane nodes and two agent nodepools.
+- Automatic HA with the default setting of three control-plane nodes and two agent nodes.
+- Node pools for both control-plane and agent nodes. Different locations possible (super-HA).
 - Ability to add or remove as many nodes as you want while the cluster stays running.
-- Automatic Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
+- Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
 - Tons of flexible configuration options to suits all needs.
 
 _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._

From b45e77e4374af570750af7d4f5bd1f4d6f8fd743 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 14:48:43 +0200
Subject: [PATCH 09/19] fixed ips

---
 control_planes.tf | 5 +++--
 init.tf           | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/control_planes.tf b/control_planes.tf
index e2e9349..607f7aa 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -3,7 +3,7 @@ module "control_planes" {
 
   for_each = local.control_plane_nodepools
 
-  name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}control-plane"
+  name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}${each.value.nodepool_name}"
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
   private_key            = var.private_key
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${module.control_planes[each.key].private_ipv4_address == "10.1.0.101" ? "10.1.0.102" : "10.1.0.101"}:6443"
+      server                   = "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address && length(module.control_planes) > 1 ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
@@ -54,6 +54,7 @@ resource "null_resource" "control_planes" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[each.key].private_ipv4_address
       advertise-address        = module.control_planes[each.key].private_ipv4_address
+      tls-san                  = module.control_planes[each.key].ipv4_address
       node-label               = each.value.labels
       node-taint               = each.value.taints
     })
diff --git a/init.tf b/init.tf
index fd6ff68..3e83242 100644
--- a/init.tf
+++ b/init.tf
@@ -18,6 +18,7 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
+      tls-san                  = module.control_planes[keys(module.control_planes)[0]].ipv4_address
       node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
       node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })

From ef34f4fd7363ff08030a71c5e34ce9912bd02b06 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 15:23:59 +0200
Subject: [PATCH 10/19] tweaked tfvars.example

---
 terraform.tfvars.example | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 99199f0..4815efb 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -16,11 +16,16 @@ private_key  = "/home/username/.ssh/id_ed25519"
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
 network_region   = "eu-central" # change to `us-east` if location is ash
 
-# At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
+# For the control-planes, at least 3 nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
 # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
-# For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA).
-# Of course, you can choose any number of node pools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
+# For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA). It does not matter if they are in the same nodepool or not! So they can be in different locations, and of different types.
+
+# Of course, you can choose any number of nodepools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
 # For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
+
+# IMPORTANT: Once the cluster is created, you can a nodepool count, and even set it to 0, even change its name (if the count is taken to 0), but do not remove a nodepool from the list. 
+# You can add others at the end of the list if you want. In the case of the first nodepool, it must have a minimum count of 1 always!
+# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 control_plane_nodepools = [
   {
     name        = "control-plane-fsn1",
@@ -44,9 +49,10 @@ control_plane_nodepools = [
 # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
 # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
 # For single node clusters set this equal to [] or just set the counts to 0.
+
 # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want. For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
-# For Hetzner server types see https://www.hetzner.com/cloud
+# You can add others at the end of the list if you want.
+# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 agent_nodepools = [
   {
     name        = "agent-small",

From d3bb8b260537f0e0bb04917d523e15da1f565925 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 15:37:50 +0200
Subject: [PATCH 11/19] tweaked tfvars.example

---
 terraform.tfvars.example | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 4815efb..9cc16a2 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -23,9 +23,16 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 # Of course, you can choose any number of nodepools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
 # For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
 
-# IMPORTANT: Once the cluster is created, you can a nodepool count, and even set it to 0, even change its name (if the count is taken to 0), but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want. In the case of the first nodepool, it must have a minimum count of 1 always!
-# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
+# IMPORTANT: Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
+# You can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated.
+# You can freely add others nodepools the end of the list if you want, and increase the count of any.
+# Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
+
+# Before initializing the cluster, you can change all parameters and add or remove any nodepools. 
+# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1.
+
+# Example below:
+
 control_plane_nodepools = [
   {
     name        = "control-plane-fsn1",
@@ -45,14 +52,6 @@ control_plane_nodepools = [
   }
 ]
 
-
-# As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
-# and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
-# For single node clusters set this equal to [] or just set the counts to 0.
-
-# IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want.
-# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 agent_nodepools = [
   {
     name        = "agent-small",
@@ -86,6 +85,7 @@ agent_nodepools = [
 
 # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
 load_balancer_type     = "lb11"
+
 load_balancer_location = "fsn1"
 
 ### The following values are fully optional

From f1a98c72902ccc7885b859dd2025989bf867ed3d Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 17:35:09 +0200
Subject: [PATCH 12/19] fixed single node cluster

---
 README.md                | 9 ++-------
 control_planes.tf        | 3 +--
 init.tf                  | 1 -
 locals.tf                | 4 +++-
 output.tf                | 4 +++-
 terraform.tfvars.example | 9 +++++----
 6 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 8cd1060..a40325f 100644
--- a/README.md
+++ b/README.md
@@ -170,14 +170,9 @@ spec:
 
 <summary>Single-node cluster</summary>
 
-Running a development cluster on a single node, without any high-availability is possible as well.
-In this case, we don't deploy an external load-balancer, but use [k3s service load balancer](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) on the host itself and open up port 80 & 443 in the firewall.
+Running a development cluster on a single node, without any high-availability is possible as well. You need one control plane nodepool with a count of 1, and one agent nodepool with a count of 0.
 
-``` terraform
-control_plane_count = 1
-allow_scheduling_on_control_plane = true
-agent_nodepools = []
-```
+In this case, we don't deploy an external load-balancer, but use the default [k3s service load balancer](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) on the host itself and open up port 80 & 443 in the firewall (done automatically).
 
 </details>
 
diff --git a/control_planes.tf b/control_planes.tf
index 607f7aa..37655a7 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address && length(module.control_planes) > 1 ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
+      server                   = length(module.control_planes) == 1 ? null : "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
@@ -54,7 +54,6 @@ resource "null_resource" "control_planes" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[each.key].private_ipv4_address
       advertise-address        = module.control_planes[each.key].private_ipv4_address
-      tls-san                  = module.control_planes[each.key].ipv4_address
       node-label               = each.value.labels
       node-taint               = each.value.taints
     })
diff --git a/init.tf b/init.tf
index 3e83242..fd6ff68 100644
--- a/init.tf
+++ b/init.tf
@@ -18,7 +18,6 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
-      tls-san                  = module.control_planes[keys(module.control_planes)[0]].ipv4_address
       node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
       node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })
diff --git a/locals.tf b/locals.tf
index e3feeb0..bda11db 100644
--- a/locals.tf
+++ b/locals.tf
@@ -211,6 +211,8 @@ locals {
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
 
+  allow_scheduling_on_control_plane = local.is_single_node_cluster ? true : var.allow_scheduling_on_control_plane
+
   # Default k3s node taints
-  default_control_plane_taints = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+  default_control_plane_taints = concat([], local.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 }
diff --git a/output.tf b/output.tf
index 798f81f..1ea34fd 100644
--- a/output.tf
+++ b/output.tf
@@ -19,7 +19,9 @@ output "agents_public_ipv4" {
 
 output "load_balancer_public_ipv4" {
   description = "The public IPv4 address of the Hetzner load balancer"
-  value       = local.is_single_node_cluster ? module.control_planes[0].ipv4_address : var.traefik_enabled == false ? null : data.hcloud_load_balancer.traefik[0].ipv4
+  value = local.is_single_node_cluster ? [
+    for obj in module.control_planes : obj.ipv4_address
+  ][0] : var.traefik_enabled == false ? null : data.hcloud_load_balancer.traefik[0].ipv4
 }
 
 output "kubeconfig_file" {
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 9cc16a2..e62200b 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -28,8 +28,9 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 # You can freely add others nodepools the end of the list if you want, and increase the count of any.
 # Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
 
-# Before initializing the cluster, you can change all parameters and add or remove any nodepools. 
-# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1.
+# Before initializing the cluster, you can change all parameters and add or remove any nodepools.
+
+# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1, and one agent nodepool with a count of 0.
 
 # Example below:
 
@@ -59,7 +60,7 @@ agent_nodepools = [
     location    = "fsn1",
     labels      = [],
     taints      = [],
-    count       = 2
+    count       = 1
   },
   {
     name        = "agent-large",
@@ -106,7 +107,7 @@ load_balancer_location = "fsn1"
 # metrics_server_enabled = false
 
 # If you want to allow non-control-plane workloads to run on the control-plane nodes set "true" below. The default is "false".
-# Also good for single node clusters.
+# True by default for single node clusters.
 # allow_scheduling_on_control_plane = true
 
 # If you want to disable automatic upgrade of k3s, you can set this to false, default is "true".

From 9e1814cd9bd3f096f82eeccab9ca104472a33cfd Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 18:00:37 +0200
Subject: [PATCH 13/19] tweaked tfvars.example

---
 terraform.tfvars.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index e62200b..988785c 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -25,7 +25,7 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 
 # IMPORTANT: Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
 # You can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated.
-# You can freely add others nodepools the end of the list if you want, and increase the count of any.
+# Once the cluster is initialized, you cannot add more control plane nodepools. You can freely add others agent nodepools the end of the list if you want! 
 # Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
 
 # Before initializing the cluster, you can change all parameters and add or remove any nodepools.

From b23bb14415b581ae6b77baf9477d977a8e141642 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 19:31:31 +0200
Subject: [PATCH 14/19] tweaked readme

---
 README.md | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a40325f..371299c 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,8 @@
 
 ## About The Project
 
+ℹ️ _For the moment this README is not synced with the current release, as we currently use the master branch for active development, please find the one that aligns with the latest stable release [here](https://argocd-image-updater.readthedocs.io/en/stable/)._
+
 [Hetzner Cloud](https://hetzner.com) is a good cloud provider that offers very affordable prices for cloud instances, with data center locations in both Europe and the US.
 
 The goal of this project is to create an optimal and highly optimized Kubernetes installation that is easily maintained, secure, and automatically upgrades. We aimed for functionality as close as possible to GKE's auto-pilot.
@@ -34,8 +36,9 @@ _Please note that we are not affiliated to Hetzner, this is just an open source
 - Maintenance free with auto-upgrade to the latest version of MicroOS and k3s.
 - Proper use of the Hetzner private network to minimize latency and remove the need for encryption.
 - Automatic HA with the default setting of three control-plane nodes and two agent nodes.
-- Node pools for both control-plane and agent nodes. Different locations possible (super-HA).
-- Ability to add or remove as many nodes as you want while the cluster stays running.
+- Super-HA: Nodepools for both control-plane and agent nodes can be in different locations.
+- Possibility to have a single node cluster with a proper ingress controller (Traefik).
+- Ability to add nodes and nodepools when the cluster running.
 - Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
 - Tons of flexible configuration options to suits all needs.
 
@@ -98,6 +101,14 @@ To scale the number of nodes up or down, just make sure to properly `kubectl dra
 
 About nodepools, `terraform.tfvars.example` has clear example how to configure them.
 
+There are some limitations (to scaling down mainly) that you need to be aware of:
+
+_Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1), you can also rename a nodepool (if the count is taken to 0), but should not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated to the nodes, and how the Hetzner API works._
+
+_However when a cluster is already initialized, you cannot add more control plane nodepools (you can only add nodes to the already created control plane nodepools). As for the angent nodepools, you can freely add others agent nodepools the end of the list if you want._
+
+_Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state._
+
 ## High Availability
 
 By default, we have 3 control planes and 3 agents configured, with automatic upgrades and reboots of the nodes.

From ca8a42722ccd45af324a269e8491bcb6531bdc23 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 19:32:53 +0200
Subject: [PATCH 15/19] tweaked readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 371299c..00b2899 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@
 
 ## About The Project
 
-ℹ️ _For the moment this README is not synced with the current release, as we currently use the master branch for active development, please find the one that aligns with the latest stable release [here](https://argocd-image-updater.readthedocs.io/en/stable/)._
+ℹ️ _For the moment this README is not synced with the current release, as we currently use the master branch for active development, please find the one that aligns with the latest stable release [here](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/tree/v0.1.3)._
 
 [Hetzner Cloud](https://hetzner.com) is a good cloud provider that offers very affordable prices for cloud instances, with data center locations in both Europe and the US.
 

From 6d33e69b697a9e45e63e270e4d6ff0c7259b8017 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Wed, 13 Apr 2022 07:17:01 +0200
Subject: [PATCH 16/19] placement group fix attempt

---
 agents.tf         |  2 +-
 control_planes.tf |  2 +-
 locals.tf         |  5 ++++-
 main.tf           | 17 ++++++++++-------
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/agents.tf b/agents.tf
index 538f1e5..38000be 100644
--- a/agents.tf
+++ b/agents.tf
@@ -9,7 +9,7 @@ module "agents" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = hcloud_placement_group.k3s.id
+  placement_group_id     = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id
diff --git a/control_planes.tf b/control_planes.tf
index 37655a7..b637d93 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -9,7 +9,7 @@ module "control_planes" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = hcloud_placement_group.k3s.id
+  placement_group_id     = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
diff --git a/locals.tf b/locals.tf
index bda11db..addbbcb 100644
--- a/locals.tf
+++ b/locals.tf
@@ -1,6 +1,9 @@
 locals {
   # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB
-  is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
+  total_node_count       = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0]))
+  control_plane_count    = sum(concat([for v in var.control_plane_nodepools : v.count], [0]))
+  agent_count            = sum(concat([for v in var.agent_nodepools : v.count], [0]))
+  is_single_node_cluster = local.total_node_count == 1
   ssh_public_key         = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
diff --git a/main.tf b/main.tf
index e074930..6bdabfd 100644
--- a/main.tf
+++ b/main.tf
@@ -36,13 +36,16 @@ resource "hcloud_firewall" "k3s" {
   }
 }
 
-resource "hcloud_placement_group" "k3s" {
-  name = var.cluster_name
-  type = "spread"
-  labels = {
-    "provisioner" = "terraform",
-    "engine"      = "k3s"
-  }
+resource "hcloud_placement_group" "control_plane" {
+  count = ceil(local.control_plane_count / 10)
+  name  = "${var.cluster_name}-${count.index + 1}"
+  type  = "spread"
+}
+
+resource "hcloud_placement_group" "agent" {
+  count = ceil(local.agent_count / 10)
+  name  = "${var.cluster_name}-${count.index + 1}"
+  type  = "spread"
 }
 
 data "hcloud_load_balancer" "traefik" {

From b17f1569f2297f3fb2832daa2090a1636212836d Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Wed, 13 Apr 2022 09:29:29 +0200
Subject: [PATCH 17/19] placement group fix

---
 agents.tf                | 2 +-
 control_planes.tf        | 2 +-
 main.tf                  | 7 ++++---
 terraform.tfvars.example | 7 +++++--
 variables.tf             | 7 ++++++-
 5 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/agents.tf b/agents.tf
index 38000be..1cda0ea 100644
--- a/agents.tf
+++ b/agents.tf
@@ -9,7 +9,7 @@ module "agents" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
+  placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.agent.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id
diff --git a/control_planes.tf b/control_planes.tf
index b637d93..d73d6c8 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -9,7 +9,7 @@ module "control_planes" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
+  placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
diff --git a/main.tf b/main.tf
index 6bdabfd..c09a324 100644
--- a/main.tf
+++ b/main.tf
@@ -38,13 +38,13 @@ resource "hcloud_firewall" "k3s" {
 
 resource "hcloud_placement_group" "control_plane" {
   count = ceil(local.control_plane_count / 10)
-  name  = "${var.cluster_name}-${count.index + 1}"
+  name  = "${var.cluster_name}-control-plane-${count.index + 1}"
   type  = "spread"
 }
 
 resource "hcloud_placement_group" "agent" {
   count = ceil(local.agent_count / 10)
-  name  = "${var.cluster_name}-${count.index + 1}"
+  name  = "${var.cluster_name}-agent-${count.index + 1}"
   type  = "spread"
 }
 
@@ -74,9 +74,10 @@ resource "null_resource" "destroy_traefik_loadbalancer" {
     local_sensitive_file.kubeconfig,
     null_resource.control_planes[0],
     hcloud_network_subnet.subnet,
+    hcloud_placement_group.control_plane,
+    hcloud_placement_group.agent,
     hcloud_network.k3s,
     hcloud_firewall.k3s,
-    hcloud_placement_group.k3s,
     hcloud_ssh_key.k3s
   ]
 }
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 988785c..60ce5b5 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -84,9 +84,8 @@ agent_nodepools = [
   }
 ]
 
-# That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
+# LB location and type, the latter will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
 load_balancer_type     = "lb11"
-
 load_balancer_location = "fsn1"
 
 ### The following values are fully optional
@@ -147,3 +146,7 @@ load_balancer_location = "fsn1"
 # If you want to configure additional Arguments for traefik, enter them here as a list and in the form of traefik CLI arguments; see https://doc.traefik.io/traefik/reference/static-configuration/cli/
 # Example: traefik_additional_options = ["--log.level=DEBUG", "--tracing=true"]
 # traefik_additional_options = []
+
+# If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/
+# That may be useful if you need to deploy more than 500 nodes! The default is "false".
+# placement_group_disable = true
diff --git a/variables.tf b/variables.tf
index df7aaf8..365ad40 100644
--- a/variables.tf
+++ b/variables.tf
@@ -138,5 +138,10 @@ variable "cluster_name" {
 variable "traefik_additional_options" {
   type    = list(string)
   default = []
-
+}
+
+variable "placement_group_disable" {
+  type        = bool
+  default     = false
+  description = "Whether to disable placement groups"
 }

From 0f52a4e53525789acfb8ccaf0a60f933d098bcda Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Wed, 13 Apr 2022 11:56:09 +0200
Subject: [PATCH 18/19] subnet dissociated

---
 agents.tf         |  6 ++----
 control_planes.tf |  4 ++--
 locals.tf         |  2 +-
 main.tf           | 13 ++++++++++++-
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/agents.tf b/agents.tf
index 1cda0ea..4211d81 100644
--- a/agents.tf
+++ b/agents.tf
@@ -12,11 +12,9 @@ module "agents" {
   placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.agent.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id
+  ipv4_subnet_id         = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
 
-  # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
-  # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1], each.value.index + 101)
+  private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 1].ip_range, each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
diff --git a/control_planes.tf b/control_planes.tf
index d73d6c8..d3bc0f3 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -12,11 +12,11 @@ module "control_planes" {
   placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
+  ipv4_subnet_id         = hcloud_network_subnet.control_plane[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0]].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1], each.value.index + 101)
+  private_ipv4 = cidrhost(hcloud_network_subnet.control_plane[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
diff --git a/locals.tf b/locals.tf
index addbbcb..c71cd25 100644
--- a/locals.tf
+++ b/locals.tf
@@ -205,7 +205,7 @@ locals {
 
   # The first two subnets are respectively the default subnet 10.0.0.0/16 use for potientially anything and 10.1.0.0/16 used for control plane nodes.
   # the rest of the subnets are for agent nodes in each nodepools.
-  network_ipv4_subnets = [for index in range(length(var.control_plane_nodepools) + length(var.agent_nodepools) + 1) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
+  network_ipv4_subnets = [for index in range(256) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
 
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
diff --git a/main.tf b/main.tf
index c09a324..dd27a1d 100644
--- a/main.tf
+++ b/main.tf
@@ -13,7 +13,18 @@ resource "hcloud_network" "k3s" {
   ip_range = local.network_ipv4_cidr
 }
 
-resource "hcloud_network_subnet" "subnet" {
+# We start from the end of the subnets cird array, 
+# as we would have fewer control plane nodepools, than angent ones.
+resource "hcloud_network_subnet" "control_plane" {
+  count        = length(local.control_plane_nodepools)
+  network_id   = hcloud_network.k3s.id
+  type         = "cloud"
+  network_zone = var.network_region
+  ip_range     = local.network_ipv4_subnets[255 - count.index]
+}
+
+# Here we start at the beginning of the subnets cird array
+resource "hcloud_network_subnet" "agent" {
   count        = length(local.network_ipv4_subnets)
   network_id   = hcloud_network.k3s.id
   type         = "cloud"

From 2e75234b7c97600dbd63591b926eb9f2647f55e2 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Wed, 13 Apr 2022 14:14:22 +0200
Subject: [PATCH 19/19] subnet dissociated fix

---
 README.md                | 10 +++-------
 agents.tf                |  8 ++++----
 control_planes.tf        |  4 ++--
 init.tf                  |  2 +-
 main.tf                  |  5 +++--
 terraform.tfvars.example | 22 +++++++++++++++++-----
 6 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 00b2899..f1b7f89 100644
--- a/README.md
+++ b/README.md
@@ -97,17 +97,13 @@ _Once you start with Terraform, it's best not to change the state manually in He
 
 ### Scaling Nodes
 
-To scale the number of nodes up or down, just make sure to properly `kubectl drain` the nodes in question first if scaling down. Then just edit your `terraform.tfvars` and re-apply terraform with `terraform apply -auto-approve`.
-
-About nodepools, `terraform.tfvars.example` has clear example how to configure them.
+Two things can be scaled, the number of nodepools or the count of nodes in these nodepools. You have two list of nodepools you can add to in terraform.tfvars, the control plane nodepool list and the agent nodepool list. Both combined cannot exceed 255 nodepools (you extremely unlikely to reach this limit). As for the count of nodes per nodepools, if you raise your limits in Hetzner, you can have up to 64,670 nodes per nodepool (also very unlikely to need that much).
 
 There are some limitations (to scaling down mainly) that you need to be aware of:
 
-_Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1), you can also rename a nodepool (if the count is taken to 0), but should not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated to the nodes, and how the Hetzner API works._
+_Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1), you can also rename a nodepool (if the count is taken to 0), but should not remove a nodepool from the list after the cluster is created. This is due to how subnets and IPs are allocated. The only nodepools you can remove are the ones at the end of each list of nodepools._
 
-_However when a cluster is already initialized, you cannot add more control plane nodepools (you can only add nodes to the already created control plane nodepools). As for the angent nodepools, you can freely add others agent nodepools the end of the list if you want._
-
-_Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state._
+_However you can freely add others nodepools the end of the list if you want, and of course increase the node count. You can also decrease the node count, but make sure you drain the node in question before, otherwise it will leave your cluster in a bad state. The only nodepool that needs at least to have a count of 1 always, is the first control-plane nodepool, for obvious reasons._
 
 ## High Availability
 
diff --git a/agents.tf b/agents.tf
index 4211d81..5c50972 100644
--- a/agents.tf
+++ b/agents.tf
@@ -12,9 +12,9 @@ module "agents" {
   placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.agent.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
+  ipv4_subnet_id         = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].id
 
-  private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 1].ip_range, each.value.index + 101)
+  private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -22,7 +22,7 @@ module "agents" {
   }
 
   depends_on = [
-    hcloud_network_subnet.subnet
+    hcloud_network_subnet.agent
   ]
 }
 
@@ -78,6 +78,6 @@ resource "null_resource" "agents" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.subnet
+    hcloud_network_subnet.agent
   ]
 }
diff --git a/control_planes.tf b/control_planes.tf
index d3bc0f3..a7fd377 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -24,7 +24,7 @@ module "control_planes" {
   }
 
   depends_on = [
-    hcloud_network_subnet.subnet
+    hcloud_network_subnet.control_plane
   ]
 }
 
@@ -83,6 +83,6 @@ resource "null_resource" "control_planes" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.subnet
+    hcloud_network_subnet.control_plane
   ]
 }
diff --git a/init.tf b/init.tf
index fd6ff68..367ee1d 100644
--- a/init.tf
+++ b/init.tf
@@ -57,7 +57,7 @@ resource "null_resource" "first_control_plane" {
   }
 
   depends_on = [
-    hcloud_network_subnet.subnet["control_plane"]
+    hcloud_network_subnet.control_plane
   ]
 }
 
diff --git a/main.tf b/main.tf
index dd27a1d..9145a4d 100644
--- a/main.tf
+++ b/main.tf
@@ -25,7 +25,7 @@ resource "hcloud_network_subnet" "control_plane" {
 
 # Here we start at the beginning of the subnets cird array
 resource "hcloud_network_subnet" "agent" {
-  count        = length(local.network_ipv4_subnets)
+  count        = length(local.agent_nodepools)
   network_id   = hcloud_network.k3s.id
   type         = "cloud"
   network_zone = var.network_region
@@ -84,7 +84,8 @@ resource "null_resource" "destroy_traefik_loadbalancer" {
   depends_on = [
     local_sensitive_file.kubeconfig,
     null_resource.control_planes[0],
-    hcloud_network_subnet.subnet,
+    hcloud_network_subnet.control_plane,
+    hcloud_network_subnet.agent,
     hcloud_placement_group.control_plane,
     hcloud_placement_group.agent,
     hcloud_network.k3s,
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 60ce5b5..d68f771 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -23,12 +23,16 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 # Of course, you can choose any number of nodepools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
 # For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
 
-# IMPORTANT: Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
-# You can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated.
-# Once the cluster is initialized, you cannot add more control plane nodepools. You can freely add others agent nodepools the end of the list if you want! 
+# IMPORTANT: Before the your cluster is created, you can do anything you want with the nodepools, but you need at least one of each control plane and agent. 
+# Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
+# you can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. 
+
+# The only nodepools that are safe to remove from the list when you edit it, are the ones at the end of the lists. This is due to how IPs are allocated.
+# You can however freely add others nodepools the end of each list if you want! The maximum number of nodepools you can create, combined for both lists is 255.
 # Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
 
-# Before initializing the cluster, you can change all parameters and add or remove any nodepools.
+# Before initializing the cluster, you can change all parameters and add or remove any nodepools. You just need at least one nodepool of each kind, control plane and agent.
+# The nodepool names are fully arbitrary, you can choose whatever you want, but no special characters or underscore, only alphanumeric characters and dashes are allowed.
 
 # If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1, and one agent nodepool with a count of 0.
 
@@ -41,7 +45,7 @@ control_plane_nodepools = [
     location    = "fsn1",
     labels      = [],
     taints      = [],
-    count       = 2
+    count       = 1
   },
   {
     name        = "control-plane-nbg1",
@@ -50,6 +54,14 @@ control_plane_nodepools = [
     labels      = [],
     taints      = [],
     count       = 1
+  },
+  {
+    name        = "control-plane-hel1",
+    server_type = "cpx11",
+    location    = "hel1",
+    labels      = [],
+    taints      = [],
+    count       = 1
   }
 ]