O Código deste documento encontra-se dentro do projecto Kubernetes
Para criar um cluster de Kubernetes dentro do Windows Azure com Terraform, criamos um novo projecto no Gitlab baseado
template criado no ponto anterior, e adicionamos alguns ficheiros de terraform:
O ficheiro kubernetes-cluster-1.tf declara o cluster em si,
com as respectivas node pools:
| kubernetes-cluster-1.tf |
|---|
| resource "azurerm_kubernetes_cluster" "k8s_cluster_1" {
# https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster
name = "${var.context_name}-${lower(var.environment)}"
location = var.location
resource_group_name = var.resource_group_name
dns_prefix = var.context_name
kubernetes_version = "1.28.13"
oidc_issuer_enabled = true
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
sku_tier = lower(var.environment) == "prod" ? "Standard" : lower(var.environment) == "dev" ? "Free" : "Free"
cost_analysis_enabled = false
image_cleaner_interval_hours = 168
node_os_upgrade_channel = "None"
default_node_pool {
name = "default"
type = "VirtualMachineScaleSets"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 2 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 10 : lower(var.environment) == "dev" ? 2 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS3_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_disk_type = "Ephemeral"
os_disk_size_gb = 128
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
service_principal {
client_id = var.service_principal_client_id
client_secret = var.service_principal_client_secret
}
role_based_access_control_enabled = true
azure_policy_enabled = false
http_application_routing_enabled = false
api_server_access_profile {
authorized_ip_ranges = []
}
network_profile {
network_plugin = "azure"
dns_service_ip = "10.0.0.10"
service_cidr = "10.0.0.0/16"
load_balancer_sku = "standard"
load_balancer_profile {
outbound_ip_address_ids = [ azurerm_public_ip.k8s_cluster1_public_ip_outbound_1.id ]
outbound_ports_allocated = 0
idle_timeout_in_minutes = 30
}
}
}
resource "azurerm_kubernetes_cluster_node_pool" "node_pool_2" {
name = "nodepool2"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "System"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 1 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 20 : lower(var.environment) == "dev" ? 2 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS3_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_disk_type = "Ephemeral"
os_disk_size_gb = 128
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "microservices_node_pool" {
name = "microsvc"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 7 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 20 : lower(var.environment) == "dev" ? 7 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_D4ads_v5" : lower(var.environment) == "dev" ? "Standard_D4ads_v5" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "microservices"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "flexible_node_pool" {
name = "flexible"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 10 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 40 : lower(var.environment) == "dev" ? 5 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS3_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "flexible"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "mongodb_node_pool" {
name = "mongodb"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 3 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 3 : lower(var.environment) == "dev" ? 1 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_E8bds_v5" : lower(var.environment) == "dev" ? "Standard_E4ads_v5" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "mongodb"
}
linux_os_config {
sysctl_config {
vm_max_map_count = 262144
}
transparent_huge_page_enabled = "never"
transparent_huge_page_defrag = "never"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "opensearch_node_pool" {
name = "opensearch"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 12 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 12 : lower(var.environment) == "dev" ? 1 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS13_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "opensearch"
}
linux_os_config {
sysctl_config {
vm_max_map_count = 262144
}
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "observability_node_pool" {
name = "o11y" # o11y == observability
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 3 : lower(var.environment) == "dev" ? 3 : 0
max_count = lower(var.environment) == "prod" ? 12 : lower(var.environment) == "dev" ? 8 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS4_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "observability"
}
linux_os_config {
sysctl_config {
vm_max_map_count = 262144
}
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "frontend_node_pool" {
name = "frontend"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 2 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 3 : lower(var.environment) == "dev" ? 2 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_DS3_v2" : lower(var.environment) == "dev" ? "Standard_DS3_v2" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "frontend"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "impulsemode_node_pool" {
name = "impulsemode"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 2 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 3 : lower(var.environment) == "dev" ? 2 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_D2ads_v5" : lower(var.environment) == "dev" ? "Standard_D2ads_v5" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "impulsemode"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_kubernetes_cluster_node_pool" "kafka_node_pool" {
name = "kafka"
kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s_cluster_1.id
mode = "User"
orchestrator_version = "1.28.13"
auto_scaling_enabled = true
min_count = lower(var.environment) == "prod" ? 1 : lower(var.environment) == "dev" ? 1 : 0
max_count = lower(var.environment) == "prod" ? 1 : lower(var.environment) == "dev" ? 1 : 0
vm_size = lower(var.environment) == "prod" ? "Standard_D2ads_v5" : lower(var.environment) == "dev" ? "Standard_D2ads_v5" : ""
zones = ["1", "2", "3"]
max_pods = 250
os_sku = "Ubuntu"
upgrade_settings {
drain_timeout_in_minutes = 30
node_soak_duration_in_minutes = 0
max_surge = 1
}
node_labels = {
"node-pool": "kafka"
}
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
|
Todos os secrets são injectados de fora do repositório
O ficheiro kubernetes-cluster-1-public-ips.tf define os novos IPs para serem usados no cluster
| kubernetes-cluster-1-public-ips.tf |
|---|
| /*
INGRESS IP, NEVER DELETE THIS IP!
It's associated to DNS api.primetag.com and api.primetag.net
*/
resource "azurerm_public_ip" "k8s_cluster1_public_ip_ingress_1" {
# https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip
name = "${var.context_name}-publicIP-ingress-${lower(var.environment)}"
resource_group_name = var.resource_group_name
location = var.location
allocation_method = "Static"
sku = "Standard"
zones = ["1", "2", "3"]
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
resource "azurerm_public_ip" "k8s_cluster1_public_ip_outbound_1" {
# https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip
name = "${var.context_name}-publicIP-${lower(var.environment)}"
resource_group_name = var.resource_group_name
location = var.location
allocation_method = "Static"
sku = "Standard"
zones = ["1", "2", "3"]
tags = {
environment = lower(var.environment)
created-by = lower("terraform")
}
}
|
Já o ficheiro kubernetes-cluster-1-gitlab-variables.tf salva nas variaveis de ambiente do Gitlab todos os novos valores que sejam relevantes para serem reutilizados (como por exemplo os IPs gerados)
| kubernetes-cluster-1-gitlab-variables.tf |
|---|
| resource "gitlab_group_variable" "gitlab-variable-kubectl-config" {
# https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs/resources/group_variable
group = "7818666"
// {key: [can contain only letters, digits and '_'.]}
key = format("TF_*****_%s", upper(var.environment))
value = azurerm_kubernetes_cluster.k8s_cluster_1.kube_config_raw
protected = false
masked = false
variable_type = "file"
depends_on = [ azurerm_kubernetes_cluster.k8s_cluster_1 ]
}
resource "gitlab_group_variable" "gitlab-variable-public-ip-ingress" {
# https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs/resources/group_variable
group = "7818666"
key = format("TF_K8S_CLUSTER1_INGRESS_IP_%s", upper(var.environment))
value = azurerm_public_ip.k8s_cluster1_public_ip_ingress_1.ip_address
protected = false
masked = false
variable_type = "env_var"
depends_on = [ azurerm_public_ip.k8s_cluster1_public_ip_ingress_1 ]
}
resource "gitlab_group_variable" "gitlab-variable-public-ip-outbound" {
# https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs/resources/group_variable
group = "7818666"
key = format("TF_K8S_CLUSTER1_OUTBOUND_IP_%s", upper(var.environment))
value = azurerm_public_ip.k8s_cluster1_public_ip_outbound_1.ip_address
protected = false
masked = false
variable_type = "env_var"
depends_on = [ azurerm_public_ip.k8s_cluster1_public_ip_outbound_1 ]
}
|
Por ultimo, o ficheiro kubernetes-cluster-1-output.tf imprime todos os outputs relevantes
| kubernetes-cluster-1-output.tf |
|---|
| output "kube_config" {
value = azurerm_kubernetes_cluster.k8s_cluster_1.kube_config_raw
sensitive = true
depends_on = [ azurerm_kubernetes_cluster.k8s_cluster_1 ]
}
output "host" {
value = azurerm_kubernetes_cluster.k8s_cluster_1.kube_config[0].host
sensitive = true
depends_on = [azurerm_kubernetes_cluster.k8s_cluster_1]
}
output "k8s_cluster1_public_ip_ingress_1" {
value = azurerm_public_ip.k8s_cluster1_public_ip_ingress_1.ip_address
sensitive = false
depends_on = [ azurerm_public_ip.k8s_cluster1_public_ip_ingress_1 ]
}
output "k8s_cluster1_public_ip_outbound_1" {
value = azurerm_public_ip.k8s_cluster1_public_ip_outbound_1.ip_address
sensitive = false
depends_on = [ azurerm_public_ip.k8s_cluster1_public_ip_outbound_1 ]
}
|
Os restantes ficheiros fazem parte do template, e já foram explicados no passo anterior