khuedoan · khuedoan · Aug 20, 2021 · Jul 14, 2021 · Jul 17, 2021 · Jul 18, 2021
diff --git a/.gitignore b/.gitignore
@@ -5,9 +5,9 @@ terraform-*-tmp-*/
 *.iso
 *.jpg
 *.log
+*kubeconfig.yaml
 .terraform*
 backend.tfvars
 ignition.json
-kube_config.yaml
 terraform.tfstate*
 terraform.tfvars
diff --git a/apps/Makefile b/apps/Makefile
@@ -1,7 +1,7 @@
 .POSIX:
 .EXPORT_ALL_VARIABLES:
 
-KUBECONFIG = ../infra/kube_config.yaml
+KUBECONFIG = ../infra/kubeconfig.yaml
 
 default: apply
 

diff --git a/experimental/cluster-api/README.md b/experimental/cluster-api/README.md
@@ -0,0 +1,14 @@
+# Infrastructure
+
+Bare metal servers life cycle management:
+
+- Automatically discover bare metal servers
+- Automatically wipe the disk
+- Install Linux on empty machines via the network (using iPXE)
+
+Kubernetes cluster creation:
+
+- Create ephemeral management cluster (using Kubernetes in Docker)
+- Create target cluster
+- Pivot the management plane over to the new cluster
+- Remove the ephemeral management cluster
diff --git a/experimental/cluster-api/bootstrap.sh b/experimental/cluster-api/bootstrap.sh
@@ -0,0 +1,100 @@
+#!/bin/sh
+
+set -euo pipefail
+
+# Create ephemeral cluster
+kind create cluster \
+    --wait 60s \
+    --kubeconfig ephemeral-cluster/kind-kubeconfig.yaml \
+    --config ephemeral-cluster/kind.yaml
+
+export KUBECONFIG="$PWD/ephemeral-cluster/kind-kubeconfig.yaml"
+
+# Install Sidero
+clusterctl init \
+    --bootstrap talos \
+    --control-plane talos \
+    --infrastructure sidero \
+    --config clusterctl.yaml
+
+# Create DHCP server
+# kubectl create configmap dhcp-server \
+#     --from-file dhcp-server/dhcpd.conf
+# kubectl apply \
+#     --filename dhcp-server/deployment.yaml
+docker run --detach \
+    --name bootstrap-dhcp-server \
+    --network=host \
+    --volume $PWD/dhcp-server/dhcpd.conf:/data/dhcpd.conf \
+    networkboot/dhcpd:1.1.0
+
+# Wait for all pods to be ready
+kubectl wait pods \
+    --all \
+    --all-namespaces \
+    --timeout 300s \
+    --for=condition=Ready
+
+# Apply server classes
+kubectl apply \
+    --filename serverclasses/
+
+# Waker servers up
+wol '00:23:24:d1:f3:f0'
+
+# Create cluster
+while true; do
+  kubectl get server --output jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' && break
+done
+
+export CONTROL_PLANE_ENDPOINT=$(kubectl get server \
+    --output jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') && break
+
+clusterctl config cluster \
+    --infrastructure sidero \
+    --config clusterctl.yaml \
+    homelab | kubectl apply --filename -
+
+# Get kube config
+while true; do
+  clusterctl get kubeconfig homelab > cluster/homelab-kubeconfig.yaml && break
+done
+
+export KUBECONFIG="$PWD/cluster/homelab-kubeconfig.yaml"
+
+while true; do
+  kubectl cluster-info && break
+done
+
+# TODO Untaint master node
+# kubectl taint node node-role.kubernetes.io/master:NoSchedule-
+
+# Pivot Sidero to new cluster
+clusterctl init \
+    --bootstrap talos \
+    --control-plane talos \
+    --infrastructure sidero \
+    --kubeconfig cluster/homelab-kubeconfig.yaml \
+    --config clusterctl.yaml
+
+clusterctl move \
+  --kubeconfig=ephemeral-cluster/kind-kubeconfig.yaml \
+  --to-kubeconfig=cluster/homelab-kubeconfig.yaml
+
+# Cleanup ephemeral cluster
+kind delete cluster --name bootstrap-cluster
+rm ephemeral-cluster/kind-kubeconfig.yaml
+docker rm --force bootstrap-dhcp-server
+
+# TODO Scale the cluster up
+# wol '00:23:24:d1:f4:d6'
+# wol '00:23:24:d1:f5:69'
+# wol '00:23:24:e7:04:60'
+
+# clusterctl config cluster \
+#     --infrastructure sidero \
+#     --config clusterctl.yaml \
+#     --config clusterctl.yaml \
+#     --worker-machine-count 3 \
+#     homelab > cluster/homelab.yaml
+# kubectl apply --filename cluster/homelab.yaml
diff --git a/experimental/cluster-api/cluster/homelab.yaml b/experimental/cluster-api/cluster/homelab.yaml
@@ -0,0 +1,112 @@
+apiVersion: cluster.x-k8s.io/v1alpha3
+kind: Cluster
+metadata:
+  name: homelab
+  namespace: default
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.244.0.0/16
+    services:
+      cidrBlocks:
+      - 10.96.0.0/12
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+    kind: TalosControlPlane
+    name: homelab-cp
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+    kind: MetalCluster
+    name: homelab
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: MetalCluster
+metadata:
+  name: homelab
+  namespace: default
+spec:
+  controlPlaneEndpoint:
+    host: 192.168.1.12
+    port: 6443
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: MetalMachineTemplate
+metadata:
+  name: homelab-cp
+  namespace: default
+spec:
+  template:
+    spec:
+      serverClassRef:
+        apiVersion: metal.sidero.dev/v1alpha1
+        kind: ServerClass
+        name: m5.xlarge
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+kind: TalosControlPlane
+metadata:
+  name: homelab-cp
+  namespace: default
+spec:
+  controlPlaneConfig:
+    controlplane:
+      generateType: controlplane
+      talosVersion: v0.11.2
+    init:
+      generateType: init
+      talosVersion: v0.11.2
+  infrastructureTemplate:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+    kind: MetalMachineTemplate
+    name: homelab-cp
+  replicas: 1
+  version: v1.21.1
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: TalosConfigTemplate
+metadata:
+  name: homelab-workers
+  namespace: default
+spec:
+  template:
+    spec:
+      generateType: join
+      talosVersion: v0.11.2
+---
+apiVersion: cluster.x-k8s.io/v1alpha3
+kind: MachineDeployment
+metadata:
+  name: homelab-workers
+  namespace: default
+spec:
+  clusterName: homelab
+  replicas: 3
+  selector:
+    matchLabels: null
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+          kind: TalosConfigTemplate
+          name: homelab-workers
+      clusterName: homelab
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+        kind: MetalMachineTemplate
+        name: homelab-workers
+      version: v1.21.1
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: MetalMachineTemplate
+metadata:
+  name: homelab-workers
+  namespace: default
+spec:
+  template:
+    spec:
+      serverClassRef:
+        apiVersion: metal.sidero.dev/v1alpha1
+        kind: ServerClass
+        name: m5.xlarge
diff --git a/experimental/cluster-api/clusterctl.yaml b/experimental/cluster-api/clusterctl.yaml
@@ -0,0 +1,11 @@
+SIDERO_CONTROLLER_MANAGER_API_ENDPOINT: 192.168.1.19
+SIDERO_CONTROLLER_MANAGER_AUTO_ACCEPT_SERVERS: true
+SIDERO_CONTROLLER_MANAGER_HOST_NETWORK: true
+
+CONTROL_PLANE_SERVERCLASS: m5.xlarge
+WORKER_SERVERCLASS: m5.xlarge
+
+CONTROL_PLANE_PORT: 6443
+
+KUBERNETES_VERSION: v1.21.1
+TALOS_VERSION: v0.11.2
diff --git a/experimental/cluster-api/dhcp-server/deployment.yaml b/experimental/cluster-api/dhcp-server/deployment.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: dhcp-server
+  name: dhcp-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dhcp-server
+  template:
+    metadata:
+      labels:
+        app: dhcp-server
+    spec:
+      containers:
+      - image: networkboot/dhcpd:1.1.0
+        name: dhcpd
+        volumeMounts:
+        - name: config
+          mountPath: /data/dhcpd.conf
+          subPath: dhcpd.conf
+      hostNetwork: true
+      volumes:
+      - name: config
+        configMap:
+          name: dhcp-server
+          items:
+          - key: dhcpd.conf
+            path: dhcpd.conf
diff --git a/experimental/cluster-api/dhcp-server/dhcpd.conf b/experimental/cluster-api/dhcp-server/dhcpd.conf
@@ -0,0 +1,20 @@
+option space pxelinux;
+option pxelinux.magic code 208 = string;
+option pxelinux.configfile code 209 = text;
+option pxelinux.pathprefix code 210 = text;
+option pxelinux.reboottime code 211 = unsigned integer 32;
+option architecture-type code 93 = unsigned integer 16;
+
+subnet 192.168.1.0 netmask 255.255.255.0 {
+    option routers 192.168.1.1;
+    range 192.168.1.2 192.168.1.254;
+
+    class "pxeclients" {
+        match if substring (option vendor-class-identifier, 0, 9) = "PXEClient";
+        next-server 192.168.1.19;
+
+        if option architecture-type = 00:07 {
+            filename "ipxe.efi";
+        }
+    }
+}
diff --git a/experimental/cluster-api/environment.yaml b/experimental/cluster-api/environment.yaml
@@ -0,0 +1,26 @@
+apiVersion: metal.sidero.dev/v1alpha1
+kind: Environment
+metadata:
+  name: default
+spec:
+  initrd:
+    url: https://github.com/talos-systems/talos/releases/download/v0.10.3/initramfs-amd64.xz
+  kernel:
+    args:
+    - console=tty0
+    - console=ttyS0
+    - consoleblank=0
+    - earlyprintk=ttyS0
+    - ima_appraise=fix
+    - ima_hash=sha512
+    - ima_template=ima-ng
+    - init_on_alloc=1
+    - initrd=initramfs.xz
+    - nvme_core.io_timeout=4294967295
+    - printk.devkmsg=on
+    - pti=on
+    - random.trust_cpu=on
+    - slab_nomerge=
+    - talos.config=http://192.168.1.19:8081/configdata?uuid=
+    - talos.platform=metal
+    url: https://github.com/talos-systems/talos/releases/download/v0.10.3/vmlinuz-amd64
diff --git a/experimental/cluster-api/ephemeral-cluster/kind.yaml b/experimental/cluster-api/ephemeral-cluster/kind.yaml
@@ -0,0 +1,14 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: bootstrap-cluster
+nodes:
+- role: control-plane
+  extraPortMappings:
+  # TFTP
+  - containerPort: 69
+    hostPort: 69
+    protocol: UDP
+  # HTTP
+  - containerPort: 8081
+    hostPort: 8081
+    protocol: TCP
diff --git a/experimental/cluster-api/serverclasses/m5.xlarge.yaml b/experimental/cluster-api/serverclasses/m5.xlarge.yaml
@@ -0,0 +1,13 @@
+apiVersion: metal.sidero.dev/v1alpha1
+kind: ServerClass
+metadata:
+  name: m5.xlarge # The names are free-form and only need to make sense to you
+spec:
+  qualifiers:
+    cpu:
+    - manufacturer: Intel(R) Corporation
+      version: Intel(R) Core(TM) i5-6600T CPU @ 2.70GHz
+  configPatches:
+  - op: replace
+    path: /machine/install/disk
+    value: /dev/sda
diff --git a/infra/cluster.tf b/infra/cluster.tf
@@ -69,7 +69,7 @@ resource "rke_cluster" "cluster" {
 }
 
 resource "local_file" "kube_config_yaml" {
-  filename          = "${path.root}/kube_config.yaml"
+  filename          = "${path.root}/kubeconfig.yaml"
   sensitive_content = rke_cluster.cluster.kube_config_yaml
   file_permission   = "0600"
 }