Talos

talos kubernetes setup under libvirt with 3 control and 3 worker nodes

Download latest talosctl

talosctl_install(){ sudo curl --proto =https --tlsv1.2 -fL https://github.com/siderolabs/talos/releases/latest/download/talosctl-linux-amd64 -o /usr/local/bin/talosctl ; sudo chmod 755 /usr/local/bin/talosctl ; }

Download latest talos system image.

talos_latest_iso(){
  LATEST=$(curl -sk https://github.com/siderolabs/talos/releases | elinks -dump -no-references -no-connect -no-home -no-numbering -dump-width 500 | grep -E '^ +v.*Latest' | awk '{print $1}')
  curl -kL https://github.com/siderolabs/talos/releases/download/${LATEST}/metal-amd64.iso -o talos.metal-amd64.${LATEST}.iso
}

If talosctl enable file completion

type talosctl >/dev/null 2>&1 ; if [ $? -eq 0 ] ; then . <(talosctl completion bash) ; fi

Remove old

for i in control1 control2 control3 worker1 worker2 worker3 ; do virsh destroy $i ; virsh undefine $i ; done
sudo find /var/lib/libvirt/images/ -type f -delete

Generate vm:s

for HOST in control1 control2 control3 worker1 worker2 worker3 ; do
  echo '*' $HOST
  sudo virt-install --name ${HOST} --vcpus 2 --memory 4096 --disk path=/var/lib/libvirt/images/${HOST}.qcow2,size=20 --cdrom /install/system/linux/talos/metal-amd64.iso --network bridge=br0,model=virtio --os-variant fedora40 --noautoconsole
done

Generate a secret

talosctl gen secrets -o secrets.yaml

Generate config

talosctl gen config --with-secrets secrets.yaml talos https://10.111.222.14:6443

Change to correct disk name

sed -i 's%/dev/sda%/dev/vda%g' *

If colliding Colliding subnets

sed -i 's%10.96.0.0/12%10.96.0.0/13%g' *

Add vip: vim controlplane.yaml

    network:
      interfaces:
        - deviceSelector:
            physical: true
          dhcp: true
          vip:
            ip: 10.111.222.14

Push config to control nodes. I use arp cache to figure out ip address of nodes.

nmap -sn -n 10.111.222.* >/dev/null ; for VM in control1 control2 control3 ; do echo $VM $( export VM=$VM ; MAC=$(virsh domiflist $VM | grep -oE '([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})') ; arp -a | grep $MAC | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' ) ; done | while read NODE_NAME IP_ADDRESS ; do
  echo '*' ${NODE_NAME} ${IP_ADDRESS}
  talosctl apply-config --insecure -n ${IP_ADDRESS} --file controlplane.yaml
done

Push config to worker nodes. I use arp cache to figure out ip address of nodes

nmap -sn -n 10.111.222.0/24 >/dev/null ; for VM in worker1 worker2 worker3 ; do echo $VM $( export VM=$VM ; MAC=$(virsh domiflist $VM | grep -oE '([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})') ; arp -a | grep $MAC | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' ) ; done | while read NODE_NAME 
IP_ADDRESS ; do
  echo '*' ${NODE_NAME} ${IP_ADDRESS}
  talosctl apply-config --insecure -n ${IP_ADDRESS} --file worker.yaml
done

Add control nodes to talosconfig

talosctl config endpoints $(nmap -sn -n 10.111.222.0/24 >/dev/null ; for VM in control1 control2 control3 ; do echo $VM $( export VM=$VM ; MAC=$(virsh domiflist $VM | grep -oE '([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})') ; arp -a | grep $MAC | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | grep -v 10.111.222.14 ) ; done | while read NODE_NAME IP_ADDRESS ; do   host $IP_ADDRESS | awk '{print $NF}'; done | xargs) --talosconfig talosconfig

Add worker and control nodes to talosconfig

talosctl config nodes $(nmap -sn -n 10.111.222.0/24 >/dev/null ; for VM in worker1 worker2 worker3 control1 control2 control3 ; do echo $VM $( export VM=$VM ; MAC=$(virsh domiflist $VM | grep -oE '([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})') ; arp -a | grep $MAC | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | grep -v 10.111.222.14 ) ; done | while read NODE_NAME IP_ADDRESS ; do   host $IP_ADDRESS | awk '{print $NF}'; done | xargs) --talosconfig talosconfig

bootstrap

CONTROL=$(nmap -sn -n 10.111.222.* >/dev/null ; for VM in control1 ; do echo $VM $( export VM=$VM ; MAC=$(virsh domiflist $VM | grep -oE '([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})') ; arp -a | grep $MAC | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' ) ; done | while read NODE_NAME 
IP_ADDRESS ; do   host $IP_ADDRESS | awk '{print $NF}'; done)
talosctl bootstrap -n $CONTROL --talosconfig talosconfig

get kubectl

talosctl kubeconfig alternative-kubeconfig --nodes $CONTROL --endpoints $CONTROL --talosconfig=talosconfig

Troubleshooting

list disks

talosctl -n <ip> disks --insecure

List network cards

talosctl -n <ip> get links --insecure
talosctl -n <ip> get links <interface> -o yaml --insecure

look at config

talosctl config info --talosconfig talosconfig

logs

talosctl logs <service> -n <ip>

Which services exists

talosctl service

Health check of talos

talosctl health -n ip154

Gathering logs

talosctl support

Which nodes belong to cluster

talosctl -n <ip> get members

List etcd members. According to node you ask

talosctl -n <ip> etcd members list

List alarms

talosctl -n <ip> etcd alarm list

List staticpods

talosctl -n <ip> get staticpods

List containers

talosctl -n <ip> containers==

Get manifests

talosctl -n <ip> get manifests

Checking Controller Runtime Logs

talosctl -n <ip> logs controller-runtime