Skip to content

Instantly share code, notes, and snippets.

@uhlhosting

uhlhosting/README.md

Forked from usrbinkat/README.md
Created August 3, 2025 10:52
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save uhlhosting/f1d2d6aca1f071aa48fea7a70c32a1f2 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save uhlhosting/f1d2d6aca1f071aa48fea7a70c32a1f2 to your computer and use it in GitHub Desktop.
Download ZIP
Talos + Kubevirt Bare Metal & Nested Tenant Cluster
Raw
README.md

Scratch nodes WIP

talosctl gen config talos-kubevirt https://talos-kubevirt.home.arpa:6443 --additional-sans 192.168.1.70,talos-kubevirt --install-disk /dev/vda --output-dir $(pwd)/talos
❯ cat deploy.sh
#kubectl --kubeconfig $HOME/.kube/poweredge delete -f ./kubevirt/
#sleep 12
#kubectl --kubeconfig $HOME/.kube/poweredge apply -f ./kubevirt/

#sleep 120

talosctl apply-config --insecure --nodes 192.168.1.71 -e 192.168.1.71 --file ./talos/cp01.yml --talosconfig ./talos/talosconfig
talosctl apply-config --insecure --nodes 192.168.1.72 -e 192.168.1.72 --file ./talos/cp02.yml --talosconfig ./talos/talosconfig
talosctl apply-config --insecure --nodes 192.168.1.73 -e 192.168.1.73 --file ./talos/cp03.yml --talosconfig ./talos/talosconfig

# sleep 60
# talosctl bootstrap --nodes 192.168.1.71 -e 192.168.1.71 --talosconfig ./talos/talosconfig

# sleep 6
# talosctl kubeconfig --force --context talos-kubevirt --force-context-name talos-kubevirt --merge --nodes 192.168.1.71 --endpoints 192.168.1.70 --talosconfig ./talos/talosconfig

# kubectl taint --all nodes node-role.kubernetes.io/master-

# helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph
# helm install --create-namespace --namespace rook-ceph rook-ceph-cluster --set operatorNamespace=rook-ceph rook-release/rook-ceph-cluster --values rook-ceph/cluster.values.yaml

# talosctl dashboard --nodes 192.168.1.71,192.168.1.73,192.168.1.75 -e 192.168.1.70 --talosconfig ./talos/talosconfig

# export RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
# kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/$RELEASE/kubevirt-operator.yaml
# kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/$RELEASE/kubevirt-cr.yaml
# kubectl -n kubevirt wait kv kubevirt --for condition=Available
# kubectl get po -n kubevirt
Raw
control-plane-n01.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
name: talos-cp01
labels:
app: talos
spec:
running: true
template:
spec:
terminationGracePeriodSeconds: 0
domain:
clock:
timer: {}
utc: {}
cpu:
model: host-passthrough
threads: 8
machine:
type: q35
resources:
requests:
devices.kubevirt.io/kvm: "1"
memory: 16G
devices:
rng: {}
autoattachSerialConsole: true
autoattachGraphicsDevice: true
autoattachPodInterface: false
interfaces:
- bridge: {}
model: virtio
name: eth0
macAddress: de:00:00:00:01:de
disks:
- name: talos-cp01-disk-vda-root
bootOrder: 1
disk:
bus: virtio
- name: talos-cp01-disk-vdb-data
disk:
bus: virtio
networks:
- name: eth0
multus:
networkName: nadbr0
volumes:
- name: talos-cp01-disk-vda-root
dataVolume:
name: talos-cp01-volume-vda-root
- name: talos-cp01-disk-vdb-data
dataVolume:
name: talos-cp01-volume-vdb-data
dataVolumeTemplates:
- metadata:
name: talos-cp01-volume-vda-root
spec:
pvc:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 32G
source:
registry:
url: docker://docker.io/containercraft/talos:v1.2.0-alpha.0
imagePullPolicy: Always
- metadata:
name: talos-cp01-volume-vdb-data
spec:
pvc:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 64G
source:
blank: {}
Raw
network-addons-operator-config.yaml
apiVersion: networkaddonsoperator.network.kubevirt.io/v1
kind: NetworkAddonsConfig
metadata:
name: cluster
spec:
imagePullPolicy: IfNotPresent
multus: {}
# ovs: {}
# macvtap: {}
# kubeMacPool: {}
# linuxBridge: {}
Raw
network-attachment-definition.yaml
---
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: nadbr0
spec:
config: >
{
"cniVersion": "0.3.1",
"name": "br0",
"plugins": [
{
"type": "bridge",
"bridge": "br0",
"ipam": {}
},
{
"type": "tuning"
}
]
}
Raw
reset.sh
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.71 -e 192.168.1.71
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.72 -e 192.168.1.72
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.73 -e 192.168.1.73
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.74 -e 192.168.1.74
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.75 -e 192.168.1.75
talosctl reset \
--talosconfig ./talos/talosconfig \
--system-labels-to-wipe=EPHEMERAL \
--system-labels-to-wipe=STATE \
--reboot --graceful=false \
--nodes 192.168.1.76 -e 192.168.1.76
Raw
talos-controlplane-n01.yml
version: v1alpha1
debug: true
persist: true
machine:
type: controlplane
token: j9oiyf.ek11sl98qfe36w9s
ca:
crt: LS0tLS1CRUdJTiBDRVJ....truncated....USUZJkNCOHFBREFnRUN
key: LS0tLS1CRUdJTiBFRDI....truncated....1NTE5IFBSSVZBVEUgS0
certSANs:
- 192.168.1.70
- talos-kubevirt
- talos-kubevirt.home.arpa
kubelet:
image: ghcr.io/siderolabs/kubelet:v1.24.2
extraConfig:
serverTLSBootstrap: true
network:
interfaces:
- interface: br0
bridge:
stp:
enabled: true
interfaces:
- eth0
addresses:
- 192.168.1.71/24
routes:
- network: 0.0.0.0/0
gateway: 192.168.1.1
metric: 1024
mtu: 1500
vip:
ip: 192.168.1.70
nameservers:
- 192.168.1.1
extraHostEntries:
- ip: 192.168.1.70
aliases:
- talos-kubevirt
- talos-kubevirt.home.arpa
- ip: 192.168.1.71
aliases:
- talos-cp01
- talos-cp01.home.arpa
- ip: 192.168.1.72
aliases:
- talos-cp02
- talos-cp02.home.arpa
- ip: 192.168.1.73
aliases:
- talos-cp03
- talos-cp03.home.arpa
kubespan:
enabled: false
install:
disk: /dev/vda
image: ghcr.io/siderolabs/installer:v1.2.0-alpha.0
bootloader: true
wipe: true
features:
rbac: true
controlPlane:
scheduler:
disabled: false
sysctls:
kernel.domainname: home.arpa
sysfs:
devices.system.cpu.cpu0.cpufreq.scaling_governor: performance
cluster:
id: m3WfmbKxtTx...truncated...j_D-sX9z_GgCMjVt
secret: nCrlcmhQWeX...truncated...uVMT6qGtSQ80RPc
controlPlane:
endpoint: https://talos-kubevirt.home.arpa:6443
clusterName: talos-kubevirt
network:
dnsDomain: cluster.local
podSubnets:
- 10.244.0.0/16
serviceSubnets:
- 10.96.0.0/12
token: w22k05.fsm...truncated....pi8jy
aescbcEncryptionSecret: sMh+5EH....truncated....w8buSfnoy54Y8YU
ca:
crt: LS0tLS1CRUdJTiBDRV....truncated....JUSUZJQ0FURS0tLS0tCk1JSUJpa
key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
aggregatorCA:
crt: LS0tLS1CRUdJTiBDRV....truncated....JUSUZJQ0FURS0tLS0tCk1JSUJYa
key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
serviceAccount:
key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
apiServer:
image: k8s.gcr.io/kube-apiserver:v1.24.2
certSANs:
- 192.168.1.70
- talos-kubevirt
- talos-kubevirt.home.arpa
disablePodSecurityPolicy: true
controllerManager:
image: k8s.gcr.io/kube-controller-manager:v1.24.2
proxy:
image: k8s.gcr.io/kube-proxy:v1.24.2
scheduler:
image: k8s.gcr.io/kube-scheduler:v1.24.2
discovery:
enabled: true
registries:
kubernetes: {}
service: {}
etcd:
ca:
crt: LS0tLS1CRUdJT....truncated....iBDRVJUSUZJQ0FURS0tLS0tCk
key: LS0tLS1CRUdJT....truncated....iBFQyBQUklWQVRFIEtFWS0tLS
extraManifests: []
inlineManifests: []
Raw
vm.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
name: ubuntu
labels:
app: kc2
kc2: ubuntu
ubuntu: bastion
spec:
running: true
template:
spec:
domain:
cpu:
cores: 1
model: host-model
sockets: 1
threads: 4
devices:
autoattachGraphicsDevice: true
autoattachPodInterface: false
autoattachSerialConsole: true
disks:
- bootOrder: 1
disk:
bus: virtio
name: root-disk
- disk:
bus: virtio
name: cloudinitdisk
interfaces:
- bootOrder: 2
bridge: {}
model: virtio
name: enp1s0
networkInterfaceMultiqueue: true
rng: {}
machine:
type: q35
resources:
requests:
memory: 6G
hostname: ubuntu
networks:
- name: enp1s0
multus:
networkName: nadbr0
terminationGracePeriodSeconds: 0
volumes:
- name: root-disk
containerDisk:
image: docker.io/containercraft/ubuntu:22.04
imagePullPolicy: IfNotPresent
- name: cloudinitdisk
cloudInitNoCloud:
networkData: |
version: 2
ethernets:
enp1s0:
dhcp4: true
dhcp6: false
userData: |
#cloud-config
hostname: ubuntu
ssh_pwauth: true
chpasswd:
list: |
kc2user:kc2user
expire: False
users:
- name: kc2user
sudo: ['ALL=(ALL) NOPASSWD:ALL']
groups: sudo
shell: /bin/bash
ssh-authorized-keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfKxpwT/k6ZI/LKMZYLEnTSTlco74kJsP9sfrilR2JP39y8LYmqqbiTZ2m6r4lrbyWjFvch217cOP8UaA3DpSh4BoV0Qs8W5zPcoFt5/hGfIVVCoGkyjDWbYoyjsGFTXgUpRmIl+68ph4BX3ds5wYE+SfNGUupgHxk+vjGbCPMWClBrJ0AmWP2U/uJWN/pv6OdaC8AsTkF5LkIhq2pKH/X4a8gMmXihUl1hgrWeiCmS3B49wQNn8Bc3DORKx4XZRinQ4LYklOhQYfH+P5Trz17RyEzb/ttKG3nkvHWTM4MA6/RcxGTjorYbR0lb9KtHNqaClV61voBVGbUz/Ufn4gDCZzfFMzm5UwpwY6eOYKDU8RQbNIB/+GGW4eC9XGVs8JZys0lovCLdrbeNPpJGVZ991nncvY0ouXoYZZX5atVEJW5zJnk2/BqEek/kSIZM57rj49AiZ9pFNnE+PoYxHlbXy4nQrHsCkZkMkrN4CtDumdds05TvOZKIlGXOc3/kjU= [email protected]
package_upgrade: true
packages:
- ubuntu-desktop-minimal
- docker.io
- xrdp
- gnome-tweaks
runcmd:
- "snap remove lxd"
- "mkdir -p /home/kc2user/.kube"
- "chown -R kc2user:kc2user /home/kc2user"
- "usermod -a -G xrdp kc2user"
- "usermod -a -G ssl-cert kc2user"
- "usermod -a -G docker kc2user"
- "apt-get -y upgrade"
Raw
vyos.config.sh
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp01 mac-address de:00:00:00:01:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp01 ip-address 192.168.1.71
set system static-host-mapping host-name talos-cp01.home.arpa inet 192.168.1.71
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp02 mac-address de:00:00:00:02:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp02 ip-address 192.168.1.72
set system static-host-mapping host-name talos-cp02.home.arpa inet 192.168.1.72
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp03 mac-address de:00:00:00:03:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp03 ip-address 192.168.1.73
set system static-host-mapping host-name talos-cp03.home.arpa inet 192.168.1.73
set system static-host-mapping host-name talos-kubevirt.home.arpa inet 192.168.1.70
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment