Skip to content

Instantly share code, notes, and snippets.

@Knappek

Knappek/custom-clusterclass-custom-audit-policy.yaml

Created January 23, 2025 14:35
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Knappek/b6c7f931ef4c709d22d5bab66c00c5f8 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Knappek/b6c7f931ef4c709d22d5bab66c00c5f8 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
custom-clusterclass-custom-audit-policy.yaml
apiVersion: cluster.x-k8s.io/v1beta1
kind: ClusterClass
metadata:
annotations:
run.tanzu.vmware.com/resolve-tkr: ""
name: custom-audit-policy
namespace: auditing
spec:
controlPlane:
machineHealthCheck:
maxUnhealthy: 100%
nodeStartupTimeout: 2h0m0s
unhealthyConditions:
- status: Unknown
timeout: 5m0s
type: Ready
- status: "False"
timeout: 12m0s
type: Ready
machineInfrastructure:
ref:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
name: tkc-control-plane
namespace: auditing
metadata:
annotations:
run.tanzu.vmware.com/resolve-os-image: os-name=photon
ref:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
name: tkc-control-plane
namespace: auditing
infrastructure:
ref:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereClusterTemplate
name: tkc-infrastructure
namespace: auditing
patches:
- definitions:
- jsonPatches:
- op: replace
path: /spec/template/spec/className
valueFrom:
variable: vmClass
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: replace
path: /spec/template/spec/className
valueFrom:
variable: vmClass
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: vmClass
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/storageClass
valueFrom:
variable: storageClass
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/storageClass
valueFrom:
variable: storageClass
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: storageClass
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files
valueFrom:
template: |
{{ $clusterName := .builtin.cluster.name }}
{{- range .trust.additionalTrustedCAs }}
- contentFrom:
secret:
name: {{ $clusterName }}-user-trusted-ca-secret
key: {{ .name }}
owner: root:root
path: /etc/ssl/certs/tkg-{{.name}}-ca.pem
encoding: base64
permissions: "0644"
{{- end }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files
valueFrom:
template: |
{{ $clusterName := .builtin.cluster.name }}
{{- range .trust.additionalTrustedCAs }}
- contentFrom:
secret:
name: {{ $clusterName }}-user-trusted-ca-secret
key: {{ .name }}
owner: root:root
path: /etc/ssl/certs/tkg-{{.name}}-ca.pem
encoding: base64
permissions: "0644"
{{- end }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .trust }}true{{end}}'
name: trust
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hostname
permissions: "0644"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hostname
permissions: "0644"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .trust }}true{{end}}'
name: FilesEtcHostName
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files
valueFrom:
template: |
- content: |
{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hostname
permissions: "0644"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files
valueFrom:
template: |
- content: |
{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hostname
permissions: "0644"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if not .trust }}true{{end}}'
name: InitFilesEtcHostName
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
HTTP_PROXY={{ .proxy.httpProxy }}
HTTPS_PROXY={{ .proxy.httpsProxy }}
NO_PROXY={{- range .proxy.noProxy }},{{.}}{{- end}}{{- range .builtin.cluster.network.services }},{{.}}{{- end }}{{- range .builtin.cluster.network.pods }},{{.}}{{- end }},localhost,127.0.0.1
owner: root:root
path: /etc/systemd/system/containerd.service.d/http-proxy.env
permissions: "0644"
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
[Service]
EnvironmentFile=/etc/systemd/system/containerd.service.d/http-proxy.env
owner: root:root
path: /etc/systemd/system/containerd.service.d/http-proxy.conf
permissions: "0644"
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
HTTP_PROXY={{ .proxy.httpProxy }}
HTTPS_PROXY={{ .proxy.httpsProxy }}
NO_PROXY={{- range .proxy.noProxy }},{{.}}{{- end}}{{- range .builtin.cluster.network.services }},{{.}}{{- end }}{{- range .builtin.cluster.network.pods }},{{.}}{{- end }},localhost,127.0.0.1
owner: root:root
path: /etc/systemd/system/kubelet.service.d/http-proxy.env
permissions: "0644"
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
[Service]
EnvironmentFile=/etc/systemd/system/kubelet.service.d/http-proxy.env
owner: root:root
path: /etc/systemd/system/kubelet.service.d/http-proxy.conf
permissions: "0644"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
HTTP_PROXY={{ .proxy.httpProxy }}
HTTPS_PROXY={{ .proxy.httpsProxy }}
NO_PROXY={{- range .proxy.noProxy }},{{.}}{{- end}}{{- range .builtin.cluster.network.services }},{{.}}{{- end }}{{- range .builtin.cluster.network.pods }},{{.}}{{- end }},localhost,127.0.0.1
owner: root:root
path: /etc/systemd/system/containerd.service.d/http-proxy.env
permissions: "0644"
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
[Service]
EnvironmentFile=/etc/systemd/system/containerd.service.d/http-proxy.env
owner: root:root
path: /etc/systemd/system/containerd.service.d/http-proxy.conf
permissions: "0644"
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
HTTP_PROXY={{ .proxy.httpProxy }}
HTTPS_PROXY={{ .proxy.httpsProxy }}
NO_PROXY={{- range .proxy.noProxy }},{{.}}{{- end}}{{- range .builtin.cluster.network.services }},{{.}}{{- end }}{{- range .builtin.cluster.network.pods }},{{.}}{{- end }},localhost,127.0.0.1
owner: root:root
path: /etc/systemd/system/kubelet.service.d/http-proxy.env
permissions: "0644"
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
[Service]
EnvironmentFile=/etc/systemd/system/kubelet.service.d/http-proxy.env
owner: root:root
path: /etc/systemd/system/kubelet.service.d/http-proxy.conf
permissions: "0644"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .proxy }}true{{end}}'
name: proxy
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
::1 ipv6-localhost ipv6-loopback
127.0.0.1 localhost {{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hosts
permissions: "0644"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
::1 ipv6-localhost ipv6-loopback
127.0.0.1 localhost {{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}
owner: root:root
path: /etc/hosts
permissions: "0644"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: FilesEtcHostContent
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
---
apiVersion: apiserver.k8s.io/v1alpha1
kind: AdmissionConfiguration
plugins:
owner: root:root
path: /etc/kubernetes/extra-config/admission-control-config.yaml
permissions: "0640"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ semverCompare "<=1.24.x-0" .builtin.controlPlane.version}}'
name: controlPlaneFilesAdmissionConfiguration
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
{{ $namespace_exemptions := printf "%q, %q, %q" "kube-system" "tkg-system" "vmware-system-cloud-provider" -}}
{{- if .podSecurityStandard.exemptions.namespaces -}}
{{ range $namespace := .podSecurityStandard.exemptions.namespaces -}}
{{ $namespace_exemptions = printf "%s, %q" $namespace_exemptions $namespace -}}
{{- end -}}
{{- end -}}
---
apiVersion: apiserver.k8s.io/v1alpha1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1
kind: PodSecurityConfiguration
defaults:
{{- if .podSecurityStandard.warn }}
warn: "{{ .podSecurityStandard.warn }}"
{{- else }}
warn: "restricted"
{{- end }}
{{- if .podSecurityStandard.warnVersion }}
warn-version: "{{ .podSecurityStandard.warnVersion }}"
{{- else }}
warn-version: "latest"
{{- end }}
{{- if .podSecurityStandard.audit }}
audit: "{{ .podSecurityStandard.audit }}"
{{- else }}
audit: "restricted"
{{- end }}
{{- if .podSecurityStandard.auditVersion }}
audit-version: "{{ .podSecurityStandard.auditVersion }}"
{{- else }}
audit-version: "latest"
{{- end }}
{{- if .podSecurityStandard.enforce }}
enforce: "{{ .podSecurityStandard.enforce }}"
{{- end }}
{{- if .podSecurityStandard.enforceVersion }}
enforce-version: "{{ .podSecurityStandard.enforceVersion }}"
{{- end }}
exemptions:
{{- if .podSecurityStandard.exemptions.namespaces }}
namespaces: [{{ $namespace_exemptions }}]
{{- else }}
namespaces: [kube-system, tkg-system, vmware-system-cloud-provider]
{{- end }}
owner: root:root
path: /etc/kubernetes/extra-config/admission-control-config.yaml
permissions: "0640"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ and (not .podSecurityStandard.deactivated) (semverCompare "~1.25.x-0"
.builtin.controlPlane.version) }}'
name: controlPlaneFilesAdmissionConfigurationk8s125
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
{{ $namespace_exemptions := printf "%q, %q, %q" "kube-system" "tkg-system" "vmware-system-cloud-provider" -}}
{{- if .podSecurityStandard.exemptions.namespaces -}}
{{ range $namespace := .podSecurityStandard.exemptions.namespaces -}}
{{ $namespace_exemptions = printf "%s, %q" $namespace_exemptions $namespace -}}
{{- end -}}
{{- end -}}
---
apiVersion: apiserver.k8s.io/v1alpha1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1
kind: PodSecurityConfiguration
defaults:
{{- if .podSecurityStandard.enforce }}
enforce: "{{ .podSecurityStandard.enforce }}"
{{- else }}
enforce: "restricted"
{{- end }}
{{- if .podSecurityStandard.enforceVersion }}
enforce-version: "{{ .podSecurityStandard.enforceVersion }}"
{{- else }}
enforce-version: "latest"
{{- end }}
{{- if .podSecurityStandard.warn }}
warn: "{{ .podSecurityStandard.warn }}"
{{- end }}
{{- if .podSecurityStandard.warnVersion }}
warn-version: "{{ .podSecurityStandard.warnVersion }}"
{{- end }}
{{- if .podSecurityStandard.audit }}
audit: "{{ .podSecurityStandard.audit }}"
{{- end }}
{{- if .podSecurityStandard.auditVersion }}
audit-version: "{{ .podSecurityStandard.auditVersion }}"
{{- end }}
exemptions:
{{- if .podSecurityStandard.exemptions.namespaces }}
namespaces: [{{ $namespace_exemptions }}]
{{- else }}
namespaces: [kube-system, tkg-system, vmware-system-cloud-provider]
{{- end }}
owner: root:root
path: /etc/kubernetes/extra-config/admission-control-config.yaml
permissions: "0640"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ and (not .podSecurityStandard.deactivated) (semverCompare "^1.26.x-0"
.builtin.controlPlane.version) }}'
name: controlPlaneFilesAdmissionConfigurationk8s126
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
encoding: base64
owner: root:root
path: /etc/kubernetes/extra-config/encryption-provider-config.yaml
permissions: "0640"
content: |
{{ .clusterEncryptionConfigYaml }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlaneFilesEncryptionConfiguration
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/rolloutBefore
valueFrom:
template: |
certificatesExpiryDays: {{ .controlPlaneCertificateRotation.daysBefore }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ .controlPlaneCertificateRotation.activate }}'
name: kcptCertificateRotation
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
owner: root:root
path: /etc/ssl/certs/extensions-tls.crt
permissions: "0644"
contentFrom:
secret:
name: {{ .extensionCert.contentSecret.name }}
key: {{ .extensionCert.contentSecret.key }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
owner: root:root
path: /etc/ssl/certs/extensions-tls.crt
permissions: "0644"
contentFrom:
secret:
name: {{ .extensionCert.contentSecret.name }}
key: {{ .extensionCert.contentSecret.key }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: ExtensionCerts
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/ntp
valueFrom:
template: |
enabled: true
servers:
- {{ .ntp }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/ntp
valueFrom:
template: |
enabled: true
servers:
- {{ .ntp }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: NTP
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
{{ .defaultRegistrySecret.data }}
owner: root:root
encoding: base64
path: /etc/ssl/certs/{{ .defaultRegistrySecret.name }}-{{ .defaultRegistrySecret.namespace }}-ca.pem
permissions: "0644"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files/-
valueFrom:
template: |
content: |
{{ .defaultRegistrySecret.data }}
owner: root:root
encoding: base64
path: /etc/ssl/certs/{{ .defaultRegistrySecret.name }}-{{ .defaultRegistrySecret.namespace }}-ca.pem
permissions: "0644"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .defaultRegistrySecret }}true{{end}}'
name: defaultRegistrySecretFile
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
{{if semverCompare "^1.24.0-0" .builtin.controlPlane.version}}apiVersion: audit.k8s.io/v1{{else}}apiVersion: audit.k8s.io/v1beta1{{ end }}
# Log all requests at the Metadata level.
kind: Policy
rules:
- level: Metadata
owner: root:root
path: /etc/kubernetes/extra-config/audit-policy.yaml
permissions: "0640"
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
owner: root:root
path: /var/log/kubernetes/kube-apiserver.log
permissions: "0600"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: AuditLogging
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
template: |
content: |
---
apiVersion: v1
kind: Config
# clusters refers to the remote service.
clusters:
- name: guest-cluster-auth-service
cluster:
# Use the Kubernetes CA to verify the guest cluster auth service.
certificate-authority: /etc/ssl/certs/extensions-tls.crt # CA for verifying the remote service.
server: https://localhost:5443/tokenreview
# Users refers to the API server's webhook configuration.
users:
- name: guest-apiserver
user:
client-certificate: /etc/kubernetes/pki/apiserver.crt
client-key: /etc/kubernetes/pki/apiserver.key
# kubeconfig files require a context. Provide one for the API server.
current-context: webhook
contexts:
- context:
cluster: guest-cluster-auth-service
user: guest-apiserver
name: webhook
owner: root:root
path: /etc/kubernetes/auth-webhook-config.yaml
permissions: "0600"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlaneFilesAuthWebhookConfig
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands
valueFrom:
template: |
- touch /root/kubeadm-complete
- vmware-rpctool 'info-set guestinfo.kubeadm.phase complete'
- vmware-rpctool 'info-set guestinfo.kubeadm.error ---'
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/postKubeadmCommands
valueFrom:
template: |
- touch /root/kubeadm-complete
- vmware-rpctool 'info-set guestinfo.kubeadm.phase complete'
- vmware-rpctool 'info-set guestinfo.kubeadm.error ---'
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: controlPlanePostKubeadmCommandsSuccess
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .controlPlaneVolumes }}
{{ $mountPathTransformed := .mountPath | replace "/" "_" }}
{{ $part := "1" }}
- umount {{ .mountPath }}
- '[ "$(ls -A {{ .mountPath }} )" ] && mkdir -p /var/tmp/{{ $mountPathTransformed }} &&
mv {{.mountPath}}/* /var/tmp/{{ $mountPathTransformed }}'
- mount -t ext4 /dev/sd{{ index $disks $i }}{{ $part }} {{ .mountPath }}
- rm -rf {{ .mountPath }}/lost+found
- '[ "$(ls -A /var/tmp/{{ $mountPathTransformed }})" ] && mv /var/tmp/{{ $mountPathTransformed }}/*
{{ .mountPath }} && rmdir /var/tmp/{{ $mountPathTransformed }}'
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/0
valueFrom:
template: |
set -xe
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/volumes
valueFrom:
template: |
{{- range .controlPlaneVolumes }}
- capacity:
storage: {{ .capacity.storage }}
name: {{ .name }}
{{ if .storageClass }}storageClass: {{ .storageClass }}{{end}}
{{- end }}
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/diskSetup/filesystems
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .controlPlaneVolumes }}
- device: /dev/sd{{ index $disks $i }}
extraOpts:
- -F
filesystem: ext4
label: ""
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/kubeadmConfigSpec/diskSetup/partitions
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .controlPlaneVolumes }}
- device: /dev/sd{{ index $disks $i }}
layout: true
overwrite: false
tableType: gpt
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/kubeadmConfigSpec/mounts
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .controlPlaneVolumes }}
{{ $part := "1" }}
- - /dev/sd{{ index $disks $i }}{{ $part }}
- {{ .mountPath }}
{{ $i = add1 $i }}
{{- end }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if .controlPlaneVolumes }}true{{end}}'
name: controlPlaneVolumes
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
set -xe
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if .controlPlaneVolumes }}true{{end}}'
name: controlPlanePreKubeadmCommandsCmdBashOptions
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands
valueFrom:
template: |
- set -xe
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if not .controlPlaneVolumes }}true{{end}}'
name: controlPlaneInitPreKubeadmCommandsCmdBashOptions
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
cloud-init single --name write-files --frequency always
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdCloudInitExecuteWriteFile
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
cloud-init single --name users-groups --frequency always
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdCloudInitAddUsers
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
vmware-rpctool 'info-set guestinfo.userdata ---'
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdClearuserData
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
hostname "{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdHostname
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
'sed -i -e "s/^preserve_hostname: .*/preserve_hostname: true/" /etc/cloud/cloud.cfg'
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdPreserveHostname
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
echo -e 'kernel.panic_on_oops=1\nkernel.panic=10\nvm.overcommit_memory=1' >> /etc/sysctl.d/kubelet.conf && sysctl -p /etc/sysctl.d/kubelet.conf
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdForEnablingProtectKernelDefaults
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep photon && /usr/bin/rehash_ca_certificates.sh || echo "not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdPhotonRehashCerts
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && cp /etc/ssl/certs/extensions-tls.crt >> /usr/local/share/ca-certificates/ || echo "not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdUbuntuCpExtensionsCrt
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && /usr/sbin/update-ca-certificates || echo "not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdUbuntuRehashCerts
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
cat /etc/kubernetes/pki/ca.crt >> /etc/ssl/certs/extensions-tls.crt
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdDockerCmdCatExtensionsAndK8sPKI
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep photon && systemctl daemon-reload || echo "not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdSystemctlReload
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
systemctl enable containerd
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdEnableContainerd
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
systemctl is-enabled --quiet containerd.service && systemctl restart containerd.service
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdContainerdRestart
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
"if systemctl is-enabled --quiet containerd.service ; then running=false; for _ in {1..15}; do crictl ps > /dev/null 2>&1 && running=true && break; sleep 1s; done; if [[ \"${running}\" != true ]]; then echo 'WARNING: containerd may not be running'; exit 1; fi; fi"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdContainerdWait
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep photon && systemctl start docker.service || echo "not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdPhotonDockerWait
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && systemctl enable kubelet && systemctl start kubelet || echo "Not applicable"
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsUbuntuCommonKubeletCmds
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
valueFrom:
template: |
if [ -f /root/kubeadm-complete ]; then echo "Kubeadm already completed - terminating early"; exit 0; fi
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlanePreKubeadmCommandsCmdCheckKubeadmSuccess
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authentication-token-webhook-version
value: v1
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ semverCompare "^1.22-0" .builtin.controlPlane.version }}'
name: controlPlaneAuthTokenWebhookVersion
- definitions:
- jsonPatches:
- op: replace
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/enable-admission-plugins
value: PodSecurity,NodeRestriction,NamespaceLifecycle,ServiceAccount
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ semverCompare "~1.24.x-0" .builtin.controlPlane.version }}'
name: admissionPlugins
- definitions:
- jsonPatches:
- op: replace
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/enable-admission-plugins
value: PodSecurity,NodeRestriction,NamespaceLifecycle,ServiceAccount
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ semverCompare "^1.25.x-0" .builtin.controlPlane.version }}'
name: disablePSP
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/node-ip
value: '{{ ds.meta_data.local_ipv4 }}'
- op: add
path: /spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/node-ip
value: '{{ ds.meta_data.local_ipv4 }}'
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ semverCompare ">=1.29.0-0" .builtin.controlPlane.version }}'
name: controlPlaneSetNodeIP
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/node-ip
value: '{{ ds.meta_data.local_ipv4 }}'
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ semverCompare ">=1.29.0-0" .builtin.machineDeployment.version }}'
name: machineDeploymentSetNodeIP
- definitions:
- jsonPatches:
- op: replace
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/tls-cipher-suites
value: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/tls-min-version
value: VersionTLS12
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/controllerManager/extraArgs/tls-min-version
value: VersionTLS12
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/scheduler/extraArgs/tls-min-version
value: VersionTLS12
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd/local/extraArgs
valueFrom:
template: |
auto-tls: "false"
- op: add
path: /spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/streaming-connection-idle-timeout
value: 5m
- op: add
path: /spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/streaming-connection-idle-timeout
value: 5m
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: controlPlaneSTIG
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/streaming-connection-idle-timeout
value: 5m
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: workerNodeSTIG
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/users
valueFrom:
template: |
- lockPassword: false
name: vmware-system-user
passwdFrom:
secret:
name: {{ .user.passwordSecret.name }}
key: {{ .user.passwordSecret.key }}
sshAuthorizedKeys:
- |
{{ .user.sshAuthorizedKey }}
sudo: ALL=(ALL) NOPASSWD:ALL
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/users
valueFrom:
template: |
- lockPassword: false
name: vmware-system-user
passwdFrom:
secret:
name: {{ .user.passwordSecret.name }}
key: {{ .user.passwordSecret.key }}
sshAuthorizedKeys:
- |
{{ .user.sshAuthorizedKey }}
sudo: ALL=(ALL) NOPASSWD:ALL
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .user }}true{{end}}'
name: Users
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/clusterName
valueFrom:
variable: builtin.cluster.name
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).kubernetesSpec.imageRepository
}}'
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns/imageRepository
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).kubernetesSpec.imageRepository
}}'
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns/imageTag
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).kubernetesSpec.coredns.imageTag
}}'
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd/local/imageRepository
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).kubernetesSpec.imageRepository
}}'
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd/local/imageTag
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).kubernetesSpec.etcd.imageTag
}}'
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if .TKR_DATA }}true{{end}}'
name: tkrConfiguration
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/imageName
valueFrom:
template: '{{ (index .TKR_DATA .builtin.controlPlane.version).osImageRef.name
}}'
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if .TKR_DATA }}true{{end}}'
name: tkrConfigurationControlPlane
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/imageName
valueFrom:
template: '{{ (index .TKR_DATA .builtin.machineDeployment.version).osImageRef.name
}}'
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .TKR_DATA }}true{{end}}'
name: tkrConfigurationMachineDeployment
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/node-labels
valueFrom:
template: |
run.tanzu.vmware.com/tkr={{ index (index .TKR_DATA .builtin.controlPlane.version).labels "run.tanzu.vmware.com/tkr" }},run.tanzu.vmware.com/kubernetesDistributionVersion={{ index (index .TKR_DATA .builtin.controlPlane.version).labels "run.tanzu.vmware.com/tkr" }}
- op: add
path: /spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/node-labels
valueFrom:
template: |
run.tanzu.vmware.com/tkr={{ index (index .TKR_DATA .builtin.controlPlane.version).labels "run.tanzu.vmware.com/tkr" }},run.tanzu.vmware.com/kubernetesDistributionVersion={{ index (index .TKR_DATA .builtin.controlPlane.version).labels "run.tanzu.vmware.com/tkr" }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/node-labels
valueFrom:
template: |
run.tanzu.vmware.com/tkr={{ index (index .TKR_DATA .builtin.machineDeployment.version).labels "run.tanzu.vmware.com/tkr" }},run.tanzu.vmware.com/kubernetesDistributionVersion={{ index (index .TKR_DATA .builtin.machineDeployment.version).labels "run.tanzu.vmware.com/tkr" }},{{- range .nodePoolLabels }}{{ .key }}={{ .value }},{{- end }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodeLabels
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/register-with-taints
valueFrom:
template: |
{{- range $i, $e := .nodePoolTaints }}{{ if $e.value }}{{if $i}},{{end}}{{$e.key}}={{$e.value}}:{{$e.effect}}{{ else }}{{if $i}},{{end}}{{$e.key}}:{{$e.effect}}{{ end }}{{- end }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .nodePoolTaints}}true{{end}}'
name: nodePoolTaints
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .nodePoolVolumes }}
{{ $mountPathTransformed := .mountPath | replace "/" "_" }}
{{ $part := "1" }}
- umount {{ .mountPath }}
- '[ "$(ls -A {{ .mountPath }} )" ] && mkdir -p /var/tmp/{{ $mountPathTransformed }} &&
mv {{.mountPath}}/* /var/tmp/{{ $mountPathTransformed }}'
- mount -t ext4 /dev/sd{{ index $disks $i }}{{ $part }} {{ .mountPath }}
- rm -rf {{ .mountPath }}/lost+found
- '[ "$(ls -A /var/tmp/{{ $mountPathTransformed }})" ] && mv /var/tmp/{{ $mountPathTransformed }}/*
{{ .mountPath }} && rmdir /var/tmp/{{ $mountPathTransformed }}'
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/preKubeadmCommands/0
valueFrom:
template: |
set -xe
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
- jsonPatches:
- op: add
path: /spec/template/spec/volumes
valueFrom:
template: |
{{- range .nodePoolVolumes }}
- capacity:
storage: {{ .capacity.storage }}
name: {{ .name }}
{{ if .storageClass }}storageClass: {{ .storageClass }}{{end}}
{{- end }}
selector:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
- jsonPatches:
- op: add
path: /spec/template/spec/diskSetup/filesystems
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .nodePoolVolumes }}
- device: /dev/sd{{ index $disks $i }}
extraOpts:
- -F
filesystem: ext4
label: ""
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/diskSetup/partitions
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .nodePoolVolumes }}
- device: /dev/sd{{ index $disks $i }}
layout: true
overwrite: false
tableType: gpt
{{ $i = add1 $i }}
{{- end }}
- op: add
path: /spec/template/spec/mounts
valueFrom:
template: |
{{ $disks := splitList "/" "b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z" }}
{{ $i := 0 }}
{{- range .nodePoolVolumes }}
{{ $part := "1" }}
- - /dev/sd{{ index $disks $i }}{{ $part }}
- {{ .mountPath }}
{{ $i = add1 $i }}
{{- end }}
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .nodePoolVolumes }}true{{end}}'
name: nodePoolVolumes
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
set -xe
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if .nodePoolVolumes }}true{{end}}'
name: nodePoolPreKubeadmCommandsCmdBashOptions
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands
valueFrom:
template: |
- set -xe
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
enabledIf: '{{ if not .nodePoolVolumes }}true{{end}}'
name: nodePoolInitPreKubeadmCommandsCmdBashOptions
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
cloud-init single --name write-files --frequency always
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdCloudInitExecuteWriteFile
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
cloud-init single --name users-groups --frequency always
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdAddUsers
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
vmware-rpctool 'info-set guestinfo.userdata ---'
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdClearuserData
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
hostname "{{ "{{" }} ds.meta_data.hostname.split('.') | first {{ "}}" }}"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdHostname
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
'sed -i -e "s/^preserve_hostname: .*/preserve_hostname: true/" /etc/cloud/cloud.cfg'
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdPreserveHostname
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
echo -e 'kernel.panic_on_oops=1\nkernel.panic=10\nvm.overcommit_memory=1' >> /etc/sysctl.d/kubelet.conf && sysctl -p /etc/sysctl.d/kubelet.conf
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdForEnablingProtectKernelDefaults
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep photon && /usr/bin/rehash_ca_certificates.sh
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdPhotonRehashCerts
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && cp /etc/ssl/certs/extensions-tls.crt /usr/local/share/ca-certificates/
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdUbuntuCpExtensionsCrt
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && /usr/sbin/update-ca-certificates
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdUbuntuRehashCerts
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
systemctl set-property docker.service TasksMax=infinity
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdDockerSystemTasksLimit
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
systemctl daemon-reload
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdSystemctlReload
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
systemctl enable containerd
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdEnableContainerd
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
systemctl is-enabled --quiet containerd.service && systemctl restart containerd.service
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdContainerdRestart
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
"if systemctl is-enabled --quiet containerd.service ; then running=false; for _ in {1..15}; do crictl ps > /dev/null 2>&1 && running=true && break; sleep 1s; done; if [[ \"${running}\" != true ]]; then echo 'WARNING: containerd may not be running'; exit 1; fi; fi"
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdContainerdWait
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep photon && systemctl start docker.service
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdPhotonDockerWait
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && systemctl enable kubelet
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdUbuntuCmdEnableKubelet
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
uname -a | grep ubuntu && systemctl start kubelet
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdUbuntuCmdStartKubelet
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/preKubeadmCommands/-
valueFrom:
template: |
if [ -f /root/kubeadm-complete ]; then echo "Kubeadm already completed - terminating early"; exit 0; fi
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- node-pool
name: nodePoolPreKubeadmCommandsCmdCheckKubeadmSuccess
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs
valueFrom:
variable: kubeAPIServerFQDNs
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
enabledIf: '{{ if .kubeAPIServerFQDNs }}true{{end}}'
name: kubeAPIServerFQDNs
variables:
- metadata: {}
name: TKR_DATA
required: false
schema:
openAPIV3Schema:
additionalProperties:
properties:
kubernetesSpec:
properties:
coredns:
properties:
imageRepository:
type: string
imageTag:
type: string
type: object
etcd:
properties:
imageRepository:
type: string
imageTag:
type: string
type: object
imageRepository:
type: string
kube-vip:
properties:
imageRepository:
type: string
imageTag:
type: string
type: object
pause:
properties:
imageRepository:
type: string
imageTag:
type: string
type: object
version:
type: string
type: object
labels:
additionalProperties:
type: string
type: object
osImageRef:
type: object
x-kubernetes-preserve-unknown-fields: true
type: object
type: object
- metadata: {}
name: vmClass
required: true
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: storageClass
required: true
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: storageClasses
required: false
schema:
openAPIV3Schema:
items:
type: string
type: array
- metadata: {}
name: defaultStorageClass
required: false
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: volumeSnapshotClasses
required: false
schema:
openAPIV3Schema:
items:
type: string
type: array
- metadata: {}
name: defaultVolumeSnapshotClass
required: false
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: extensionCert
required: false
schema:
openAPIV3Schema:
properties:
contentSecret:
properties:
key:
type: string
name:
type: string
type: object
type: object
- metadata: {}
name: clusterEncryptionConfigYaml
required: false
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: defaultRegistrySecret
required: false
schema:
openAPIV3Schema:
properties:
data:
type: string
name:
type: string
namespace:
type: string
type: object
- metadata: {}
name: ntp
required: false
schema:
openAPIV3Schema:
type: string
- metadata: {}
name: user
required: false
schema:
openAPIV3Schema:
properties:
passwordSecret:
properties:
key:
type: string
name:
type: string
type: object
sshAuthorizedKey:
type: string
type: object
- metadata: {}
name: nodePoolTaints
required: false
schema:
openAPIV3Schema:
items:
properties:
effect:
type: string
key:
type: string
timeAdded:
type: integer
value:
type: string
type: object
type: array
- metadata: {}
name: nodePoolLabels
required: false
schema:
openAPIV3Schema:
items:
properties:
key:
type: string
value:
type: string
type: object
type: array
- metadata: {}
name: proxy
required: false
schema:
openAPIV3Schema:
properties:
httpProxy:
type: string
httpsProxy:
type: string
noProxy:
items:
type: string
type: array
type: object
- metadata: {}
name: trust
required: false
schema:
openAPIV3Schema:
properties:
additionalTrustedCAs:
items:
properties:
name:
type: string
type: object
type: array
type: object
- metadata: {}
name: controlPlaneVolumes
required: false
schema:
openAPIV3Schema:
items:
properties:
capacity:
properties:
storage:
type: string
type: object
mountPath:
type: string
name:
type: string
storageClass:
type: string
type: object
type: array
- metadata: {}
name: nodePoolVolumes
required: false
schema:
openAPIV3Schema:
items:
properties:
capacity:
properties:
storage:
type: string
type: object
mountPath:
type: string
name:
type: string
storageClass:
type: string
type: object
type: array
- metadata: {}
name: controlPlaneCertificateRotation
required: false
schema:
openAPIV3Schema:
default: {}
properties:
activate:
default: true
type: boolean
daysBefore:
default: 90
format: int32
minimum: 7
type: integer
type: object
- metadata: {}
name: kubeAPIServerFQDNs
required: false
schema:
openAPIV3Schema:
items:
type: string
type: array
- metadata: {}
name: podSecurityStandard
required: false
schema:
openAPIV3Schema:
default: {}
properties:
audit:
description: audit sets the level for the audit PodSecurityConfiguration
mode. One of "", privileged, baseline, restricted.
enum:
- ""
- privileged
- baseline
- restricted
type: string
auditVersion:
description: auditVersion sets the version for the audit PodSecurityConfiguration
mode.
type: string
deactivated:
description: deactivated disables the patches for Pod Security Standard
via AdmissionConfiguration.
type: boolean
enforce:
description: enforce sets the level for the enforce PodSecurityConfiguration
mode. One of "", privileged, baseline, restricted.
enum:
- ""
- privileged
- baseline
- restricted
type: string
enforceVersion:
description: enforceVersion sets the version for the enforce PodSecurityConfiguration
mode.
type: string
exemptions:
description: exemption configuration for the PodSecurityConfiguration.
properties:
namespaces:
description: namespaces excluded to apply PodSecurityConfiguration
Admission.
items:
type: string
type: array
type: object
warn:
description: warn sets the level for the warn PodSecurityConfiguration
mode. One of "", privileged, baseline, restricted.
enum:
- ""
- privileged
- baseline
- restricted
type: string
warnVersion:
description: warnVersion sets the version for the warn PodSecurityConfiguration
mode.
type: string
type: object
workers:
machineDeployments:
- class: node-pool
machineHealthCheck:
maxUnhealthy: 100%
nodeStartupTimeout: 2h0m0s
unhealthyConditions:
- status: Unknown
timeout: 5m0s
type: Ready
- status: "False"
timeout: 12m0s
type: Ready
template:
bootstrap:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: tkc-md
namespace: auditing
infrastructure:
ref:
apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
kind: VSphereMachineTemplate
name: tkc-md
namespace: auditing
metadata:
annotations:
run.tanzu.vmware.com/resolve-os-image: os-name=photon
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment