1. Docker via GS VPN

If running Kind in a VM accessible via GS VPN (e.g. Neoedge), you must change the docker bridge network or it will set it to 172.18.0.0 which conflicts with our Gridscale VPN endpoint.

Set this in /etc/docker/daemon.json

{
  "default-address-pools":[
    {"base":"10.253.0.0/16","size":24}
  ]
}

2. kind config

Also here

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
  - role: control-plane
    kubeadmConfigPatches:
      - |
        kind: ClusterConfiguration
        apiServer:
          certSANs:
            - "192.168.1.200"  # Add your LAN IP here
    extraPortMappings:
    - containerPort: 10000
      hostPort: 10000
      listenAddress: "0.0.0.0"
      protocol: TCP
    - containerPort: 10001
      hostPort: 10001
      listenAddress: "0.0.0.0"
      protocol: TCP
    - containerPort: 10002
      hostPort: 10002
      listenAddress: "0.0.0.0"
      protocol: TCP
    - containerPort: 10003
      hostPort: 10003
      listenAddress: "0.0.0.0"
      protocol: TCP
    - containerPort: 10004
      hostPort: 10004
      listenAddress: "0.0.0.0"
      protocol: TCP
    - containerPort: 6443
      hostPort: 6443
      listenAddress: "0.0.0.0"
      protocol: TCP
networking:
  disableDefaultCNI: true

3. Deploy Kind cluster

# Check the name of your ETHERNET NIC, example: eth0
NIC=eth0

kind create cluster --name networked-kind --config kindconfig.yaml
kind get kubeconfig --name networked-kind > kubeconfig.yaml
ip=$(ifconfig $NIC | awk '/inet / { print $2 }')
sed -i 's|^\(.*https://\).*|\1'$ip':6443|' kubeconfig.yaml
export KUBECONFIG=./kubeconfig.yaml

4. Install Cilium

cilium install --set encryption.enabled=true --set encryption.type=wireguard

5. Install keadm

# Check your arch, might be amd64
CPU_ARCH=arm64
KEADM_VERSION=1.21.0
wget https://github.com/kubeedge/kubeedge/releases/download/v${KEADM_VERSION}/keadm-v${KEADM_VERSION}-linux-$CPU_ARCH.tar.gz
tar -zxvf keadm-v${KEADM_VERSION}-linux-$CPU_ARCH.tar.gz
cp keadm-v${KEADM_VERSION}-linux-$CPU_ARCH/keadm/keadm /usr/local/bin/keadm

6. Initialize cloud component with keadm

keadm init  --kubeedge-version=v${KEADM_VERSION} --kube-config=kubeconfig.yaml --advertise-address=192.168.1.200

7. Reconfigure Cilium for kubedge

wget https://raw.githubusercontent.com/kubeedge/kubeedge/master/hack/configure_cilium.sh

sudo KUBECONFIG=./kubeconfig.yaml ./configure_cilium.sh cloudcore

8. Edge device

• 8a. Install containerd | runc | CNI plugin

CPU_ARCH=arm64

# Install containerd
wget https://github.com/containerd/containerd/releases/download/v1.7.28/containerd-1.7.28-linux-$CPU_ARCH.tar.gz
sudo tar Cxzvf /usr/local containerd-1.7.28-linux-$CPU_ARCH.tar.gz

# Switch to systemdcgroups
sudo mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/,/^\s*\[/{s/SystemdCgroup = false/SystemdCgroup = true/}' /etc/containerd/config.toml

# Create systemd unit
wget https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
sudo mv containerd.service /etc/systemd/system
systemctl daemon-reload
systemctl enable --now containerd

# Install runc
wget https://github.com/opencontainers/runc/releases/download/v1.3.0/runc.$CPU_ARCH
sudo install -m 755 runc.$CPU_ARCH /usr/local/sbin/runc

# Install CNI plugin
wget https://github.com/containernetworking/plugins/releases/download/v1.7.1/cni-plugins-linux-$CPU_ARCH-v1.7.1.tgz
mkdir -p /opt/cni/bin
sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-$CPU_ARCH-v1.7.1.tgz

# Enable cgroup limits
sudo sed -i '$ s/$/ cgroup_enable=cpuset cgroup_enable=memory cgroup_memory=1 swapaccount=1/' /boot/firmware/cmdline.txt

• 8b. Install/Join KubeEdge

# On Cloud core
keadm gettoken --kube-config kubeconfig.yaml

# On Edge core
keadm join --cloudcore-ipport="192.168.1.200":10000 --token ${JOIN_TOKEN} --kubeedge-version=v1.21.0 --remote-runtime-endpoint=unix:///run/containerd/containerd.sock --cgroupdriver=systemd

• 8c. Run cilium script

wget https://raw.githubusercontent.com/kubeedge/kubeedge/master/hack/configure_cilium.sh

sudo ./configure_cilium.sh edgecore

9. Helpful tools

sudo apt-get update && apt-get -y install golang-go make net-tools mosquitto-clients

# crictl client
CPU_ARCH=arm64
VERSION="v1.32.0"
curl -LO https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-$CPU_ARCH.tar.gz
sudo tar -C /usr/local/bin -xzf crictl-$VERSION-linux-$CPU_ARCH.tar.gz

sudo tee /etc/crictl.yaml > /dev/null <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF