Skip to content

Instantly share code, notes, and snippets.

@apconole

apconole/sdnctl

Created February 3, 2025 17:14
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save apconole/ed78c9a2e76add9942dc3d6cbcfff4ca to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save apconole/ed78c9a2e76add9942dc3d6cbcfff4ca to your computer and use it in GitHub Desktop.
Download ZIP
Raw
sdnctl
#!/bin/bash
# Copyright (C) 2025, Red Hat, Inc.
#
# Set up an openshift SDN-lite environment
#
# Based on:
# https://github.com/openshift/sdn/blob/master/pkg/network/node/ovscontroller.go
# https://github.com/openshift/sdn/blob/master/pkg/network/node/pod.go
# https://github.com/openshift/sdn/blob/master/pkg/network/node/iptables.go
#
# Usage:
# $ ./sdnctl start
# ... output
# $ ./sdnctl status
# OVS Running: [OK]
# Vxlan/tun ports up: [OK]
# OVS Flows: 55
# PODS: 0
# Services: 0
# $ ./sdnctl add-pod
# ... output
# ✅ Pod with ns->pod1 and ip->10.128.220.225 should be added.
# $ sudo ip netns exec pod1 ping 192.168.1.1
# PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
# 64 bytes from 192.168.1.1: icmp_seq=1 ttl=63 time=0.730 ms
# 64 bytes from 192.168.1.1: icmp_seq=2 ttl=63 time=0.494 ms
# 64 bytes from 192.168.1.1: icmp_seq=3 ttl=63 time=0.427 ms
# ...
# --- 192.168.1.1 ping statistics ---
# 3 packets transmitted, 3 received, 0% packet loss, time 2037ms
# rtt min/avg/max/mdev = 0.427/0.550/0.730/0.129 ms
# $ ./sdnctl add-pod
# ... output
# ✅ Pod with ns->pod2 and ip->10.131.1.220 should be added.
# $ sudo ip netns exec pod2 ping 10.128.220.225
#
# add/del/list/flush pods
# start/stop
#
# output from this 'cluster' is implemented by way of MASQ which
# should internally become an SNAT for testing, but it is easy enough
# to adjust the tables. The tables for the openflow pipeline are basically
# documented. The IPTables ruleset is not well documented.
set -e
EXT_IF="enp1s0" # External interface
TUN_IF="tun0" # Virtual tunnel interface
TUN_MTU="1462" # Tunnel MTU
POD_SUBNET="10.128.0.0/14" # Pod IP addresses
LOCAL_SUBNET="192.168.122.0/24" # Underlay addresses
LOCAL_SUBNET_GW="192.168.122.1" # local GW
SERVICE_SUBNET="172.31.110.0/24" # Services subnet
NUM_PODS_DEFAULT="0" # Number of pods by default
function info() {
logger $*
echo $*
}
function fatal() {
echo $* >&2
exit 1
}
function status() {
# check that OVS is running
echo -n "OVS Running: "
sudo ovs-vsctl show 2>&1 >/dev/null && echo "[OK]" || echo "[NO]"
GOOD="no"
echo -n "Vxlan/tun ports up: "
ip l s vxlan_sys_4789 >/dev/null 2>&1 && \
ip l s $TUN_IF && GOOD="yes" && echo "[OK]"
[[ $GOOD == "no" ]] && echo "[NO]"
echo -n "OVS Flows: "
FLOWS=$(sudo ovs-ofctl dump-flows br0 | wc -l)
[[ -z "$FLOWS" ]] && FLOWS=0
echo "$FLOWS"
echo -n "PODS: "
PODS=$(ip netns list | grep -E pod[0-9]+ | wc -l)
[[ -z "$PODS" ]] && PODS=0
echo $PODS
echo -n "Services: "
svcs=$(sudo iptables -t nat -S OPENSHIFT-SERVICE-FORWARD | wc -l)
svcs=$((svcs-1))
echo $svcs
}
function dot1_ip_from_cidr() {
# Convert CIDR notation to network and netmask
local cidr="$1"
local network=$(ipcalc -n "$cidr" | cut -d= -f2)
local netmask=$(ipcalc -m "$cidr" | cut -d= -f2)
# Convert network to integer
IFS=. read -r i1 i2 i3 i4 <<< "$network"
local net_int=$(( (i1<<24) + (i2<<16) + (i3<<8) + i4 + 1))
printf "%d.%d.%d.%d" $(( (net_int>>24) & 255 )) $(( (net_int>>16) & 255 )) $(( (net_int>>8) & 255 )) $(( net_int & 255 ))
}
function random_ip_from_cidr() {
# Convert CIDR notation to network and netmask
local cidr="$1"
local network=$(ipcalc -n "$cidr" | cut -d= -f2)
local netmask=$(ipcalc -m "$cidr" | cut -d= -f2)
# Convert network to integer
IFS=. read -r i1 i2 i3 i4 <<< "$network"
local net_int=$(( (i1<<24) + (i2<<16) + (i3<<8) + i4 ))
# Calculate the number of usable IPs in the subnet
local prefix=${cidr#*/} # Extract the subnet prefix
local host_bits=$((32 - prefix))
local ip_range=$(( (1 << host_bits) - 2 )) # Excluding network & broadcast
# Generate a random host offset within the valid range
local rand_offset=$(( (RANDOM<<15 | RANDOM) % ip_range + 1 ))
# Compute the final random IP
local rand_ip_int=$(( net_int + rand_offset ))
printf "%d.%d.%d.%d" $(( (rand_ip_int>>24) & 255 )) $(( (rand_ip_int>>16) & 255 )) $(( (rand_ip_int>>8) & 255 )) $(( rand_ip_int & 255 ))
}
function hw_addr_if() {
local interface="$1"
# Ensure the interface name is provided
if [[ -z "$interface" ]]; then
fatal "Error: No interface specified"
fi
# Extract the MAC address using the 'ip' command
local mac_address=$(ip link show "$interface" | awk '/ether/ {print $2}')
# Check if we got a result
if [[ -z "$mac_address" ]]; then
fatal "Error: No interface mac for $interface"
fi
echo "$mac_address"
}
function ip_addr_if() {
local interface="$1"
# Ensure the interface name is provided
if [[ -z "$interface" ]]; then
fatal "Error: No interface specified"
fi
# Extract the first IP address using the 'ip' command
local ip_address=$(ip -4 addr show $interface | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
# Check if we got a result
if [[ -z "$ip_address" ]]; then
fatal "Error: No interface IP for $interface"
fi
echo "$ip_address"
}
function basic_iptables_start() {
info " -- 🔥 Removing iptables NAT and FILTER rules..."
flushiptables
info " -- 🔧 Building initial tables needed chains..."
# construct the chains we need
sudo iptables -t filter -N OPENSHIFT-FIREWALL-ALLOW
sudo iptables -t filter -N OPENSHIFT-ADMIN-OUTPUT-RULES
sudo iptables -t filter -N OPENSHIFT-FIREWALL-FORWARD
sudo iptables -t nat -N OPENSHIFT-SERVICE-FORWARD
sudo iptables -t nat -N OPENSHIFT-MASQUERADE
sudo iptables -t nat -N OPENSHIFT-MASQUERADE-2
# setup the forward path of filter table
info " -- 🚀 Connecting input/forwarding path..."
sudo iptables -t filter -A INPUT -j OPENSHIFT-FIREWALL-ALLOW
sudo iptables -t filter -A OPENSHIFT-FIREWALL-ALLOW -p udp --dport 4789 -j ACCEPT
sudo iptables -t filter -A OPENSHIFT-FIREWALL-ALLOW -i $TUN_IF -j ACCEPT
sudo iptables -t filter -A FORWARD -j OPENSHIFT-FIREWALL-FORWARD
sudo iptables -t filter -A OPENSHIFT-FIREWALL-FORWARD -i $TUN_IF '!' -o $TUN_IF -m comment --comment "Admin Overrides"
info " -- 🚀 Connecting masquerade path..."
sudo iptables -t nat -A POSTROUTING -j OPENSHIFT-MASQUERADE
sudo iptables -t nat -A OPENSHIFT-MASQUERADE -s $POD_SUBNET -j OPENSHIFT-MASQUERADE-2
sudo iptables -t nat -A OPENSHIFT-MASQUERADE-2 -d $POD_SUBNET -j RETURN
sudo iptables -t nat -A OPENSHIFT-MASQUERADE-2 -j MASQUERADE
sudo iptables -t filter -A OPENSHIFT-FIREWALL-FORWARD -s $POD_SUBNET -m conntrack --ctstate INVALID -j DROP
sudo iptables -t filter -A OPENSHIFT-FIREWALL-FORWARD -d $POD_SUBNET -j ACCEPT
sudo iptables -t filter -A OPENSHIFT-FIREWALL-FORWARD -s $POD_SUBNET -j ACCEPT
info " -- 🚀 Connecting service forward path..."
sudo iptables -t nat -A PREROUTING -j OPENSHIFT-SERVICE-FORWARD
}
function flushiptables() {
sudo iptables -t nat -F
sudo iptables -t nat -X OPENSHIFT-MASQUERADE || true
sudo iptables -t nat -X OPENSHIFT-MASQUERADE-2 || true
sudo iptables -t filter -F
sudo iptables -t filter -X OPENSHIFT-FIREWALL-ALLOW || true
sudo iptables -t filter -X OPENSHIFT-ADMIN-OUTPUT-RULES || true
sudo iptables -t filter -X OPENSHIFT-FIREWALL-FORWARD || true
sudo iptables -t nat -X OPENSHIFT-SERVICE-FORWARD || true
sudo iptables -t filter -P INPUT ACCEPT
sudo iptables -t filter -P FORWARD ACCEPT
sudo iptables -t filter -P OUTPUT ACCEPT
}
function basic_flow_start_ovs() {
info " -- 🚀 Building initial OVS flows..."
sudo ovs-ofctl del-flows br0
sudo ovs-ofctl add-flow br0 \
"table=0, priority=1000, ip, ct_state=-trk actions=ct(table=0)"
info " -- 🌍 Basic vxlan port ..."
sudo ovs-ofctl add-flow br0 \
"table=0, priority=200, in_port=vxlan0, arp, nw_src=$POD_SUBNET, nw_dst=$LOCAL_SUBNET, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=200, in_port=vxlan0, ip, nw_src=$POD_SUBNET, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=200, in_port=1, ip, nw_dst=$POD_SUBNET, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10"
# explicit drop counter the way mom^WOVN controller does it
sudo ovs-ofctl add-flow br0 \
"table=0, priority=150, in_port=vxlan0 actions=drop"
info " -- 🌍 $TUN_IF communications"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=400, in_port=${TUN_IF}, ip, nw_src=$LOCAL_SUBNET_GW actions=goto_table:30"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=300, in_port=${TUN_IF}, ip, nw_src=${LOCAL_SUBNET}, nw_dst=$POD_SUBNET actions=goto_table:25"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=300, in_port=${TUN_IF}, ip, nw_src=${POD_SUBNET}, nw_dst=$POD_SUBNET actions=goto_table:25"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=250, in_port=${TUN_IF}, ip, nw_dst=224.0.0.0/4 actions=drop"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=200, in_port=${TUN_IF}, arp, nw_src=$(dot1_ip_from_cidr $POD_SUBNET) actions=goto_table:30"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=200, in_port=${TUN_IF}, ip actions=goto_table:25"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=150, in_port=${TUN_IF} actions=drop"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=100, arp actions=goto_table:20"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=100, ip actions=goto_table:20"
sudo ovs-ofctl add-flow br0 \
"table=0, priority=0, actions=drop"
# Table 10: vxlan ingress filtering
info " -- 🌍 VXLAN ingress filtering ..."
sudo ovs-ofctl add-flow br0 \
"table=10, priority=210, ip, nw_dst=${LOCAL_SUBNET_GW}, eth_dst=$(hw_addr_if $EXT_IF) actions=set_field:$(hw_addr_if $TUN_IF)->eth_dst, resubmit:10"
sudo ovs-ofctl add-flow br0 \
"table=10, priority=200, ip, nw_dst=${LOCAL_SUBNET}, eth_dst=$(hw_addr_if $EXT_IF), actions=move:nw_dst->eth_dst[0..31], set_field:0a:58:00:00:00:00/ff:ff:00:00:00:00->eth_dst, resubmit:10"
sudo ovs-ofctl add-flow br0 \
"table=10, priority=0, actions=drop"
# Table 20: Pod addresses - this is filled during pod creation so just
# add basic flows here
info " -- 🌍 Pod handling table ..."
sudo ovs-ofctl add-flow br0 \
"table=20, priority=300, udp, udp_dst=4789 actions=drop"
sudo ovs-ofctl add-flow br0 \
"table=20, priority=0 actions=drop"
# Table 25: IP via Service
# This is filled in when a pod is setup with a service map
info " -- 🌍 Service IP handling table ..."
sudo ovs-ofctl add-flow br0 \
"table=25, priority=50, in_port=${TUN_IF},ip,ct_state=+rpl actions=goto_table:27"
sudo ovs-ofctl add-flow br0 \
"table=25, priority=0 actions=drop"
# Table 27: Network Policy use
# NOTE: we don't do the drop here as openshift does to be
# lighter - we won't ever use a network policy plugin
# most likely
info " -- 🌍 NetworkPolicy spare table ..."
sudo ovs-ofctl add-flow br0 \
"table=27, priority=0 actions=goto_table:30"
# Table 30: General 'Routing'; but this one is used by networkpolicy
# to add conntrack rules. HOWEVER, we keep this one because it lets us
# inject rules
info " -- 🌍 General Routing pass-through table ..."
sudo ovs-ofctl add-flow br0 \
"table=30, priority=0 actions=goto_table:31"
# Table 31: 'Real' version of General 'Routing'
# This is where normal forwarding path happens
info " -- 🌍 General Routing table ..."
sudo ovs-ofctl add-flow br0 \
"table=31, priority=300, arp, nw_dst=${LOCAL_SUBNET_GW} actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=300, ip, nw_dst=${LOCAL_SUBNET_GW} actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=250, ip, nw_dst=${POD_SUBNET}, ct_state=+rpl, actions=ct(nat,table=70)"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=200, arp, nw_dst=${POD_SUBNET} actions=goto_table:40"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=200, ip, nw_dst=${POD_SUBNET} actions=goto_table:70"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=100, ip, nw_dst=${SERVICE_SUBNET} actions=goto_table:60"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=100, ip, nw_dst=${LOCAL_SUBNET} actions=goto_table:90"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=50, in_port=vxlan0, ip, nw_dst=224.0.0.0/4 actions=goto_table:120"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=25, ip, nw_dst=224.0.0.0/4 actions=goto_table:110"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=0, ip, actions=goto_table:90"
sudo ovs-ofctl add-flow br0 \
"table=31, priority=0, arp, actions=drop"
# Table 40: Direct pod mapping
# ex: table=40, priority=100, arp, nw_dst=${pod_ip} actions=output:$pod_port
info " -- 🌍 Direct port mapping table ..."
sudo ovs-ofctl add-flow br0 \
"table=40, priority=0, actions=drop"
sudo ovs-ofctl add-flow br0 \
"table=40, priority=10, arp, nw_dst=$(ip_addr_if $TUN_IF) actions=output:$TUN_IF"
# Table 50: ARP to remote
# ex: table=50, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:vxlan0
info " -- 🌍 ARP to remote table ..."
sudo ovs-ofctl add-flow br0 \
"table=50, priority=0 actions=drop"
# Table 60: IP to service from pod
info " -- 🌍 IP to service from pod table ..."
sudo ovs-ofctl add-flow br0 \
"table=60, priority=0 actions=output:$TUN_IF"
# Table 70: IP to local container
info " -- 🌍 IP to local table ..."
sudo ovs-ofctl add-flow br0 \
"table=70, priority=0 actions=drop"
# Table 80: ingress network policy
info " -- 🌍 Ingress network policy table ..."
sudo ovs-ofctl add-flow br0 \
"table=80, priority=300,ip,nw_src=${LOCAL_SUBNET_GW}/32 actions=output:NXM_NX_REG2[]"
sudo ovs-ofctl add-flow br0 \
"table=80,priority=200,ip,nw_src=${POD_SUBNET},nw_dst=${POD_SUBNET} actions=output:NXM_NX_REG2[]"
sudo ovs-ofctl add-flow br0 \
"table=80,priority=150,ip,nw_dst=${POD_SUBNET},ct_state=+trk-inv actions=output:NXM_NX_REG2[]"
sudo ovs-ofctl add-flow br0 \
"table=80, priority=0 actions=drop"
# Table 90: IP to remote container
info " -- 🌍 IP to remote table ..."
sudo ovs-ofctl add-flow br0 \
"table=90, priority=50, actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=90, priority=0 actions=drop"
# Table 99: Legacy DNS rules to override ingress/egress IP and
# network policy
info " -- 🌍 Legacy DNS table ..."
sudo ovs-ofctl add-flow br0 \
"table=99, priority=200, tcp, tcp_dst=53, nw_dst=$(ip_addr_if $EXT_IF) actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=99, priority=200, udp, udp_dst=53, nw_dst=$(ip_addr_if $EXT_IF) actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=99, priority=0 actions=goto_table:100"
# Table 100: Egress network policy
# eg, table=100, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop
info " -- 🌍 Egress Network Policy table ..."
sudo ovs-ofctl add-flow br0 \
"table=100, priority=0 actions=goto_table:101"
# Table 101: egress routing
info " -- 🌍 Egress Routing table ..."
sudo ovs-ofctl add-flow br0 \
"table=101, priority=150, ct_state=+rpl actions=output:$TUN_IF"
sudo ovs-ofctl add-flow br0 \
"table=101, priority=0 actions=drop"
# Table 111: multicast delivery from local pods to the vxlan
info " -- 🌍 Multicast delivery table ..."
sudo ovs-ofctl add-flow br0 \
"table=111, priority=100 actions=goto_table:120"
sudo ovs-ofctl add-flow br0 \
"table=120, priority=0 actions=drop"
}
function start() {
info "🚀 Starting OVS SDN environment..."
if ! command -v ovs-vsctl &> /dev/null; then
fatal "🔧 Install Open vSwitch!"
fi
info "🌍 Enabling IP forwarding..."
echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward
info "🔧 Checking OVS is started..."
sudo ovs-vsctl show 2>&1 >/dev/null || sudo systemctl start openvswitch
info "🔧 Creating OVS bridges..."
sudo ovs-vsctl add-br br0 || true
sudo ip link set br0 up
info "🔌 Attaching $EXT_IF to br-ex..."
sudo ovs-vsctl add-port br0 $EXT_IF || true
sudo ip link set $EXT_IF up
SNAT_ADDR=$(ip_addr_if $EXT_IF)
info "🚀 Creating vxlan port ..."
sudo ovs-vsctl add-port br0 vxlan0 -- set Interface vxlan0 \
type=vxlan options:remote_ip="flow" options:key="flow"
info "🚀 Creating $TUN_IF ..."
#sudo ip tuntap add mode tun dev tun0 || true
sudo ovs-vsctl add-port br0 $TUN_IF -- set Interface $TUN_IF \
type=internal mtu_request=$TUN_MTU
sudo ip link set $TUN_IF up
sudo ip addr add $(dot1_ip_from_cidr $POD_SUBNET)/$(echo "$POD_SUBNET" | cut -d/ -f2) dev $TUN_IF
info "⚡ Configuring OpenFlow rules..."
basic_flow_start_ovs
info "⚡ Configuring IPTables rules..."
basic_iptables_start
[[ -z "${NUM_PODS_DEFAULT}" ]] ||
for ADDPOD in $(seq $NUM_PODS_DEFAULT); do
addpod
done
info "✅ OVS SDN Setup Complete!"
sudo ovs-vsctl show
}
function stop() {
info "🛑 Stopping OVS SDN environment..."
info "🧹 Removing Service / IP tables ..."
flushiptables
info "🧹 Removing Pods ..."
flushpods
info "🧹 Removing $TUN_IF ..."
sudo ovs-vsctl del-port $TUN_IF || true
info "🧹 Removing OpenFlow rules..."
sudo ovs-ofctl del-flows br0 || true
info "🧹 Removing OVS bridges..."
sudo ovs-vsctl del-br br0 || true
info "🧹 Removing OVS bridges..."
sudo systemctl stop openvswitch
info "✅ OVS SDN Environment Stopped!"
}
function restart() {
info "🔄 Restarting OVS SDN networking..."
stop
start
}
function find_first_pod_id() {
# start at 1
local EXISTING_ID=1
while [[ $(ip netns list | grep -E "pod${EXISTING_ID} ") ]]; do
EXISTING_ID=$((EXISTING_ID+1))
done
echo $EXISTING_ID
}
function addpod() {
info "🔗 Starting pod creation..."
local PODS=$(ip netns list | grep -E pod[0-9]+ | wc -l)
[[ -z "$PODS" ]] && PODS=0
local I=$(find_first_pod_id)
info "🔥 Detected $PODS previous pods already, starting pod$I"
info " -- 🔧 Configuring pod${I} veths..."
sudo ip link add pod${I}veth0 type veth peer name pod${I}veth1 || true
info " -- 🔧 Attaching veths..."
sudo ovs-vsctl add-port br0 pod${I}veth0 || true
info " -- 🔧 Setting UP status for veths..."
sudo ip link set pod${I}veth0 up
sudo ip link set pod${I}veth1 up
info "⚡ Configuring netns pod${I}..."
sudo ip netns add pod${I}
sudo ip link set pod${I}veth1 netns pod${I}
sudo ip netns exec pod${I} ip link set pod${I}veth1 up
info "🌐 Assigning IP to veth1..."
# pick a random IP
local IP=""
while [[ -z "$IP" ]]; do
local NEXT_IP=$(random_ip_from_cidr $POD_SUBNET)
if ! ovs-ofctl dump-flows br0 | grep $NEXT_IP >/dev/null 2>&1; then
IP=$NEXT_IP
fi
done
sudo ip netns exec pod${I} ip addr add $IP/$(echo "$POD_SUBNET" | cut -d/ -f2) dev pod${I}veth1 || true
local OFPORT=$(sudo ovs-vsctl get Interface pod${I}veth0 ofport)
local PODIP1=$(echo "$IP" | cut -d. -f1)
local PODIP2=$(echo "$IP" | cut -d. -f2)
local PODIP3=$(echo "$IP" | cut -d. -f3)
local PODIP4=$(echo "$IP" | cut -d. -f4)
local PODMAC=$(printf "00:00:%02x:%02x:%02x:%02x" $PODIP1 $PODIP2 $PODIP3 $PODIP4)
local FLOWMAC=${PODMAC}/"00:00:ff:ff:ff:ff"
info "🌐 Setting hwaddr to ${PODMAC}..."
sudo ip netns exec pod${I} ip link set pod${I}veth1 address $PODMAC
info "⚡ Configuring pod${I}veth0 related openflow rules..."
info " ⚡ ARP and IP VNI related rules in tables 20/25..."
sudo ovs-ofctl add-flow br0 \
"table=20,priority=100,in_port=pod${I}veth0,arp,nw_src=$IP,arp_sha=${FLOWMAC} actions=load:${I}->NXM_NX_REG2[],goto_table:30"
sudo ovs-ofctl add-flow br0 \
"table=20,priority=100,in_port=pod${I}veth0,ip,nw_src=$IP actions=load:${I}->NXM_NX_REG0[],goto_table:27"
sudo ovs-ofctl add-flow br0 \
"table=25,priority=100,ip,nw_src=${IP} actions=load:${I}->NXM_NX_REG0[],goto_table:27"
info " ⚡ ARP req/rep related rules in table 40..."
sudo ovs-ofctl add-flow br0 \
"table=40,priority=100,arp,nw_dst=$IP actions=output:pod${I}veth0"
info " ⚡ IP traffic related rules in table 70..."
sudo ovs-ofctl add-flow br0 \
"table=70, priority=100,ip,nw_dst=$IP actions=load:${I}->NXM_NX_REG1[],load:${OFPORT}->NXM_NX_REG2[],goto_table:80"
info "🌐 Adding POD route via $TUN_IF in the main netns"
local TUNIF=$(ip -4 addr show $TUN_IF | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
sudo ip netns exec pod${I} ip route add default via $TUNIF
info "✅ Pod with ns->pod${I} and ip->$IP should be added."
}
function delpod() {
[[ -z "$1" ]] && fatal "Please specify a pod ID"
local I=$1
info "🛑 Stopping pod $I from running..."
sudo ip netns exec pod$I ls >/dev/null 2>&1 || return 0;
# This looks quite a bit different because the flow deletion routines
# don't use the txn interface
local POD_IP=$(sudo ip netns exec pod${I} ip -4 addr show pod${I}veth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}' || echo "")
#POD_MAC=$(sudo ip netns exec pod${I} hw_addr_if pod${I}veth1)
if [[ ! -z "$POD_IP" ]]; then
info "🔥 Removing openflow rules..."
sudo ovs-ofctl del-flows br0 "table=20,in_port=pod${I}veth0,arp,nw_src=${POD_IP}"
sudo ovs-ofctl del-flows br0 "table=20,in_port=pod${I}veth0,ip,nw_src=${POD_IP}"
sudo ovs-ofctl del-flows br0 "table=25,ip,nw_src=${POD_IP}"
sudo ovs-ofctl del-flows br0 "table=40,arp,nw_dst=${POD_IP}"
sudo ovs-ofctl del-flows br0 "table=70,ip,nw_dst=${POD_IP}"
else
info "⚡ IP not detected - assuming failed pod."
fi
info "🔥 Killing any PIDs..."
local pids=$(sudo ip netns pids pod${I} | tr '\n' ' ')
[[ -z "$pids" ]] ||
for pid in $pids; do
info "🛑 Stopping $pid"
sudo kill -TERM $pid
done
info "🔥 Dropping services..."
delservice pod${I}
info "🔥 Removing ports and ns..."
sudo ovs-vsctl del-port pod${I}veth0 || true
sudo ip link del pod${I}veth0 || true
sudo ip netns del pod${I}
info "✅ Pod with ns->pod${I} should be deleted."
}
function flushpods() {
info "🛑 Stopping all pods in environment..."
local PODS=$(ip netns list | grep -E pod[0-9]+)
[[ -z "$PODS" ]] ||
for I in $PODS; do
local podid=$(echo $I | sed 's@^pod@@')
delpod $podid
done
}
function pods() {
echo -e "Pod NS\tIPv4 Address\tofport\tIF name\t\tpids"
for pod in $(sudo ip netns list | grep -o -E 'pod[0-9]+'); do
local podIf="${pod}veth1"
local podIp=$(sudo ip netns exec $pod ip -4 addr show $podIf | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
local podPort=$(sudo ovs-vsctl get Interface ${pod}veth0 ofport)
local pids=$(sudo ip netns pids $pod | tr '\n' ',' | sed 's@,$@@')
echo -e "${pod}\t${podIp}\t${podPort}\t${podIf}\t${pids}"
done
}
function addservice() {
local service_ip="$1"
local service_port="$2"
local service_proto="$3"
local podId="$4"
if [[ -z "$service_ip" || -z "$service_port" || -z "$service_proto" || -z "$podId" ]]; then
fatal "🛑 Need service details <ip, port, proto, pod> ..."
fi
info "⚡ Creating service link ${service_proto}:${service_ip}:${service_port} :: $podId"
if [[ ${podId:0:3} == "pod" ]]; then
podId=$(sudo ip netns exec $podId ip -4 addr show $podIf | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
fi
if [[ -z "$podId" ]]; then
fatal "🛑 couldn't map correct ip from pod details ..."
fi
sudo iptables -t nat -A OPENSHIFT-SERVICE-FORWARD -p $service_proto -d $service_ip --dport $service_port -j DNAT --to-destination $podId
info "✅ Service should be added."
}
function delservice() {
local service_descriptor="$1"
if [[ -z "$service_descriptor" ]]; then
fatal "🛑 Need service details [ip|pod] ..."
fi
if [[ ${service_descriptor:0:3} == "pod" ]]; then
service_id=$(sudo ip netns exec $service_descriptor ip -4 addr show $podIf | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
else
service_id=$service_descriptor
fi
local ipridx=0
sudo iptables -t nat -S OPENSHIFT-SERVICE-FORWARD | while IFS= read -r rule; do
if echo $rule | grep $service_id >/dev/null 2>&1; then
info "🧹 Service [$I] deletion started"
sudo iptables -t nat -D OPENSHIFT-SERVICE-FORWARD $ipridx
else
ipridx=$((ipridx+1))
fi
done
info "✅ Services for $service_descriptor should be deleted."
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
status)
status
;;
add-pod)
addpod $2 $3 $4 $5
;;
del-pod)
delpod $2 $3 $4 $5
;;
list-pods)
pods
;;
flush-pods)
flushpods
;;
add-service)
addservice $2 $3 $4 $5
;;
del-service)
delservice $2 $3 $4 $5
;;
*)
echo "Usage: $0 {start|stop|restart|add-pod|del-pod|list-pods|flush-pods|add-service|del-service}"
exit 1
;;
esac
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment