Multipath OLSRd (SEREADMO)

Some years ago patch for OLSRd v0.5.6-r2 surfaced that added support for loadbalancing single flow over multiple paths. Documentation on how to set it up is scarce. I did get it working to some extent. It's somewhat unstable and doesn't really like single mesh node having multiple network interfaces, but it does loadbalance traffic over multiple paths as promised.



Rarball of patched OLSRd contains actually two different implentations of multipath code. On is kernel module (ser_iptables) and another is implemented as userspace process (ser_routing) communicating with kernel using nfnetlink_queue. To use kernel version manual editing of OLSRd multipath plugin is required.

See http://www.jiaziyi.com/documents/MP-OLSR_thesis_full.pdf for more details.

###########################

# Install CentOS 5.9 i386

# Enable extra repos
wget http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.2-2.el5.rf.i386.rpm
rpm -Uvh rpmforge*

# Install some tools and update system
yum -y install joe screen wget bison flex unrar kernel-devel gcc automake
yum -y update

# Install vanilla kernel 2.6.21.5
yum -y install rpm-build
mkdir -p /usr/src/vanilla
cd /usr/src/vanilla
wget http://www.kernel.org/pub/linux/kernel/v2.6/linux-2.6.21.tar.bz2
wget http://pkgs.fedoraproject.org/repo/pkgs/kernel/patch-2.6.21.5.bz2/0a8b2823c5758f39fcc627d5f1b771b1/patch-2.6.21.5.bz2
tar xjf linux-2.6.21.tar.bz2
bunzip2 patch-2.6.21.5.bz2 
mv linux-2.6.21 linux-2.6.21.5
ln -sf /usr/src/vanilla/linux-2.6.21.5 /usr/src/linux
cd linux-2.6.21.5
patch -p1 <../patch-2.6.21.5
make clean && make mrproper 
cp /boot/config-`uname -r` .config
make oldconfig
make rpm
rpm -ivh --nodeps /usr/src/redhat/RPMS/i386/kernel-2.6.21.5-1.i386.rpm
mkinitrd /boot/initrd-2.6.21.5.img 2.6.21.5

# Add following lines to /etc/grub.conf before other kernels
title CentOS (2.6.21.5-1)
        root (hd0,0)
        kernel /vmlinuz-2.6.21.5 ro root=/dev/VolGroup00/LogVol00
        initrd /initrd-2.6.21.5.img
# Reboot

###########################

# Download and compile boost 1.38
mkdir -p /opt/boost
cd /opt/boost
wget http://sourceforge.net/projects/boost/files/boost/1.38.0/boost_1_38_0.tar.gz/download
tar xvzf boost_1_38_0.tar.gz
cd boost_1_38_0
./configure
make
make install
echo "/usr/local/lib/" >>/etc/ld.so.conf
ldconfig

# Download and unpack mpolsr 
mkdir -p /opt/mpolsr
cd /opt/mpolsr
wget http://www.irccyn.ec-nantes.fr/IMG/zip/mpolsr_testbed.rar.zip
unzip mpolsr_testbed.rar.zip
unrar x mpolsr_testbed.rar

# Compile kernel module
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_module
make

# Module config
cat <<'__EOF__'>/etc/modprobe.d/ser_iptables.conf
options ser_iptables act_hook=1
options ser_iptables act_device=1
options ser_iptables act_recovery=1
options ser_iptables dbg_hook=3
options ser_iptables dbg_device=9
options ser_iptables dbg_module=9
options ser_iptables dbg_djk=9
options ser_iptables nb_path=3
options ser_iptables coef_fe=2
options ser_iptables coef_fp=2
options ser_iptables udp_port=104
__EOF__

# Compile mpolsrd
cd /opt/mpolsr/mpolsr_testbed/src_olsr/olsrd-0.5.6-r2/
chmod a+x gcc-warnings ld-warnings
make
make libs

# Compile sereadmo plugin manually
cd lib
rm -rf old_sereadmo
cd sereadmo
make clean
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/olsrd_plugin.o src/olsrd_plugin.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_data.o src/ser_data.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_print.o src/ser_print.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_tc.o src/ser_tc.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/ser_device.o src/ser_device.cpp
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/eventlistener.o src/eventlistener.cpp
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/olsreventserver.o src/olsreventserver.cpp
g++ -shared -Wl,-soname,olsrd_sereadmo -Wl,--version-script=version-script.txt  -Wl,--warn-common -fPIC \
    -o olsrd_sereadmo.so.0.1 src/olsrd_plugin.o src/ser_data.o src/ser_print.o -lpthread \
    -L/usr/local/lib -lboost_thread-gcc41-mt -lboost_serialization-gcc41-mt -lboost_system-gcc41-mt \
    src/ser_tc.o src/ser_device.o src/eventlistener.o src/olsreventserver.o

# Install mpolsrd and plugins
cd ../..
make install_all

# Configure mpolsrd
sed -i.bak /etc/olsrd.conf \
    -e's/"XXX" "YYY"/"eth1" "eth2" "eth3"/g' \
    -e's/ClearScreen     yes/ClearScreen     no/g'
cat <<'__EOF__'>>/etc/olsrd.conf
LoadPlugin "olsrd_sereadmo.so.0.1"
{
PlParam "device" "/dev/sereadmo"
}
__EOF__

# libnfnetlink
mkdir -p /opt/libnfnetlink
cd /opt/libnfnetlink
wget http://ftp.netfilter.org/pub/libnfnetlink/libnfnetlink-0.0.41.tar.bz2
tar xvjf libnfnetlink-0.0.41.tar.bz2
cd libnfnetlink-0.0.41
./configure
make
make install

# libnetfilter_queue 0.0.17
mkdir -p /opt/libnetfilter_queue
cd /opt/libnetfilter_queue
wget http://ftp.netfilter.org/pub/libnetfilter_queue/libnetfilter_queue-0.0.17.tar.bz2
tar xvjf libnetfilter_queue-0.0.17.tar.bz2 
cd libnetfilter_queue-0.0.17
PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure
make
make install

# ser_routing
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_routing
chmod a+x configure
./configure
make

# Config file for ser_routing (same defaults as in source)
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_routing/src
cat <<'__EOF__'>ser_routing.conf
udp_port=104
DebugLevel_nfqueue=10
DebugLevel_olsrevent=10
DebugLevel_dijkstra=10
Dijkstra_nb_path=3
Dijkstra_coef_fe=2
Dijkstra_coef_fp=2
__EOF__

# Configure mesh network interfaces eth1, eth2 and eth3
# Reboot

###########################


# Launch mpolsrd
olsrd -nofork

# Launch ser_routing
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_routing/src
./ser_routing

# Set iptables rules
iptables -I INPUT -p udp --dport 104 -j NFQUEUE
iptables -I OUTPUT -p udp --dport 104 -j NFQUEUE
iptables -I FORWARD -p udp --dport 104 -j NFQUEUE
iptables -I INPUT --protocol 99 -j NFQUEUE
iptables -I OUTPUT --protocol 99 -j NFQUEUE
iptables -I FORWARD --protocol 99 -j NFQUEUE

# Test connection (server)
nc -v -u -l 0.0.0.0 104

# Test connection (client)
# 10.99.2.2 is our mesh interface IP and 10.99.2.1 is destination via mesh
echo foo | nc -v -u -s 10.99.2.2 10.99.2.1 104


###########################

# For kernel module version do NOT start ser_routing but instead load
# ser_iptables module. This WILL NOT work unless you have also hacked
# OLSRd plugin to switch from 7171/tcp communication channel with ser_routing
# back to /dev/sereadmo based communication.

# Load module and create /dev/sereadmo
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_module
sh ./inst_module.csh 

###########################

It's also possible to run multipath modules with OLSRd v0.6.4 requiring minimal changes.

# Grab clean olsrd sources
mkdir -p /opt/mpupdate
cd /opt/mpupdate
wget http://www.olsr.org/releases/0.5/olsrd-0.5.6-r2.tar.bz2
wget http://www.olsr.org/releases/0.6/olsrd-0.6.4.tar.bz2

# Unpack mpolsrd source
unrar x -y ../mpolsr/mpolsr_testbed.rar
mv mpolsr_testbed/src_olsr/olsrd-0.5.6-r2 ./olsrd-0.5.6-r2.mp
rm -rf mpolsr_testbed

# Create backup of patched source
tar cvjf olsrd-0.5.6-r2.mp.tar.bz2 olsrd-0.5.6-r2.mp/

# Extract clean 0.5.6-r2
tar xvjf olsrd-0.5.6-r2.tar.bz2

# Remove unneeded files prior diff from mp version
rm -rf \
 olsrd-0.5.6-r2.mp/lib/old_sereadmo/ \
 olsrd-0.5.6-r2.mp/lib/sereadmo/Doxyfile \
 olsrd-0.5.6-r2.mp/lib/sereadmo/sereadmo.kdev* \
 olsrd-0.5.6-r2.mp/olsrd.conf

# Diff mpolsrd and olsrd excluding files with uninteresting changes
diff -bBurN -x'tc_set.c' -x'process_routes.c' \
 -x'process_package.c' -x'mid_set.c' \
 olsrd-0.5.6-r2 olsrd-0.5.6-r2.mp >mpolsrd-0.5.6-r2.patch

# Extract clean 0.6.4
tar xvjf olsrd-0.6.4.tar.bz2

# Compile 0.6.4
cd olsrd-0.6.4
make
# Skip broken "pud" plugin, compile fails at least on Centos5
sed -i.bak -e's/ pud / /g' Makefile
make libs

# Apply mpolsrd plugin patch
patch -p1 <../mpolsrd-0.5.6-r2.patch

# Fix plugin compatibility with 0.6.4
sed -i.bak src/ser_tc.c -e's/olsr_u32_t/uint32_t/g'
sed -i.bak src/ser_device.cpp -e's/#include <stdio.h>/#include <stdio.h>\n#include <stdbool.h>/g'

# Compile sereadmo plugin manually
cd lib/sereadmo
make clean
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/olsrd_plugin.o src/olsrd_plugin.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_data.o src/ser_data.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_print.o src/ser_print.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -c -o src/ser_tc.o src/ser_tc.c
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/ser_device.o src/ser_device.cpp
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/eventlistener.o src/eventlistener.cpp
gcc    -fPIC -Isrc -I../../src -pthread -DOLSR_PLUGIN  -DUSE_FPM -Dlinux -DNDEBUG    -I/usr/local/include/boost-1_38/ \
                                                                                     -c -o src/olsreventserver.o src/olsreventserver.cpp
g++ -shared -Wl,-soname,olsrd_sereadmo -Wl,--version-script=version-script.txt  -Wl,--warn-common -fPIC \
    -o olsrd_sereadmo.so.0.1 src/olsrd_plugin.o src/ser_data.o src/ser_print.o -lpthread \
    -L/usr/local/lib -lboost_thread-gcc41-mt -lboost_serialization-gcc41-mt -lboost_system-gcc41-mt \
    src/ser_tc.o src/ser_device.o src/eventlistener.o src/olsreventserver.o

# Little cleanup before installing new version in case 0.5.x was already there
rm -f /usr/sbin/olsrd /usr/lib/olsrd_*
mv /etc/olsrd.conf /etc/olsrd.conf.old

# Install 0.6.4
cd /opt/mpupdate/olsrd-0.6.4
make install_all

# Don't forget sereadmo part
cp -a lib/sereadmo/olsrd_sereadmo.so.0.1 /usr/local/lib/

# Update config
sed -i.bak /etc/olsrd.conf \
  -e's/<OLSRd-Interface1>/eth1/g' \
  -e's/<OLSRd-Interface2>/eth2/g' 
cat <<'__EOF__'>>/etc/olsrd.conf
ClearScreen     no
LinkQualityLevel       0 # or 2? olsrd uses 2 while sereadmo sample config uses 0.
LoadPlugin "olsrd_sereadmo.so.0.1"
{
}
__EOF__

############################

# Launch mpolsrd
olsrd -nofork

# Launch ser_routing
cd /opt/mpolsr/mpolsr_testbed/src_iptables/ser_routing/src
./ser_routing

# Set iptables rules
iptables -I INPUT -p udp --dport 104 -j NFQUEUE
iptables -I OUTPUT -p udp --dport 104 -j NFQUEUE
iptables -I FORWARD -p udp --dport 104 -j NFQUEUE
iptables -I INPUT --protocol 99 -j NFQUEUE
iptables -I OUTPUT --protocol 99 -j NFQUEUE
iptables -I FORWARD --protocol 99 -j NFQUEUE

# Test connection (server)
nc -v -u -l 0.0.0.0 104

# Test connection (client)
# 10.99.7.10 is our mesh interface IP and 10.99.1.1 is destination via mesh
# You must use "primary" olsr interface IP, secondaries don't work due some
# reason, likely problem with my config
echo foo | nc -v -u -s 10.99.7.10 10.99.1.1 104

# remember there's 36 bytes overhead due sereadmo wrapping udp packets with
# extra headers.

###########################