Handbrake auto-crop and channel logos

Same as old post except with updated patch for fresh Handbrake SVN version.

Handbrake does good job auto-cropping black bars from source and calculating resulting aspect ratio. However if you're encoding digital TV recordings it's common to have channel logo on otherwise black portion of image.



Since there's no built-in method to tune auto-crop I ended up hacking source. I'm no coder, but managed to come up with something. Works for me, but obviously not the "right way" to do it. What I'm at least attempting to do below is to ignore left- and rightmost one third of top 100 rows when performing autocrop detection. Also column cropping ignores top 150 rows.
Hack is against svn 5401. This part of code was changed on 4803 so it will not apply to any earlier version than that.

# Get rebuild deps
apt-get update
apt-get -y install devscripts
apt-get -y build-dep handbrake

# Create build environment
adduser bobbuilder --shell /bin/bash \
        --disabled-password --gecos bobbuilder

# Switch to non-priviledged user for build
sudo su - bobbuilder

# Download sources
mkdir -p ~/build/handbrake
cd ~/build/handbrake
apt-get -y source handbrake
cd handbrake-* 
fakeroot debian/rules clean 

# Apply patches
patch -l -p0 <<__EOF__
--- libhb/scan.c.old    2013-04-15 08:32:28.000000000 +0300
+++ libhb/scan.c        2013-04-20 17:41:00.048844900 +0300
@@ -307,9 +307,25 @@
     int stride = buf->plane[0].stride;
     uint8_t *luma = buf->plane[0].data + stride * row;

+    // HACK-BEGIN
+    // crop top 100 rows aggressively ignoring left 1/3rd and right 1/3rd.
+    // this is to crop away channel logos on otherwise black background
+    int leftstart;
+    if ( row <= 100 )
+    {
+    width = buf->plane[0].width-(buf->plane[0].width/3);
+    leftstart = buf->plane[0].width/3;
+    }
+    else
+    { // normal autocrop for rest of frame
+    leftstart = 0;
+    }
+    // HACK-END
+ 
     // compute the average luma value of the row
     int i, avg = 0;
-    for ( i = 0; i < width; ++i )
+    //for ( i = 0; i < width; ++i )
+    for ( i = leftstart; i < width; ++i ) //HACK
     {
         avg += clampBlack( luma[i] );
     }
@@ -321,9 +337,11 @@
     // all pixels are within +-16 of the average (this range is fairly coarse
     // but there's a lot of quantization noise for luma values near black
     // so anything less will fail to crop because of the noise).
-    for ( i = 0; i < width; ++i )
+    //for ( i = 0; i < width; ++i )
+    for ( i = leftstart; i < width; ++i ) //HACK
     {
-        if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 )
+        //if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 )
+        if ( absdiff( avg, clampBlack( luma[i] ) ) > 16 ) //HACK
             return 0;
     }
     return 1;
@@ -332,12 +350,15 @@
 static int column_all_dark( hb_buffer_t* buf, int top, int bottom, int col )
 {
     int stride = buf->plane[0].stride;
-    int height = buf->plane[0].height - top - bottom;
-    uint8_t *luma = buf->plane[0].data + stride * top + col;
+    //int height = buf->plane[0].height - top - bottom;
+    //uint8_t *luma = buf->plane[0].data + stride * top + col;
+    int height = buf->plane[0].height - (top+150) - bottom; //HACK
+    uint8_t *luma = buf->plane[0].data + stride * (top+150) + col; //HACK

     // compute the average value of the column
     int i = height, avg = 0, row = 0;
-    for ( ; --i >= 0; row += stride )
+    //for ( ; --i >= 0; row += stride )
+    for ( ; --i >= 150; row += stride ) //HACK
     {
         avg += clampBlack( luma[row] );
     }
@@ -348,7 +369,8 @@
     // since we're trying to detect smooth borders, only take the column if
     // all pixels are within +-16 of the average.
     i = height, row = 0;
-    for ( ; --i >= 0; row += stride )
+    //for ( ; --i >= 0; row += stride )
+    for ( ; --i >= 150; row += stride ) //HACK
     {
         if ( absdiff( avg, clampBlack( luma[row] ) ) > 16 )
             return 0;
__EOF__

# Patch to allow custom optimization flags. 
sed -i.bak -e's/-O3/-O3 -march=native/g' make/include/gcc.defs 
sed -i.bak -e's/unset CFLAGS; //g' -e's/unset CFLAGS ; //g' -e's/unset CXXLAGS; //g' debian/rules 

# Compile new binary 
CFLAGS="-O3 -march=native" CXXFLAGS="-O3 -march=native" fakeroot debian/rules binary    

# Switch to root now

# Block updates from repository 
# to undo replace hold with install 
# to see current status use "dpkg --get-selections" 
echo "handbrake-cli hold"|dpkg --set-selections 
echo "handbrake-gtk hold"|dpkg --set-selections 

# Install freshly compiled version 
cd ~bobbuilder/build/handbrake
dpkg -i handbrake-*.deb

# Done