Friday, July 24th, 2009
On blog navigation
A few navigation choices after reading an article on a blog
- external
- search
- social network (digg, reddit, friendfeed, etc.)
- internal
- collapsing archives widget
- collpsing categorries widget
- collapsing pages widget
- category page
- archive page
- search
- index page link
- prev page link
- next page link
I added a ?nav query to each of these particular options, so that I could then find those values in the server logs. I wrote a little bash script which mostly does some grepping of the log. Here is the script, and the results for the last 12 hours or so. My plan is to collect about 30 days worth of data, and see what the data shows. I think the results will be informative.
#!/bin/bash
# calculate how people navigate my site
# first we filter out all spiders, any internal blog stuff, and requests from
# my own ip, and then pipe that to filter out page refreshes
grep -vEi '(Mediapartners-Google|bot|spider|crawler|slurp|scoutjet|httpunit|htmlparser|favicon|wp-cron.php|wp-content|wp-photos|wp-comments|feed|fail\.php|wp-login.php|POST|\/images|wp-includes|forum|71\.237\.102\.16)' experiment_log |grep -Ev '"GET (.*) .*\1' > nobots
# next we separate into internal and external referers
grep 'http://blog.robfelty.com' nobots > INTERNAL
grep -v 'http://blog.robfelty.com' nobots > EXTERNAL
EXTSEARCH=`grep -ciE '(yahoo|bing|google)' EXTERNAL`
SOCIAL=`grep -ciE '(fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL`
NOREFERER=`grep -c '"-"' EXTERNAL`
EXTREMAINING=`grep -viE '(yahoo|bing|google|fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL|grep -cv '"-"'`
COLLARCH=`grep -c '?nav=collapsing-archives' INTERNAL`
COLLCAT=`grep -c '?nav=collapsing-category' INTERNAL`
YEARLY=`grep -c '?nav=yearly' INTERNAL`
MONTHLY=`grep -c '?nav=monthly' INTERNAL`
CAT=`grep -c '?nav=category' INTERNAL`
COLLPAGE=`grep -c '?nav=collapsing-pages' INTERNAL`
SEARCH=`grep -c '?nav=search' INTERNAL`
INDEX=`grep -c '?nav=index' INTERNAL`
PREV=`grep -c '?nav=previous' INTERNAL`
NEXT=`grep -c '?nav=next' INTERNAL`
TOTAL=`wc -l nobots|cut -f1 -d ' '`
INTERNAL=`wc -l INTERNAL|cut -f1 -d ' '`
EXTERNAL=`wc -l EXTERNAL|cut -f1 -d ' '`
# Only firefox prefetches
PREFETCH=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -c 'Firefox'`
INTREMAINING=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -cv 'Firefox'`
echo "TOTAL PAGES = $TOTAL"
echo " EXTERNAL= $EXTERNAL"
echo " EXTSEARCH = $EXTSEARCH"
echo " SOCIAL = $SOCIAL"
echo " NOREFERER = $NOREFERER"
echo " REMAINING = $EXTREMAINING"
echo " INTERNAL= $INTERNAL"
echo " COLLARCH = $COLLARCH"
echo " YEARLY = $YEARLY"
echo " MONTHLY = $MONTHLY"
echo " COLLCAT = $COLLCAT"
echo " CAT = $CAT"
echo " COLLPAGE = $COLLPAGE"
echo " SEARCH = $SEARCH"
echo " INDEX = $INDEX"
echo " PREV = $PREV"
echo " NEXT = $NEXT"
echo " PREFETCH = $PREFETCH"
echo " REMAINING = $INTREMAINING"
# calculate how people navigate my site
# first we filter out all spiders, any internal blog stuff, and requests from
# my own ip, and then pipe that to filter out page refreshes
grep -vEi '(Mediapartners-Google|bot|spider|crawler|slurp|scoutjet|httpunit|htmlparser|favicon|wp-cron.php|wp-content|wp-photos|wp-comments|feed|fail\.php|wp-login.php|POST|\/images|wp-includes|forum|71\.237\.102\.16)' experiment_log |grep -Ev '"GET (.*) .*\1' > nobots
# next we separate into internal and external referers
grep 'http://blog.robfelty.com' nobots > INTERNAL
grep -v 'http://blog.robfelty.com' nobots > EXTERNAL
EXTSEARCH=`grep -ciE '(yahoo|bing|google)' EXTERNAL`
SOCIAL=`grep -ciE '(fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL`
NOREFERER=`grep -c '"-"' EXTERNAL`
EXTREMAINING=`grep -viE '(yahoo|bing|google|fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL|grep -cv '"-"'`
COLLARCH=`grep -c '?nav=collapsing-archives' INTERNAL`
COLLCAT=`grep -c '?nav=collapsing-category' INTERNAL`
YEARLY=`grep -c '?nav=yearly' INTERNAL`
MONTHLY=`grep -c '?nav=monthly' INTERNAL`
CAT=`grep -c '?nav=category' INTERNAL`
COLLPAGE=`grep -c '?nav=collapsing-pages' INTERNAL`
SEARCH=`grep -c '?nav=search' INTERNAL`
INDEX=`grep -c '?nav=index' INTERNAL`
PREV=`grep -c '?nav=previous' INTERNAL`
NEXT=`grep -c '?nav=next' INTERNAL`
TOTAL=`wc -l nobots|cut -f1 -d ' '`
INTERNAL=`wc -l INTERNAL|cut -f1 -d ' '`
EXTERNAL=`wc -l EXTERNAL|cut -f1 -d ' '`
# Only firefox prefetches
PREFETCH=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -c 'Firefox'`
INTREMAINING=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -cv 'Firefox'`
echo "TOTAL PAGES = $TOTAL"
echo " EXTERNAL= $EXTERNAL"
echo " EXTSEARCH = $EXTSEARCH"
echo " SOCIAL = $SOCIAL"
echo " NOREFERER = $NOREFERER"
echo " REMAINING = $EXTREMAINING"
echo " INTERNAL= $INTERNAL"
echo " COLLARCH = $COLLARCH"
echo " YEARLY = $YEARLY"
echo " MONTHLY = $MONTHLY"
echo " COLLCAT = $COLLCAT"
echo " CAT = $CAT"
echo " COLLPAGE = $COLLPAGE"
echo " SEARCH = $SEARCH"
echo " INDEX = $INDEX"
echo " PREV = $PREV"
echo " NEXT = $NEXT"
echo " PREFETCH = $PREFETCH"
echo " REMAINING = $INTREMAINING"
TOTAL PAGES = 197
EXTERNAL= 131
EXTSEARCH = 51
SOCIAL = 0
NOREFERER = 43
REMAINING = 37
INTERNAL= 66
COLLARCH = 3
YEARLY = 0
MONTHLY = 0
COLLCAT = 4
CAT = 0
COLLPAGE = 0
SEARCH = 1
INDEX = 1
PREV = 0
NEXT = 0
PREFETCH = 52
REMAINING = 5
