I was recently debating with my friend Danny how people navigate sites, especially blogs. As the author of 3 plugins designed to enhance site navigation (collapsing archives, categories, and pages), I of course had my opinions, but what is really compelling is data. So I finally decided to modify my links on my site a bit in order to figure out more exactly how people are navigating my site. There are several different ways in which people can navigate to a particular page on my site.

• external
  • search
  • social network (digg, reddit, friendfeed, etc.)
• internal
  • collapsing archives widget
  • collpsing categorries widget
  • collapsing pages widget
  • category page
  • archive page
  • search
  • index page link
  • prev page link
  • next page link

I added a ?nav query to each of these particular options, so that I could then find those values in the server logs. I wrote a little bash script which mostly does some grepping of the log. Here is the script, and the results for the last 12 hours or so. My plan is to collect about 30 days worth of data, and see what the data shows. I think the results will be informative.

#!/bin/bash
# calculate how people navigate my site

# first we filter out all spiders, any internal blog stuff, and requests from
# my own ip, and then pipe that to filter out page refreshes
grep -vEi '(Mediapartners-Google|bot|spider|crawler|slurp|scoutjet|httpunit|htmlparser|favicon|wp-cron.php|wp-content|wp-photos|wp-comments|feed|fail\.php|wp-login.php|POST|\/images|wp-includes|forum|71\.237\.102\.16)' experiment_log |grep -Ev '"GET (.*) .*\1' > nobots

# next we separate into internal and external referers
grep 'https://blog.robfelty.com' nobots > INTERNAL
grep -v 'https://blog.robfelty.com' nobots > EXTERNAL

EXTSEARCH=`grep -ciE '(yahoo|bing|google)' EXTERNAL`
SOCIAL=`grep -ciE '(fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL`
NOREFERER=`grep -c '"-"' EXTERNAL`
EXTREMAINING=`grep -viE '(yahoo|bing|google|fark|reddit|digg|slashdot|friendfeed|facebook|twitter)' EXTERNAL|grep -cv '"-"'`
COLLARCH=`grep -c '?nav=collapsing-archives' INTERNAL`
COLLCAT=`grep -c '?nav=collapsing-category' INTERNAL`
YEARLY=`grep -c '?nav=yearly' INTERNAL`
MONTHLY=`grep -c '?nav=monthly' INTERNAL`
CAT=`grep -c '?nav=category' INTERNAL`
COLLPAGE=`grep -c '?nav=collapsing-pages' INTERNAL`
SEARCH=`grep -c '?nav=search' INTERNAL`
INDEX=`grep -c '?nav=index' INTERNAL`
PREV=`grep -c '?nav=previous' INTERNAL`
NEXT=`grep -c '?nav=next' INTERNAL`
TOTAL=`wc -l nobots|cut -f1 -d ' '`
INTERNAL=`wc -l INTERNAL|cut -f1 -d ' '`
EXTERNAL=`wc -l EXTERNAL|cut -f1 -d ' '`
# Only firefox prefetches
PREFETCH=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -c 'Firefox'`
INTREMAINING=`grep -vE '\?nav=(collapsing-category|collapsing-archives|search|index|previous|next)' INTERNAL|grep -cv 'Firefox'`
echo "TOTAL PAGES = $TOTAL"
echo " EXTERNAL= $EXTERNAL"
echo " EXTSEARCH = $EXTSEARCH"
echo " SOCIAL = $SOCIAL"
echo " NOREFERER = $NOREFERER"
echo " REMAINING = $EXTREMAINING"
echo " INTERNAL= $INTERNAL"
echo " COLLARCH = $COLLARCH"
echo " YEARLY = $YEARLY"
echo " MONTHLY = $MONTHLY"
echo " COLLCAT = $COLLCAT"
echo " CAT = $CAT"
echo " COLLPAGE = $COLLPAGE"
echo " SEARCH = $SEARCH"
echo " INDEX = $INDEX"
echo " PREV = $PREV"
echo " NEXT = $NEXT"
echo " PREFETCH = $PREFETCH"
echo " REMAINING = $INTREMAINING"


TOTAL PAGES = 197
  EXTERNAL= 131
    EXTSEARCH = 51
    SOCIAL = 0
    NOREFERER = 43
    REMAINING = 37
  INTERNAL= 66
    COLLARCH = 3
    YEARLY = 0
    MONTHLY = 0
    COLLCAT = 4
    CAT = 0
    COLLPAGE = 0
    SEARCH = 1
    INDEX = 1
    PREV = 0
    NEXT = 0
    PREFETCH = 52
    REMAINING = 5