Archive for February, 2010

Monitoring your StackOverflow status with Python

No Comments »

I recently read this post on meta.stackoverflow.com on monitoring your SO status using Python. Since I’ve been looking into Pyhon (IronPython in particular) lately, I figured it might be fun to try the provided script out in IronPython.

I quickly experienced some problems pertaining to the sqlite3 and urllib2 modules which I was unable to solve, so I downloaded and installed Python 3.1 for Windows.

The original script does not run against Python 3.1 because of some string encoding issues. Moreover, the regular expressions used in the original script no longer match the markup of SO, so I have updated the script somewhat:

from sqlite3 import dbapi2 as sqlite
import re, os, sys, time
import urllib.request as urllib2

questLen = 60 #digits before elipses kick in
connection = sqlite.connect(“C:\\Users\\Rune Ibsen\\Projects\\SO\\profile.db”)
cursor = connection.cursor()

user = #your user id

request = urllib2.Request(url = ‘<http://stackoverflow.com/users/%i/myProfile.html>’ % (user))

profile = urllib2.urlopen(request).read()
profile = profile.decode(“utf-8″)
rep = re.compile(‘summarycount”>.*?([,\d]+)</div>.*?Reputation’, re.S).search(profile).group(1)
rep = rep.replace(‘,’,”)
badge = re.compile(‘<‘+‘div class=”summarycount ar”.{0,50}>(\d+).{1,100}Badges’, re.S).search(profile).group(1)

stQuestion = re.compile(‘Questions</h.*?Answers</h’, re.S).search(profile).group()
mQuestion = re.compile(‘question-summary narrow.*?id=”question-summary-(\d+)”.*?class=”votes”.*?(\d+).*?class=”status.+?(\d+).*?<h3><a.+?>(.+?)</a>’, re.S).findall(stQuestion)
# mQuestion contains tuples containing (id, votes, answers, title)

stAnswer  = re.compile(‘<h1>Answers</h1>.*?<script’, re.S).search(profile).group()
mAnswer   = re.compile(‘answer-summary”><a href=”/questions/(\d*).*?votes.*?>(-?\d+).*?href.*?>(.*?)<.a’, re.S).findall(stAnswer)

stTime = time.strftime(“%Y-%m-%d %H:%M:%S”)

print (stTime)
print (‘\nQuestions (‘ + str(len(mQuestion)) + ‘):’ )
for quest in mQuestion:
cursor.execute(‘SELECT count(id), votes FROM Questions WHERE id = ‘ + quest[0+ ‘ AND type=0;’)
item = cursor.fetchone()
if item[0> 0:
lastQ = (int(quest[1]) - item[1])
if lastQ==0:lastQ=“”
cursor.execute(‘UPDATE Questions SET votes = %s WHERE id = %s AND type = 0′ % (quest[1], quest[0]))
else:
cursor.execute(‘INSERT INTO Questions VALUES(“‘+quest[3]+‘”,’+quest[1]+‘,0,’+quest[0]+‘);’)
lastQ = “(NEW)”
if len(quest[2]) > questLen:
elips=“…” #in case the question is really long
nElips = 0
else:
elips=“”
nElips = 3
print (‘%s%s %s%s’ % (quest[3][:questLen].ljust(questLen+nElips,” “),elips, (“(“+str(quest[1])+“)”).ljust(5,” “), lastQ))
print (“\nAnswers (” + str(len(mAnswer)) + ‘):’)
for answer in mAnswer:
aId = answer[0]
aVotes = answer[1]
aQuestion = answer[2]
cursor.execute(‘SELECT count(id), votes FROM Questions WHERE id = ‘ + aId + ‘ AND type=1;’)
item = cursor.fetchone()
if item[0> 0:
lastQ = int(aVotes) - item[1]
if lastQ==0:lastQ=“”
cursor.execute(‘UPDATE Questions SET votes = %s WHERE id = %s AND type = 1′ % (aVotes, aId))
else:
cursor.execute(‘INSERT INTO Questions VALUES(“‘+aQuestion+‘”,’+aVotes+‘,1,’+aId+‘);’)
lastQ = “(NEW)”
if len(aQuestion) > questLen:
elips=“…”
nElips = 0
else:
elips=“”
nElips = 3
print (‘%s%s %s%s’ % (aQuestion[:questLen].ljust(questLen+nElips,” “),elips, (“(“+str(aVotes)+“)”).ljust(5,” “), lastQ))

cursor.execute(‘SELECT rep, badges, questions, answers , COUNT(date) FROM profile WHERE user = ‘ + str(user) + ‘ ORDER BY date DESC;’)
oldData = cursor.fetchone()
if oldData[4== 0:
oldData = [0,0,0,0]
cursor.execute(“INSERT INTO profile VALUES(%s,%s,%s,%s,’%s’,%i);” % (rep,badge,len(mQuestion),len(mAnswer),stTime, user) )
print (‘\n‘)
print (‘%s Questions, %s new’ % (len(mQuestion),(len(mQuestion) - oldData[2])))
print (‘%s Answers, %s new’ % (len(mAnswer),(len(mAnswer) - oldData[3])))
print (‘%s Reputation (%+i)’ % (rep, (int(rep) - oldData[0])))
print (‘%s Badges, %s new’ % (badge, (int(badge) - oldData[1])))
connection.commit()

Note that

  • You will have to create an empty SQLite database or download one here
  • You will have to insert your own user id (which is easily identified from the URL if you go to your user page).

Running the script will result in something like this:

image