Viz Gmane Viz
https://www.youtube.com/embed/LRqVPMEXByw
-
gmane.py
import sqlite3 import time import ssl import urllib.request, urllib.parse, urllib.error from urllib.parse import urljoin from urllib.parse import urlparse import re from datetime import datetime, timedelta # Not all systems have this so conditionally define parser try: import dateutil.parser as parser except: pass def parsemaildate(md) : # See if we have dateutil try: pdate = parser.parse(tdate) test_at = pdate.isoformat() return test_at except: pass # Non-dateutil version - we try our best pieces = md.split() notz = " ".join(pieces[:4]).strip() # Try a bunch of format variations - strptime() is *lame* dnotz = None for form in [ '%d %b %Y %H:%M:%S', '%d %b %Y %H:%M:%S', '%d %b %Y %H:%M', '%d %b %Y %H:%M', '%d %b %y %H:%M:%S', '%d %b %y %H:%M:%S', '%d %b %y %H:%M', '%d %b %y %H:%M' ] : try: dnotz = datetime.strptime(notz, form) break except: continue if dnotz is None : # print 'Bad Date:',md return None iso = dnotz.isoformat() tz = "+0000" try: tz = pieces[4] ival = int(tz) # Only want numeric timezone values if tz == '-0000' : tz = '+0000' tzh = tz[:3] tzm = tz[3:] tz = tzh+":"+tzm except: pass return iso+tz # Ignore SSL certificate errors ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = sqlite3.connect('content.sqlite') cur = conn.cursor() baseurl = "http://mbox.dr-chuck.net/sakai.devel/" cur.execute('''CREATE TABLE IF NOT EXISTS Messages (id INTEGER UNIQUE, email TEXT, sent_at TEXT, subject TEXT, headers TEXT, body TEXT)''') # Pick up where we left off start = None cur.execute('SELECT max(id) FROM Messages' ) try: row = cur.fetchone() if row is None : start = 0 else: start = row[0] except: start = 0 if start is None : start = 0 many = 0 count = 0 fail = 0 while True: if ( many < 1 ) : conn.commit() sval = input('How many messages:') if ( len(sval) < 1 ) : break many = int(sval) start = start + 1 cur.execute('SELECT id FROM Messages WHERE id=?', (start,) ) try: row = cur.fetchone() if row is not None : continue except: row = None many = many - 1 url = baseurl + str(start) + '/' + str(start + 1) text = "None" try: # Open with a timeout of 30 seconds document = urllib.request.urlopen(url, None, 30, context=ctx) text = document.read().decode() if document.getcode() != 200 : print("Error code=",document.getcode(), url) break except KeyboardInterrupt: print('') print('Program interrupted by user...') break except Exception as e: print("Unable to retrieve or parse page",url) print("Error",e) fail = fail + 1 if fail > 5 : break continue print(url,len(text)) count = count + 1 if not text.startswith("From "): print(text) print("Did not find From ") fail = fail + 1 if fail > 5 : break continue pos = text.find("\n\n") if pos > 0 : hdr = text[:pos] body = text[pos+2:] else: print(text) print("Could not find break between headers and body") fail = fail + 1 if fail > 5 : break continue email = None x = re.findall('\nFrom: .* <(\S+@\S+)>\n', hdr) if len(x) == 1 : email = x[0]; email = email.strip().lower() email = email.replace("<","") else: x = re.findall('\nFrom: (\S+@\S+)\n', hdr) if len(x) == 1 : email = x[0]; email = email.strip().lower() email = email.replace("<","") date = None y = re.findall('\Date: .*, (.*)\n', hdr) if len(y) == 1 : tdate = y[0] tdate = tdate[:26] try: sent_at = parsemaildate(tdate) except: print(text) print("Parse fail",tdate) fail = fail + 1 if fail > 5 : break continue subject = None z = re.findall('\Subject: (.*)\n', hdr) if len(z) == 1 : subject = z[0].strip().lower(); # Reset the fail counter fail = 0 print(" ",email,sent_at,subject) cur.execute('''INSERT OR IGNORE INTO Messages (id, email, sent_at, subject, headers, body) VALUES ( ?, ?, ?, ?, ?, ? )''', ( start, email, sent_at, subject, hdr, body)) if count % 50 == 0 : conn.commit() if count % 100 == 0 : time.sleep(1) conn.commit() cur.close() -
gbasic.py
import sqlite3 import time import zlib howmany = int(input("How many to dump? ")) conn = sqlite3.connect('index.sqlite') cur = conn.cursor() cur.execute('SELECT id, sender FROM Senders') senders = dict() for message_row in cur : senders[message_row[0]] = message_row[1] cur.execute('SELECT id, subject FROM Subjects') subjects = dict() for message_row in cur : subjects[message_row[0]] = message_row[1] # cur.execute('SELECT id, guid,sender_id,subject_id,headers,body FROM Messages') cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages') messages = dict() for message_row in cur : messages[message_row[0]] = (message_row[1],message_row[2],message_row[3],message_row[4]) print("Loaded messages=",len(messages),"subjects=",len(subjects),"senders=",len(senders)) sendcounts = dict() sendorgs = dict() for (message_id, message) in list(messages.items()): sender = message[1] sendcounts[sender] = sendcounts.get(sender,0) + 1 pieces = senders[sender].split("@") if len(pieces) != 2 : continue dns = pieces[1] sendorgs[dns] = sendorgs.get(dns,0) + 1 print('') print('Top',howmany,'Email list participants') x = sorted(sendcounts, key=sendcounts.get, reverse=True) for k in x[:howmany]: print(senders[k], sendcounts[k]) if sendcounts[k] < 10 : break print('') print('Top',howmany,'Email list organizations') x = sorted(sendorgs, key=sendorgs.get, reverse=True) for k in x[:howmany]: print(k, sendorgs[k]) if sendorgs[k] < 10 : break -
index.sqlite
after run gbasic
-
gword.py
import sqlite3 import time import zlib import string conn = sqlite3.connect('index.sqlite') cur = conn.cursor() cur.execute('SELECT id, subject FROM Subjects') subjects = dict() for message_row in cur : subjects[message_row[0]] = message_row[1] # cur.execute('SELECT id, guid,sender_id,subject_id,headers,body FROM Messages') cur.execute('SELECT subject_id FROM Messages') counts = dict() for message_row in cur : text = subjects[message_row[0]] text = text.translate(str.maketrans('','',string.punctuation)) text = text.translate(str.maketrans('','','1234567890')) text = text.strip() text = text.lower() words = text.split() for word in words: if len(word) < 4 : continue counts[word] = counts.get(word,0) + 1 x = sorted(counts, key=counts.get, reverse=True) highest = None lowest = None for k in x[:100]: if highest is None or highest < counts[k] : highest = counts[k] if lowest is None or lowest > counts[k] : lowest = counts[k] print('Range of counts:',highest,lowest) # Spread the font sizes across 20-100 based on the count bigsize = 80 smallsize = 20 fhand = open('gword.js','w') fhand.write("gword = [") first = True for k in x[:100]: if not first : fhand.write( ",\n") first = False size = counts[k] size = (size - lowest) / float(highest - lowest) size = int((size * bigsize) + smallsize) fhand.write("{text: '"+k+"', size: "+str(size)+"}") fhand.write( "\n];\n") fhand.close() print("Output written to gword.js") print("Open gword.htm in a browser to see the vizualization")
then it will create js file
-
word.js
gword = [{text: 'sakai', size: 100}, {text: 'with', size: 66}, {text: 'error', size: 53}, {text: 'password', size: 52}, {text: 'forgotten', size: 52}, {text: 'feature', size: 52}, {text: 'mysql', size: 43}, {text: 'collab', size: 32}, {text: 'site', size: 32}, {text: 'section', size: 32}, {text: 'memory', size: 30}, {text: 'worksite', size: 28}, {text: 'taxonomy', size: 28}, {text: 'problem', size: 28}, {text: 'apis', size: 28}, {text: 'creating', size: 27}, {text: 'austin', size: 25}, {text: 'sakaiportallogin', size: 25}, {text: 'presense', size: 25}, {text: 'provider', size: 25}, {text: 'regarding', size: 25}, {text: 'manager', size: 25}, {text: 'other', size: 25}, {text: 'related', size: 25}, {text: 'tool', size: 25}, {text: 'webdav', size: 25}, {text: 'lmsvle', size: 23}, {text: 'rantscomments', size: 23}, {text: 'examples', size: 23}, {text: 'problems', size: 23}, {text: 'accessing', size: 23}, {text: 'internet', size: 23}, {text: 'audio', size: 23}, {text: 'recordings', size: 23}, {text: 'presentations', size: 23}, {text: 'configuration', size: 23}, {text: 'document', size: 23}, {text: 'resources', size: 23}, {text: 'developers', size: 21}, {text: 'explorer', size: 21}, {text: 'file', size: 21}, {text: 'picker', size: 21}, {text: 'nonlegacy', size: 21}, {text: 'tools', size: 21}, {text: 'conversion', size: 21}, {text: 'update', size: 21}, {text: 'converting', size: 21}, {text: 'tables', size: 21}, {text: 'myfaces', size: 21}, {text: 'anyone', size: 21}, {text: 'schedule', size: 21}, {text: 'high', size: 21}, {text: 'level', size: 21}, {text: 'future', size: 21}, {text: 'stovepipe', size: 21}, {text: 'breakage', size: 21}, {text: 'planning', size: 21}, {text: 'sectionmanager', size: 21}, {text: 'nosuchbeandefinitionexception', size: 21}, {text: 'call', size: 20}, {text: 'participation', size: 20}, {text: 'documentation', size: 20}, {text: 'report', size: 20}, {text: 'from', size: 20}, {text: 'conference', size: 20}, {text: 'break', size: 20}, {text: 'into', size: 20}, {text: 'song', size: 20}, {text: 'ldap', size: 20}, {text: 'authentication', size: 20}, {text: 'lone', size: 20}, {text: 'zero', size: 20}, {text: 'block', size: 20}, {text: 'logo', size: 20}, {text: 'sepp', size: 20}, {text: 'library', size: 20}, {text: 'discussion', size: 20}, {text: 'group', size: 20}, {text: 'renamed', size: 20}, {text: 'open', size: 20}, {text: 'membership', size: 20}, {text: 'code', size: 20}, {text: 'samigo', size: 20}, {text: 'news', size: 20}, {text: 'land', size: 20}, {text: 'direct', size: 20}, {text: 'urls', size: 20}, {text: 'entity', size: 20}, {text: 'model', size: 20}, {text: 'presence', size: 20}, {text: 'virtual', size: 20}, {text: 'paths', size: 20}, {text: 'page', size: 20}, {text: 'cannot', size: 20}, {text: 'displayed', size: 20}, {text: 'http', size: 20} ]; -
word.html
<!DOCTYPE html> <meta charset="utf-8"> <script src="d3.v2.js"></script> <script src="d3.layout.cloud.js"></script> <script src="gword.js"></script> <body> <script> var fill = d3.scale.category20(); d3.layout.cloud().size([700, 700]) .words(gword) .rotate(function() { return ~~(Math.random() * 2) * 90; }) .font("Impact") .fontSize(function(d) { return d.size; }) .on("end", draw) .start(); function draw(words) { d3.select("body").append("svg") .attr("width", 700) .attr("height", 700) .append("g") .attr("transform", "translate(350,350)") .selectAll("text") .data(words) .enter().append("text") .style("font-size", function(d) { return d.size + "px"; }) .style("font-family", "Impact") .style("fill", function(d, i) { return fill(i); }) .attr("text-anchor", "middle") .attr("transform", function(d) { return "translate(" + [d.x, d.y] + ")rotate(" + d.rotate + ")"; }) .text(function(d) { return d.text; }); } </script>
-
gline.py
import sqlite3 import time import zlib conn = sqlite3.connect('index.sqlite') cur = conn.cursor() cur.execute('SELECT id, sender FROM Senders') senders = dict() for message_row in cur : senders[message_row[0]] = message_row[1] cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages') messages = dict() for message_row in cur : messages[message_row[0]] = (message_row[1],message_row[2],message_row[3],message_row[4]) print("Loaded messages=",len(messages),"senders=",len(senders)) sendorgs = dict() for (message_id, message) in list(messages.items()): sender = message[1] pieces = senders[sender].split("@") if len(pieces) != 2 : continue dns = pieces[1] sendorgs[dns] = sendorgs.get(dns,0) + 1 # pick the top schools orgs = sorted(sendorgs, key=sendorgs.get, reverse=True) orgs = orgs[:10] print("Top 10 Organizations") print(orgs) counts = dict() months = list() # cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages') for (message_id, message) in list(messages.items()): sender = message[1] pieces = senders[sender].split("@") if len(pieces) != 2 : continue dns = pieces[1] if dns not in orgs : continue month = message[3][:7] if month not in months : months.append(month) key = (month, dns) counts[key] = counts.get(key,0) + 1 months.sort() # print counts # print months fhand = open('gline.js','w') fhand.write("gline = [ ['Month'") for org in orgs: fhand.write(",'"+org+"'") fhand.write("]") for month in months: fhand.write(",\n['"+month+"'") for org in orgs: key = (month, org) val = counts.get(key,0) fhand.write(","+str(val)) fhand.write("]"); fhand.write("\n];\n") fhand.close() print("Output written to gline.js") print("Open gline.htm to visualize the data") -
gline.js
gline = [ ['Month','umich.edu','unl.edu','mac.com','virginia.edu','cam.ac.uk','columbia.edu','gmail.com','unicon.net','gatech.edu','weber.edu'], ['2005-12',19,7,6,6,5,5,5,5,5,5] ]; -
gline.html
<html> <head> <script type="text/javascript" src="gline.js"></script> <script type="text/javascript" src="https://www.google.com/jsapi"></script> <script type="text/javascript"> google.load("visualization", "1", {packages:["corechart"]}); google.setOnLoadCallback(drawChart); function drawChart() { var data = google.visualization.arrayToDataTable( gline ); var options = { title: 'Sakai Developer Email Participation by Organization', chartArea: {left:'10%',top:'10%', width: '65%', height: '65%'} }; var chart = new google.visualization.LineChart(document.getElementById('chart_div')); chart.draw(data, options); } </script> </head> <body> <div id="chart_div" style="width: 1300px; height: 600px;"></div> </body> </html>