Visualizing Email Data(Week 6&7)

code segment

gword.py

import sqlite3
import time
import zlib
import string conn = sqlite3.connect('index.sqlite')
cur = conn.cursor() cur.execute('SELECT id, subject FROM Subjects')
subjects = dict()
for message_row in cur :
subjects[message_row[0]] = message_row[1] # cur.execute('SELECT id, guid,sender_id,subject_id,headers,body FROM Messages')
cur.execute('SELECT subject_id FROM Messages')
counts = dict()
for message_row in cur :
text = subjects[message_row[0]]
text = text.translate(str.maketrans('','',string.punctuation))
text = text.translate(str.maketrans('','','1234567890'))
text = text.strip()
text = text.lower()
words = text.split()
for word in words:
if len(word) < 4 : continue
counts[word] = counts.get(word,0) + 1 x = sorted(counts, key=counts.get, reverse=True)
highest = None
lowest = None
for k in x[:100]:
if highest is None or highest < counts[k] :
highest = counts[k]
if lowest is None or lowest > counts[k] :
lowest = counts[k]
print('Range of counts:',highest,lowest) # Spread the font sizes across 20-100 based on the count
bigsize = 80
smallsize = 20 fhand = open('gword.js','w')
fhand.write("gword = [")
first = True
for k in x[:100]:
if not first : fhand.write( ",\n")
first = False
size = counts[k]
size = (size - lowest) / float(highest - lowest)
size = int((size * bigsize) + smallsize)
fhand.write("{text: '"+k+"', size: "+str(size)+"}")
fhand.write( "\n];\n")
fhand.close() print("Output written to gword.js")
print("Open gword.htm in a browser to see the vizualization")

gline.py

import sqlite3
import time
import zlib conn = sqlite3.connect('index.sqlite')
cur = conn.cursor() cur.execute('SELECT id, sender FROM Senders')
senders = dict()
for message_row in cur :
senders[message_row[0]] = message_row[1] cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages')
messages = dict()
for message_row in cur :
messages[message_row[0]] = (message_row[1],message_row[2],message_row[3],message_row[4]) print("Loaded messages=",len(messages),"senders=",len(senders)) sendorgs = dict()
for (message_id, message) in list(messages.items()):
sender = message[1]
pieces = senders[sender].split("@")
if len(pieces) != 2 : continue
dns = pieces[1]
sendorgs[dns] = sendorgs.get(dns,0) + 1 # pick the top schools
orgs = sorted(sendorgs, key=sendorgs.get, reverse=True)
orgs = orgs[:10]
print("Top 10 Organizations")
print(orgs) counts = dict()
months = list()
# cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages')
for (message_id, message) in list(messages.items()):
sender = message[1]
pieces = senders[sender].split("@")
if len(pieces) != 2 : continue
dns = pieces[1]
if dns not in orgs : continue
month = message[3][:7]
if month not in months : months.append(month)
key = (month, dns)
counts[key] = counts.get(key,0) + 1 months.sort()
# print counts
# print months fhand = open('gline.js','w')
fhand.write("gline = [ ['Month'")
for org in orgs:
fhand.write(",'"+org+"'")
fhand.write("]") for month in months:
fhand.write(",\n['"+month+"'")
for org in orgs:
key = (month, org)
val = counts.get(key,0)
fhand.write(","+str(val))
fhand.write("]"); fhand.write("\n];\n")
fhand.close() print("Output written to gline.js")
print("Open gline.htm to visualize the data")

Coursera课程笔记----P4E.Capstone----Week 6&7的更多相关文章

  1. Coursera课程笔记----P4E.Capstone----Week 4&5

    Spidering and Modeling Email Data(week4&5) Mailing List - Gmane Crawl the archive of a mailing l ...

  2. Coursera课程笔记----P4E.Capstone----Week 2&3

    Building a Search Engine(week 2&3) Search Engine Architecture Web Crawling Index Building Search ...

  3. 操作系统学习笔记----进程/线程模型----Coursera课程笔记

    操作系统学习笔记----进程/线程模型----Coursera课程笔记 进程/线程模型 0. 概述 0.1 进程模型 多道程序设计 进程的概念.进程控制块 进程状态及转换.进程队列 进程控制----进 ...

  4. Coursera课程笔记----C++程序设计----Week3

    类和对象(Week 3) 内联成员函数和重载成员函数 内联成员函数 inline + 成员函数 整个函数题出现在类定义内部 class B{ inline void func1(); //方式1 vo ...

  5. Coursera课程笔记----Write Professional Emails in English----Week 3

    Introduction and Announcement Emails (Week 3) Overview of Introduction & Announcement Emails Bas ...

  6. Coursera课程笔记----Write Professional Emails in English----Week 1

    Get to Know Basic Email Writing Structures(Week 1) Introduction to Course Email and Editing Basics S ...

  7. Coursera课程笔记----C程序设计进阶----Week 5

    指针(二) (Week 5) 字符串与指针 指向数组的指针 int a[10]; int *p; p = a; 指向字符串的指针 指向字符串的指针变量 char a[10]; char *p; p = ...

  8. Coursera课程笔记----Write Professional Emails in English----Week 5

    Culture Matters(Week 5) High/Low Context Communication High Context Communication The Middle East, A ...

  9. Coursera课程笔记----Write Professional Emails in English----Week 4

    Request and Apology Emails(Week 4) How to Write Request Emails Write more POLITELY & SINCERELUY ...

随机推荐

  1. [算法总结]康托展开Cantor Expansion

    目录 一.关于康托展开 1.什么是康托展开 2.康托展开实现原理 二.具体实施 1.模板 一.关于康托展开 1.什么是康托展开 求出给定一个由1n个整数组成的任意排列在1n的全排列中的位置. 解决这样 ...

  2. 使用python pip安装工具组件包:出现 requests File “<stdin>",line 1 pip install xxx ^ SyntaxError:invalid syntax

    最近想要试试python ,软件安装完成了,但是,import 组件包时,出了问题,一直不得解:安装pycharm 工具感觉麻烦,不想安装那些,只想单纯使用python . 问题复现: 1.Windo ...

  3. sql 自增序列

    一.使用set identity_insert [database][owner][table]on设置时,要在插入语句中显示列出插入的列;

  4. Ubuntu安装Elasticsearch6.3

    本文使用的 Ubuntu 版本信息: Distributor ID: Ubuntu Description: Ubuntu LTS Release: 16.04 Codename: xenial 1. ...

  5. python白帽子/黑客/实战编程教程

    Python搜索爬虫抓取超高清视频教程_第一期Python搜索爬虫抓取超高清视频教程_第二期Python搜索爬虫抓取视频教程_第三期Python搜索爬虫抓取视频教程_第四期Python搜索引擎爬虫抓取 ...

  6. Gatling 条件判断

    在使用Gatling的过程中,当前置接口异常,无法获取到数据作为其他接口的请求参数室,接口是不能请求的.或者通过feeder获取的数据要区分不同的情况请求不同的接口.此时,使用gatling的判断语句 ...

  7. 【错误】python百分号冲突not enough arguments for format string

    query = "SELECT * FROM devices WHERE devices.`id` LIKE '%{}%'".format("f2333") d ...

  8. file_put_contens小trick

    file_put_contents tricks 0x01 trick1 来自于P神的实例: <?php $text = $_GET['text']; if(preg_match('[<& ...

  9. windows 系统使用技巧

    1 自定义发送到C:\Users\adm\AppData\Roaming\Microsoft\Windows\SendTo 2 自定义关机shutdown -s -t time,可写在快捷方式中shu ...

  10. python-Django与Nginx整合gunicorn模块

    1.pip install gunicorn 2.修改Nginx配置文件 vim /etc/nginx/conf.d/virtual.conf server { listen ; #listen so ...