从GoogleClusterData统计每个用户的使用率、平均每次出价

之前将google cluster data导入了Azure上的MySQL数据库，下一步就是对这些数据进行分析，

挖掘用户的使用规律了。

首先，为了加快执行速度，对user，time等加入索引。

然后就可以使用以下代码进行统计了。

import os
import MySQLdb
import time
import thread

def use4ADay(day, users):
    conn=MySQLdb.connect(host="localhost",user="root",passwd="",db="googleclusterdata",charset="utf8")
    cursor = conn.cursor()

    msAday = 24*60*60*1000000

    for user in users:
        user = user[0]
        print user
        use4ADay.user = user

        print 'day %s' %day
        startTime = (day - 1) * msAday
        endTime = day * msAday
        dayCPUUse = 0
        dayMEMUse = 0
        dayDiskUse = 0
        order = "select job_id from job_events where time >= %s and time < %s and user = '%s'" %(startTime, endTime, user)
        print order
        cursor.execute(order)
        job_ids = cursor.fetchall()
        for job_id in job_ids:
            job_id = job_id[0]
            print 'day %s' %day
            order = "select task_index, event_type, cpu_request, memory_request, disk_space_request, time from task_events \
    where time >= %s and time < %s and job_id = %d order by task_index"\
                    %(startTime, endTime, job_id)
            print order
            cursor.execute(order)
            tasks = cursor.fetchall()
            print 'tasks get'
            i = 0
            while i < len(tasks) - 1:
                task = tasks[i]
                if task[1] == 1:
                    task_index = task[0]
                    nextEvent = tasks[i+1]
                    if (nextEvent[1] == 4 or nextEvent[1] == 5) and nextEvent[0] == task_index:
                        taskLife = (nextEvent[5] - tasks[i][5]) / (10.0**6)
                        dayCPUUse += taskLife * task[2]
                        dayMEMUse += taskLife * task[3]
                        dayDiskUse += taskLife * task[4]
                        #print 'task: ', task_index, dayCPUUse, dayMEMUse, dayDiskUse
                i = i+1
            #print 'job: ', job_id, dayCPUUse, dayMEMUse, dayDiskUse
        fOut = open('C:\\userUsageEachDay\\day%d.txt' %day, 'a')
        fOut.write('%s\t%f\t%f\t%f\n' %(user,  dayCPUUse, dayMEMUse, dayDiskUse))
        fOut.close()
    print 'day %d finish' %day
    conn.close()

conn=MySQLdb.connect(host="localhost",user="root",passwd="",db="googleclusterdata",charset="utf8")
cursor = conn.cursor()
#get all user_name
order = "select distinct user from job_events"
print order
cursor.execute(order)
users = cursor.fetchall()
conn.close()

for day in range(1, 30):
    try:
        use4ADay(day, users)
    except:
        print 'day', day, 'failed!!'
        fOut = open('C:\\failed.txt', 'a')
        fOut.write('%s\t%d\t\n' %(use4ADay.user, day))
        fOut.close()
    #print 'starting thread for day %d' %day
    #thread.start_new_thread(use4ADay, (day, users, ) )#use4ADay(2, users)

下一步，是统计每个用户整个月的消费频率，以及每次消费的平均消费量

fDay1 = open('C:\\Usage\\day1.txt')
users = []
for l in fDay1.readlines():
    l = l.split('\t')
    user = l[0]
    users.append(user)
fDay1.close()

#fOut = open('C:\\UseTraceOfAllUsers.txt', 'w')
for user in users:
    useDays = 0
    allPrice = 0
    for day in range(1,30):
        f = open('C:\\Usage\\day%d.txt' %day)
        isFind = False
        for l in f.readlines():
            if l.count(user) > 0:
                l = l.strip()
                l = l.split('\t')
                cpu = float(l[1])
                mem = float(l[2])
                disk = float(l[3])
                money = 1.92*cpu + 15.6*mem + 1.2*disk
                assert(money>=0)
                isFind = True
                break
        if isFind and money != 0:
            useDays += 1
            allPrice += money
        f.close()
    if useDays != 0:
        pass
        #fOut.write('%s\t%s\n' %(str(useDays/29.0), str(allPrice/useDays)))
fOut.close()

最后就可以使用matlab进行画图啦。

x = load('C:\UseTraceOfAllUsers.txt')
plot(x(:,1), x(:,2), 'o');

结果如下：

对平均使用量取个对数的话

x = load('C:\UseTraceOfAllUsers.txt')
plot(x(:,1), log(x(:,2)), 'o');