线性回归的Spark实现 [Linear Regression / Machine Learning / Spark]

1- 问题提出

---

2- 线性回归

---

3- 理论推导

---

4- Python/Spark实现

 # -*- coding: utf-8 -*-
 from pyspark import SparkContext

 theta = [0, 0]
 alpha = 0.001

 sc = SparkContext('local')

 def func_theta_x(x):
     return sum([i * j for i, j in zip(theta, x)])

 def cost(x):
     thx = func_theta_x(x)
     return thx - x[-1]

 def partial_theta(x):
     dif = cost(x)
     return [dif * i for i in x[:-1]]

 rdd = sc.textFile('/home/freyr/linearRegression.txt')\
         .map(lambda line: map(float, line.strip().split('\t')))

 maxiter = 400
 iter = 0
 while True:
     parTheta = rdd.map(partial_theta)\
                   .reduce(lambda x, y: [i + j for i, j in zip(x, y)])

     for i in range(2):
         theta[i] = theta[i] - alpha * parTheta[i]

     iter += 1

     if iter <= maxiter:
         if sum(map(abs, parTheta)) <= 0.01:
             print 'I get it!!!'
             print 'Iter = %s' % iter
             print 'Theta = %s' % theta
             break
     else:
         print 'Failed...'
         break

PS: 1. linearRegression.txt

---