SparkContext源码阅读
SparkContext是spark的入口,通过它来连接集群、创建RDD、广播变量等等。
class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationClient { private val creationSite: CallSite = Utils.getCallSite() //如果生命了2个sparkContext,则会使用warn来取代exception.防止退出
private val allowMultipleContexts: Boolean =
config.getBoolean("spark.driver.allowMultipleContexts", false) ..防止两个sparkcontext同时运行
SparkContext.markPartiallyConstructed(this, allowMultipleContexts) private[spark] var preferredNodeLocationData: Map[String, Set[SplitInfo]] = Map() val startTime = System.currentTimeMillis() //当提交任务执行spark-submit时,加载系统环境变量
def this() = this(new SparkConf()) def this(master: String, appName: String, conf: SparkConf) =
this(SparkContext.updatedConf(conf, master, appName)) //preferredNodeLocationData 用于启动查找nodes,启动相应的container
def this(
master: String,
appName: String,
sparkHome: String = null,
jars: Seq[String] = Nil,
environment: Map[String, String] = Map(),
preferredNodeLocationData: Map[String, Set[SplitInfo]] = Map()) =
{
this(SparkContext.updatedConf(new SparkConf(), master, appName, sparkHome, jars, environment))
if (preferredNodeLocationData.nonEmpty) {
logWarning("Passing in preferred locations has no effect at all, see SPARK-8949")
}
this.preferredNodeLocationData = preferredNodeLocationData //构造函数
private[spark] def this(master: String, appName: String) =
this(master, appName, null, Nil, Map(), Map()) private[spark] def this(master: String, appName: String, sparkHome: String) =
this(master, appName, sparkHome, Nil, Map(), Map()) private[spark] def this(master: String, appName: String, sparkHome: String, jars: Seq[String]) =
this(master, appName, sparkHome, jars, Map(), Map()) private[spark] def conf: SparkConf = _conf //clone Conf,那么在运行时就不能被修改
def getConf: SparkConf = conf.clone() def jars: Seq[String] = _jars
def files: Seq[String] = _files
def master: String = _conf.get("spark.master")
def appName: String = _conf.get("spark.app.name") private[spark] def isEventLogEnabled: Boolean = _conf.getBoolean("spark.eventLog.enabled", false)
private[spark] def eventLogDir: Option[URI] = _eventLogDir
private[spark] def eventLogCodec: Option[String] = _eventLogCodec //创建schedular
val (sched, ts) = SparkContext.createTaskScheduler(this, master)
_schedulerBackend = sched
_taskScheduler = ts
_dagScheduler = new DAGScheduler(this)
_heartbeatReceiver.ask[Boolean](TaskSchedulerIsSet) //启动taskschedular
_taskScheduler.start() applicationId = _taskScheduler.applicationId()
_applicationAttemptId = taskScheduler.applicationAttemptId()
_conf.set("spark.app.id", _applicationId)
_env.blockManager.initialize(_applicationId) //创建一个新的RDD,通过step来增加元素
def range(
start: Long,
end: Long,
step: Long = 1,
numSlices: Int = defaultParallelism): RDD[Long] = withScope {
assertNotStopped()
// when step is 0, range will run infinitely
require(step != 0, "step cannot be 0")
val numElements: BigInt = {
val safeStart = BigInt(start)
val safeEnd = BigInt(end)
if ((safeEnd - safeStart) % step == 0 || safeEnd > safeStart ^ step > 0) {
(safeEnd - safeStart) / step
} else {
(safeEnd - safeStart) / step + 1
}
} parallelize(0 until numSlices, numSlices).mapPartitionsWithIndex((i, _) => {
val partitionStart = (i * numElements) / numSlices * step + start
val partitionEnd = (((i + 1) * numElements) / numSlices) * step + start
def getSafeMargin(bi: BigInt): Long =
if (bi.isValidLong) {
bi.toLong
} else if (bi > 0) {
Long.MaxValue
} else {
Long.MinValue
}
val safePartitionStart = getSafeMargin(partitionStart)
val safePartitionEnd = getSafeMargin(partitionEnd) new Iterator[Long] {
private[this] var number: Long = safePartitionStart
private[this] var overflow: Boolean = false override def hasNext =
if (!overflow) {
if (step > 0) {
number < safePartitionEnd
} else {
number > safePartitionEnd
}
} else false override def next() = {
val ret = number
number += step
if (number < ret ^ step < 0) {
overflow = true
}
ret
}
}
})
} //创建一个RDD
def makeRDD[T: ClassTag](
seq: Seq[T],
numSlices: Int = defaultParallelism): RDD[T] = withScope {
parallelize(seq, numSlices)
} //读取本地、HDFS的文件,返回一个String的字符串
def textFile(
path: String,
minPartitions: Int = defaultMinPartitions): RDD[String] = withScope {
assertNotStopped()
hadoopFile(path, classOf[TextInputFormat], classOf[LongWritable], classOf[Text],
minPartitions).map(pair => pair._2.toString)
} //加载一个二进制文件,
@Experimental
def binaryRecords(
path: String,
recordLength: Int,
conf: Configuration = hadoopConfiguration): RDD[Array[Byte]] = withScope {
assertNotStopped()
conf.setInt(FixedLengthBinaryInputFormat.RECORD_LENGTH_PROPERTY, recordLength)
val br = newAPIHadoopFile[LongWritable, BytesWritable, FixedLengthBinaryInputFormat](path,
classOf[FixedLengthBinaryInputFormat],
classOf[LongWritable],
classOf[BytesWritable],
conf = conf)
val data = br.map { case (k, v) =>
val bytes = v.getBytes
assert(bytes.length == recordLength, "Byte array does not have correct length")
bytes
}
data
} //获得一个为HADOOP sequenceFile给定键值对类型的RDD
def sequenceFile[K, V](path: String,
keyClass: Class[K],
valueClass: Class[V],
minPartitions: Int
): RDD[(K, V)] = withScope {
assertNotStopped()
val inputFormatClass = classOf[SequenceFileInputFormat[K, V]]
hadoopFile(path, inputFormatClass, keyClass, valueClass, minPartitions)
} //1300发送一个广播变量到集群的每个节点
def broadcast[T: ClassTag](value: T): Broadcast[T] = {
assertNotStopped()
if (classOf[RDD[_]].isAssignableFrom(classTag[T].runtimeClass)) {
logWarning("Can not directly broadcast RDDs; instead, call collect() and "
+ "broadcast the result (see SPARK-5063)")
}
val bc = env.broadcastManager.newBroadcast[T](value, isLocal)
val callSite = getCallSite
logInfo("Created broadcast " + bc.id + " from " + callSite.shortForm)
cleaner.foreach(_.registerBroadcastForCleanup(bc))
bc
}
SparkContext源码阅读的更多相关文章
- SparkConf加载与SparkContext创建(源码阅读一)
即日起开始spark源码阅读之旅,这个过程是相当痛苦的,也许有大量的看不懂,但是每天一个方法,一点点看,相信总归会有极大地提高的.那么下面开始: 创建sparkConf对象,那么究竟它干了什么了类,从 ...
- Spark源码阅读之存储体系--存储体系概述与shuffle服务
一.概述 根据<深入理解Spark:核心思想与源码分析>一书,结合最新的spark源代码master分支进行源码阅读,对新版本的代码加上自己的一些理解,如有错误,希望指出. 1.块管理器B ...
- 【原】FMDB源码阅读(三)
[原]FMDB源码阅读(三) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 FMDB比较优秀的地方就在于对多线程的处理.所以这一篇主要是研究FMDB的多线程处理的实现.而 ...
- 【原】FMDB源码阅读(二)
[原]FMDB源码阅读(二) 本文转载请注明出处 -- polobymulberry-博客园 1. 前言 上一篇只是简单地过了一下FMDB一个简单例子的基本流程,并没有涉及到FMDB的所有方方面面,比 ...
- 【原】FMDB源码阅读(一)
[原]FMDB源码阅读(一) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 说实话,之前的SDWebImage和AFNetworking这两个组件我还是使用过的,但是对于 ...
- 【原】AFNetworking源码阅读(六)
[原]AFNetworking源码阅读(六) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 这一篇的想讲的,一个就是分析一下AFSecurityPolicy文件,看看AF ...
- 【原】AFNetworking源码阅读(五)
[原]AFNetworking源码阅读(五) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 上一篇中提及到了Multipart Request的构建方法- [AFHTTP ...
- 【原】AFNetworking源码阅读(四)
[原]AFNetworking源码阅读(四) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 上一篇还遗留了很多问题,包括AFURLSessionManagerTaskDe ...
- 【原】AFNetworking源码阅读(三)
[原]AFNetworking源码阅读(三) 本文转载请注明出处 —— polobymulberry-博客园 1. 前言 上一篇的话,主要是讲了如何通过构建一个request来生成一个data tas ...
随机推荐
- PHP中字符串的连接和换行
PHP中字符串的连接使用点,不是加号.换行\n需要用双引号括起来,不能用单引号. eg. <?php print_r("hello"."\n"); pri ...
- HDU 1372 Knight Moves
最近在学习广搜 这道题同样是一道简单广搜题=0= 题意:(百度复制粘贴0.0) 题意:给出骑士的骑士位置和目标位置,计算骑士要走多少步 思路:首先要做这道题必须要理解国际象棋中骑士的走法,国际象棋中 ...
- JavaWeb chapter9 JSP基础
1. Servlet的缺陷 一个动态网页中,大部分内容都是HTML代码等固定不变的内容,编写和修改HTML非常不方便,令人厌恶: 部署Servlet是繁琐而且容易出错的任务:(Servlet3.0规 ...
- 前端获取url参数
function GetQueryString(name){ var reg = new RegExp("(^|&)"+ name +"=([^&]*)( ...
- c# mvc使用 npoi下载 excel
IWorkbook book = new NPOI.HSSF.UserModel.HSSFWorkbook(); //添加一个sheet ISheet sheet1 = book.CreateShee ...
- CodeForces 688B-Lovely Palindromes
题意: 给出一串数字,要你输出它的回文数,就这么简单. 分析: 可以用数组去做,也可以用reversed()函数(这个更简单). 代码如下: #include <iostream> #in ...
- Invalid byte 3 of 3-byte UTF-8 sequence
用maven编译,tomcat启动时报错:IOException parsing XML document from class path resource [applicationContext.x ...
- Ubuntu下部分文件操作的命令
(1)创建目录 mkdir filefoldname (2)删除空目录 rmdir filefoldname (3)删除非空目录及其中所有文件 rm -rf filefoldname (4)移动文件或 ...
- RF Analyzer for Android 安卓平台连接HackRF的App
Over the last week I've been working on a new project, trying to build a spectrum analyzer for Andro ...
- js常见函数汇总
/** * 隐藏元素 * @param {String} elem */ function hide(elem){ var curDisplay = getStyle(elem, 'di ...