1.[start-all.sh]

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# # Start all spark daemons.    #启动所有守护进程
# Starts the master on this node.  #在这个节点上启动master
# Starts a worker on each node specified in conf/slaves #在conf/slaves文件下指定的节点上启动worker进程 if [ -z "${SPARK_HOME}" ]; then    #判断spark环境变量在不在
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"  #如果在的话,就把环境变量导入
fi # Load the Spark configuration    #加载配置文件
. "${SPARK_HOME}/sbin/spark-config.sh"  #调用spark-config.sh脚本 # Start Master
"${SPARK_HOME}/sbin"/start-master.sh  #启动master脚本进程 # Start Workers
"${SPARK_HOME}/sbin"/start-slaves.sh  #启动slaves脚本进程

2.[start-master.sh]

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# # Starts the master on the machine this script is executed on.  #在这个机器上执行脚本,启动master进程 if [ -z "${SPARK_HOME}" ]; then    #首先配置环境变量
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
fi # NOTE: This exact class name is matched downstream by SparkSubmit.
# Any changes need to be reflected there.
CLASS="org.apache.spark.deploy.master.Master" if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
echo "Usage: ./sbin/start-master.sh [options]"
pattern="Usage:"
pattern+="\|Using Spark's default log4j profile:"
pattern+="\|Registered signal handlers for" "${SPARK_HOME}"/bin/spark-class $CLASS --help >& | grep -v "$pattern" >&
exit
fi ORIGINAL_ARGS="$@" . "${SPARK_HOME}/sbin/spark-config.sh" . "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_MASTER_PORT" = "" ]; then
SPARK_MASTER_PORT=
fi if [ "$SPARK_MASTER_HOST" = "" ]; then
SPARK_MASTER_HOST=`hostname -f`
fi if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then
SPARK_MASTER_WEBUI_PORT=
fi "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS \
--host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \
$ORIGINAL_ARGS

spark复习笔记(4):spark脚本分析的更多相关文章

  1. spark复习笔记(2)

    之前工作的时候经常用,隔了段时间,现在学校要用学的东西也忘了,翻翻书谢谢博客吧. 1.什么是spark? Spark是一种快速.通用.可扩展的大数据分析引擎,2009年诞生于加州大学伯克利分校AMPL ...

  2. spark学习笔记总结-spark入门资料精化

    Spark学习笔记 Spark简介 spark 可以很容易和yarn结合,直接调用HDFS.Hbase上面的数据,和hadoop结合.配置很容易. spark发展迅猛,框架比hadoop更加灵活实用. ...

  3. Spark学习笔记-使用Spark History Server

    在运行Spark应用程序的时候,driver会提供一个webUI给出应用程序的运行信息,但是该webUI随着应用程序的完成而关闭端口,也就是 说,Spark应用程序运行完后,将无法查看应用程序的历史记 ...

  4. Spark 学习笔记之 Spark history Server 搭建

    在hdfs上建立文件夹/directory hadoop fs -mkdir /directory 进入conf目录  spark-env.sh 增加以下配置 export SPARK_HISTORY ...

  5. spark复习笔记(5):API分析

    0.spark是基于hadoop的mr模型,扩展了MR,高效实用MR模型,内存型集群计算,提高了app处理速度. 1.特点:(1)在内存中存储中间结果 (2)支持多种语言:java scala pyt ...

  6. spark复习笔记(4):RDD变换

    一.RDD变换 1.返回执行新的rdd的指针,在rdd之间创建依赖关系.每个rdd都有一个计算函数和指向父rdd的指针 Spark是惰性的,因此除非调用某个转换或动作,否则不会执行任何操作,否则将触发 ...

  7. spark复习笔记(1)

    使用spark实现work count ---------------------------------------------------- (1)用sc.textFile(" &quo ...

  8. spark复习笔记(7):sparkstreaming

    一.介绍 1.sparkStreaming是核心模块Spark API的扩展,具有可伸缩,高吞吐量以及容错的实时数据流处理等.数据可以从许多来源(如Kafka,Flume,Kinesis或TCP套接字 ...

  9. spark复习笔记(7):sparkSQL

    一.saprkSQL模块,使用类sql的方式访问Hadoop,实现mr计算,底层使用的是rdd 1.hive //hadoop  mr  sql 2.phenoix //hbase上构建sql的交互过 ...

随机推荐

  1. HTML中的表单<form>标签

    一.HTML表单 HTML 表单用于搜集不同类型的用户输入. HTML 表单包含表单元素,表单元素指的是不同类型的 input 元素.复选框.单选按钮.提交按钮等等. 关于表单的更多内容可以参考htt ...

  2. python 存取字典dict

    数据处理的时候主要通过两个函数(1):np.save(“test.npy”,数据结构) ----存数据(2):data =np.load('test.npy") ----取数据 1.存列表 ...

  3. 3D Computer Grapihcs Using OpenGL - 05 EBO

    本节将采用两种方法绘制两个三角形. 先看第一种方法的代码 MyGlWindow.cpp #include <gl\glew.h> #include "MyGlWindow.h&q ...

  4. SpringBoot项目属性配置-第二章

    SpringBoot入门 1. 相信很多人选择Spring Boot主要是考虑到它既能兼顾Spring的强大功能,还能实现快速开发的便捷.我们在Spring Boot使用过程中,最直观的感受就是没有了 ...

  5. 大数据笔记(十)——Shuffle与MapReduce编程案例(A)

    一.什么是Shuffle yarn-site.xml文件配置的时候有这个参数:yarn.nodemanage.aux-services:mapreduce_shuffle 因为mapreduce程序运 ...

  6. 转:SpringMVC常见面试题总结(超详细回答)

    原文:https://blog.csdn.net/a745233700/article/details/80963758  我略微修改了下某些地方 1.什么是Spring MVC ?简单介绍下你对sp ...

  7. SpringBoot:运行原理探究

    西部开源-秦疆老师:基于SpringBoot 2.1.6 的博客教程 秦老师交流Q群号: 664386224 未授权禁止转载!编辑不易 , 转发请注明出处!防君子不防小人,共勉! SpringBoot ...

  8. progress组件(进度条)

    progress组件:进度条 progress组件的属性: percent:类型:number 设置百分比 (0~100) show-info:类型:布尔 在进度条右侧显示百分比 border-rad ...

  9. leetcode-mid-Linked list-2 Add Two Numbers

    mycode 87.22% # Definition for singly-linked list. # class ListNode(object): # def __init__(self, x) ...

  10. MySQL高可用方案 MHA之一MHA安装

    MHA0.58安装 MHA(Master High Availability)由两部分组成:MHA Manager(管理节点)和MHA Node(数据节点).管理节点mha4mysql-manager ...