通过SparkListener监控spark应用
监控spark应用的方式比较多，比如spark on yarn可以通过yarnClient api监控。这里介绍的是spark内置的一种监控方式
如果是sparkStreaming，对应的则是streamingListener
package cn.com.kong; import org.apache.spark.SparkConf;
import org.apache.spark.scheduler.*;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.status.AppStatusStore;
import org.apache.spark.status.api.v1.ApplicationInfo; public class CustomSparkListener { public static void main(String[] args) { System.setProperty("HADOOP_USER_NAME","etluser"); SparkConf conf = new SparkConf();
conf.set("spark.hadoopRDD.ignoreEmptySplits", "true");
conf.set("spark.sql.adaptive.enabled", "true");
conf.set("spark.sql.adaptive.join.enabled", "true");
conf.set("spark.executor.memoryOverhead", "1024");
conf.set("spark.driver.memoryOverhead", "1024");
conf.set("spark.kryoserializer.buffer.max", "256m");
conf.set("spark.kryoserializer.buffer", "64m");
conf.set("spark.executor.extraJavaOptions", "-XX:+UseG1GC -Dlog4j.configuration=log4j.properties");
conf.set("spark.driver.extraJavaOptions", "-XX:+UseG1GC -Dlog4j.configuration=log4j.properties");
conf.set("spark.sql.parquet.writeLegacyFormat", "true"); SparkSession spark = SparkSession
.builder()
.appName("testSparkListener")
.master("local")
.config(conf)
.enableHiveSupport()
.getOrCreate(); spark.sql("use coveroptimize"); // AppStatusStore appStatusStore = spark.sparkContext().statusStore();
// ApplicationInfo applicationInfo = appStatusStore.applicationInfo();
// applicationInfo.memoryPerExecutorMB(); //可以创建一个类实现Listener接口，然后调用该类实例。
//这里测试，直接创建
spark.sparkContext().addSparkListener(new SparkListenerInterface() {
@Override
public void onExecutorRemoved( SparkListenerExecutorRemoved executorRemoved) {
} /**
* Called when a stage completes successfully or fails, with information on the completed stage.
*/
@Override
public void onStageCompleted( SparkListenerStageCompleted stageCompleted) { } @Override
public void onStageSubmitted( SparkListenerStageSubmitted stageSubmitted) { } @Override
public void onTaskStart(SparkListenerTaskStart taskStart) { }
/**
* Called when a job ends
*/
@Override
public void onJobEnd(SparkListenerJobEnd jobEnd) {
JobResult jobResult = jobEnd.jobResult();
System.err.println("自定义监听器jobEnd jobResult:"+jobResult);
}
/**
* Called when a job starts
*/
@Override
public void onJobStart(SparkListenerJobStart jobStart) {
System.err.println("自定义监听器jobStart,jobId:"+jobStart.jobId());
System.err.println("自定义监听器jobStart,该job下stage数量："+jobStart.stageInfos().size());
} @Override
public void onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { } @Override
public void onExecutorAdded(SparkListenerExecutorAdded executorAdded) { } @Override
public void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) { }
/**
* Called when the application ends
*/
@Override
public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
System.err.println("Application结束，时间："+applicationEnd.time());
} @Override
public void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) { } @Override
public void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) { } @Override
public void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) { } @Override
public void onOtherEvent(SparkListenerEvent event) { } @Override
public void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) { } @Override
public void onSpeculativeTaskSubmitted(SparkListenerSpeculativeTaskSubmitted speculativeTask) { } @Override
public void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executorBlacklisted) { } @Override
public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) { }
/**
* Called when the application starts
*/
@Override
public void onApplicationStart(SparkListenerApplicationStart applicationStart) {
System.err.println("Application启动，appName:"+applicationStart.appName()+",appID"+
applicationStart.appId());
} @Override
public void onExecutorUnblacklisted(SparkListenerExecutorUnblacklisted executorUnblacklisted) { } @Override
public void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) { } @Override
public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) { } @Override
public void onTaskEnd(SparkListenerTaskEnd taskEnd) { }
}); String sql1 = "select roadid,count(1) cn from gridmappingroad group by roadid";
spark.sql(sql1).repartition(2).write().mode(SaveMode.Overwrite)
.saveAsTable("test_listener_table"); spark.stop();
}
}
运行日志：
// :: INFO spark.SparkContext: Running Spark version 2.3.
// :: INFO spark.SparkContext: Submitted application: testSparkListener
// :: INFO spark.SecurityManager: Changing view acls to: kongshuaiwei,etluser
// :: INFO spark.SecurityManager: Changing modify acls to: kongshuaiwei,etluser
// :: INFO spark.SecurityManager: Changing view acls groups to:
// :: INFO spark.SecurityManager: Changing modify acls groups to:
// :: INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(kongshuaiwei, etluser); groups with view permissions: Set(); users with modify permissions: Set(kongshuaiwei, etluser); groups with modify permissions: Set()
// :: INFO util.Utils: Successfully started service 'sparkDriver' on port .
// :: INFO spark.SparkEnv: Registering MapOutputTracker
// :: INFO spark.SparkEnv: Registering BlockManagerMaster
// :: INFO storage.BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
// :: INFO storage.BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
// :: INFO storage.DiskBlockManager: Created local directory at C:\Users\kongshuaiwei\AppData\Local\Temp\blockmgr-b8c578de--4cf3-9e8d-928159f3aecd
// :: INFO memory.MemoryStore: MemoryStore started with capacity 898.5 MB
// :: INFO spark.SparkEnv: Registering OutputCommitCoordinator
// :: INFO util.log: Logging initialized @1729ms
// :: INFO server.Server: jetty-9.3.z-SNAPSHOT
// :: INFO server.Server: Started @1788ms
// :: INFO server.AbstractConnector: Started ServerConnector@5cc5b667{HTTP/1.1,[http/1.1]}{0.0.0.0:}
// :: INFO util.Utils: Successfully started service 'SparkUI' on port .
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@410954b{/jobs,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@10b892d5{/jobs/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3d3f761a{/jobs/job,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@579d011c{/jobs/job/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3670f00{/stages,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@452e26d0{/stages/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@46ab18da{/stages/stage,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7689ddef{/stages/stage/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@687a762c{/stages/pool,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1a2e2935{/stages/pool/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@733c423e{/storage,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@4b629f13{/storage/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@70925b45{/storage/rdd,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1b9ea3e3{/storage/rdd/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@aa22f1c{/environment,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@55e7a35c{/environment/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@37cd92d6{/executors,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5922ae77{/executors/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@4263b080{/executors/threadDump,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@2af616d3{/executors/threadDump/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@71f67a79{/static,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@34abdee4{/,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@71a9b4c7{/api,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@21ca139c{/jobs/job/kill,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@226f885f{/stages/stage/kill,null,AVAILABLE,@Spark}
// :: INFO ui.SparkUI: Bound SparkUI to 0.0.0.0, and started at http://sl1-43087-b01.BJ.DATANGMOBILE.com:4040
// :: INFO executor.Executor: Starting executor ID driver on host localhost
// :: INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port .
// :: INFO netty.NettyBlockTransferService: Server created on sl1--b01.BJ.DATANGMOBILE.com:
// :: INFO storage.BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
// :: INFO storage.BlockManagerMaster: Registering BlockManager BlockManagerId(driver, sl1--b01.BJ.DATANGMOBILE.com, , None)
// :: INFO storage.BlockManagerMasterEndpoint: Registering block manager sl1--b01.BJ.DATANGMOBILE.com: with 898.5 MB RAM, BlockManagerId(driver, sl1--b01.BJ.DATANGMOBILE.com, , None)
// :: INFO storage.BlockManagerMaster: Registered BlockManager BlockManagerId(driver, sl1--b01.BJ.DATANGMOBILE.com, , None)
// :: INFO storage.BlockManager: Initialized BlockManager: BlockManagerId(driver, sl1--b01.BJ.DATANGMOBILE.com, , None)
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@33a630fa{/metrics/json,null,AVAILABLE,@Spark}
// :: INFO internal.SharedState: loading hive config file: file:/D:/ideaIC/workspace/spark-project/coverOptimize/target/classes/hive-site.xml
// :: INFO internal.SharedState: spark.sql.warehouse.dir is not set, but hive.metastore.warehouse.dir is set. Setting spark.sql.warehouse.dir to the value of hive.metastore.warehouse.dir ('/user/hive/warehouse').
// :: INFO internal.SharedState: Warehouse path is '/user/hive/warehouse'.
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@2fb5fe30{/SQL,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@456be73c{/SQL/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@41a6d121{/SQL/execution,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@4f449e8f{/SQL/execution/json,null,AVAILABLE,@Spark}
// :: INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@27e32fe4{/static/sql,null,AVAILABLE,@Spark}
// :: INFO state.StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint
// :: INFO hive.HiveUtils: Initializing HiveMetastoreConnection version 1.2. using Spark classes.
// :: INFO hive.metastore: Trying to connect to metastore with URI thrift://worker03.xxx.xxx.cn:9083// :: INFO hive.metastore: Connected to metastore.
// :: INFO session.SessionState: Created local directory: C:/Users/KONGSH~/AppData/Local/Temp/f35b7531-c964-4d2e-8ba5-b5ade205d12a_resources
// :: INFO session.SessionState: Created HDFS directory: /tmp/hive/etluser/f35b7531-c964-4d2e-8ba5-b5ade205d12a
// :: INFO session.SessionState: Created local directory: C:/Users/KONGSH~/AppData/Local/Temp/kongshuaiwei/f35b7531-c964-4d2e-8ba5-b5ade205d12a
// :: INFO session.SessionState: Created HDFS directory: /tmp/hive/etluser/f35b7531-c964-4d2e-8ba5-b5ade205d12a/_tmp_space.db
// :: INFO client.HiveClientImpl: Warehouse location for Hive client (version 1.2.) is /user/hive/warehouse
// :: INFO parquet.ParquetFileFormat: Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO codegen.CodeGenerator: Code generated in 194.663548 ms
// :: INFO codegen.CodeGenerator: Code generated in 42.50705 ms
// :: INFO memory.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 249.3 KB, free 898.3 MB)
// :: INFO memory.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 24.6 KB, free 898.2 MB)
// :: INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on sl1--b01.BJ.DATANGMOBILE.com: (size: 24.6 KB, free: 898.5 MB)
// :: INFO spark.ContextCleaner: Cleaned accumulator
// :: INFO spark.ContextCleaner: Cleaned accumulator
// :: INFO spark.SparkContext: Created broadcast from
// :: INFO spark.ContextCleaner: Cleaned accumulator
// :: INFO spark.ContextCleaner: Cleaned accumulator
// :: WARN security.UserGroupInformation: No groups available for user etluser
// :: WARN security.UserGroupInformation: No groups available for user etluser
// :: INFO mapred.FileInputFormat: Total input paths to process :
.....
// :: INFO scheduler.DAGScheduler: Registering RDD (saveAsTable at SparkTest.java:)
// :: INFO scheduler.DAGScheduler: Got map stage job (saveAsTable at SparkTest.java:) with output partitions
// :: INFO scheduler.DAGScheduler: Final stage: ShuffleMapStage (saveAsTable at SparkTest.java:)
// :: INFO scheduler.DAGScheduler: Parents of final stage: List()
// :: INFO scheduler.DAGScheduler: Missing parents: List()
自定义监听器jobStart,jobId:
自定义监听器jobStart,该job下stage数量：
// :: INFO scheduler.DAGScheduler: Submitting ShuffleMapStage (MapPartitionsRDD[] at saveAsTable at SparkTest.java:), which has no missing parents
// :: INFO memory.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 30.1 KB, free 898.2 MB)
// :: INFO memory.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.7 KB, free 898.2 MB)
// :: INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on sl1--b01.BJ.DATANGMOBILE.com: (size: 13.7 KB, free: 898.5 MB)
// :: INFO spark.SparkContext: Created broadcast from broadcast at DAGScheduler.scala:
...
// :: INFO scheduler.TaskSetManager: Finished task 30.0 in stage 0.0 (TID ) in ms on localhost (executor driver) (/)
// :: INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
// :: INFO scheduler.DAGScheduler: ShuffleMapStage (saveAsTable at SparkTest.java:) finished in 359.468 s
// :: INFO scheduler.DAGScheduler: looking for newly runnable stages
// :: INFO scheduler.DAGScheduler: running: Set()
// :: INFO scheduler.DAGScheduler: waiting: Set()
// :: INFO scheduler.DAGScheduler: failed: Set()
// :: INFO exchange.ExchangeCoordinator: advisoryTargetPostShuffleInputSize: , targetPostShuffleInputSize .
自定义监听器jobEnd jobResult:JobSucceeded
// :: INFO spark.SparkContext: Starting job: saveAsTable at SparkTest.java:
// :: INFO scheduler.DAGScheduler: Registering RDD (saveAsTable at SparkTest.java:)
// :: INFO scheduler.DAGScheduler: Got job (saveAsTable at SparkTest.java:) with output partitions
// :: INFO scheduler.DAGScheduler: Final stage: ResultStage (saveAsTable at SparkTest.java:)
// :: INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage )
// :: INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage )
自定义监听器jobStart,jobId:
自定义监听器jobStart,该job下stage数量：
// :: INFO scheduler.DAGScheduler: Submitting ShuffleMapStage (MapPartitionsRDD[] at saveAsTable at SparkTest.java:), which has no missing parents
// :: INFO memory.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 22.2 KB, free 898.2 MB)
// :: INFO memory.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 10.5 KB, free 898.2 MB)
// :: INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on sl1--b01.BJ.DATANGMOBILE.com: (size: 10.5 KB, free: 898.5 MB)
// :: INFO spark.SparkContext: Created broadcast from broadcast at DAGScheduler.scala:
// :: INFO scheduler.DAGScheduler: Submitting missing tasks from ShuffleMapStage (MapPartitionsRDD[] at saveAsTable at SparkTest.java:) (first tasks are for partitions Vector())
// :: INFO scheduler.TaskSchedulerImpl: Adding task set 2.0 with tasks
// :: INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID , localhost, executor driver, partition , PROCESS_LOCAL, bytes)
// :: INFO executor.Executor: Running task 0.0 in stage 2.0 (TID )
// :: INFO storage.ShuffleBlockFetcherIterator: Getting non-empty blocks out of blocks
// :: INFO storage.ShuffleBlockFetcherIterator: Started remote fetches in ms
// :: INFO executor.Executor: Finished task 0.0 in stage 2.0 (TID ). bytes result sent to driver
// :: INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID ) in ms on localhost (executor driver) (/)
// :: INFO scheduler.TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool
// :: INFO scheduler.DAGScheduler: ShuffleMapStage (saveAsTable at SparkTest.java:) finished in 0.135 s
// :: INFO scheduler.DAGScheduler: looking for newly runnable stages
// :: INFO scheduler.DAGScheduler: running: Set()
// :: INFO scheduler.DAGScheduler: waiting: Set(ResultStage )
// :: INFO scheduler.DAGScheduler: failed: Set()
// :: INFO scheduler.DAGScheduler: Submitting ResultStage (ShuffledRowRDD[] at saveAsTable at SparkTest.java:), which has no missing parents
// :: INFO memory.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 148.9 KB, free 898.0 MB)
// :: INFO memory.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 52.1 KB, free 898.0 MB)
// :: INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on sl1--b01.BJ.DATANGMOBILE.com: (size: 52.1 KB, free: 898.4 MB)
// :: INFO spark.SparkContext: Created broadcast from broadcast at DAGScheduler.scala:
// :: INFO scheduler.DAGScheduler: Submitting missing tasks from ResultStage (ShuffledRowRDD[] at saveAsTable at SparkTest.java:) (first tasks are for partitions Vector(, ))
// :: INFO scheduler.TaskSchedulerImpl: Adding task set 3.0 with tasks
// :: INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID , localhost, executor driver, partition , ANY, bytes)
// :: INFO executor.Executor: Running task 0.0 in stage 3.0 (TID )
// :: INFO storage.ShuffleBlockFetcherIterator: Getting non-empty blocks out of blocks
// :: INFO storage.ShuffleBlockFetcherIterator: Started remote fetches in ms
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO parquet.ParquetWriteSupport: Initialized Parquet WriteSupport with Catalyst schema:
{
"type" : "struct",
"fields" : [ {
"name" : "roadid",
"type" : "string",
"nullable" : true,
"metadata" : {
"comment" : "??id"
}
}, {
"name" : "cn",
"type" : "long",
"nullable" : false,
"metadata" : { }
} ]
}
and corresponding Parquet message type:
message spark_schema {
optional binary roadid (UTF8);
required int64 cn;
} // :: INFO compress.CodecPool: Got brand-new compressor [.snappy]
// :: INFO output.FileOutputCommitter: Saved output of task 'attempt_20200117135657_0003_m_000000_0' to hdfs://master01.xxx.xxx.cn:8020/user/hive/warehouse/coveroptimize.db/test_listener_table/_temporary/0/task_20200117135657_0003_m_000000
// :: INFO mapred.SparkHadoopMapRedUtil: attempt_20200117135657_0003_m_000000_0: Committed
// :: INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID ). bytes result sent to driver
// :: INFO scheduler.TaskSetManager: Starting task 1.0 in stage 3.0 (TID , localhost, executor driver, partition , ANY, bytes)
// :: INFO executor.Executor: Running task 1.0 in stage 3.0 (TID )
// :: INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID ) in ms on localhost (executor driver) (/)
// :: INFO storage.ShuffleBlockFetcherIterator: Getting non-empty blocks out of blocks
// :: INFO storage.ShuffleBlockFetcherIterator: Started remote fetches in ms
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO datasources.SQLHadoopMapReduceCommitProtocol: Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
// :: INFO parquet.ParquetWriteSupport: Initialized Parquet WriteSupport with Catalyst schema:
{
"type" : "struct",
"fields" : [ {
"name" : "roadid",
"type" : "string",
"nullable" : true,
"metadata" : {
"comment" : "??id"
}
}, {
"name" : "cn",
"type" : "long",
"nullable" : false,
"metadata" : { }
} ]
}
and corresponding Parquet message type:
message spark_schema {
optional binary roadid (UTF8);
required int64 cn;
} // :: INFO output.FileOutputCommitter: Saved output of task 'attempt_20200117135657_0003_m_000001_0' to hdfs://master01.xxx.xxx.cn:8020/user/hive/warehouse/coveroptimize.db/test_listener_table/_temporary/0/task_20200117135657_0003_m_000001
// :: INFO mapred.SparkHadoopMapRedUtil: attempt_20200117135657_0003_m_000001_0: Committed
// :: INFO executor.Executor: Finished task 1.0 in stage 3.0 (TID ). bytes result sent to driver
// :: INFO scheduler.TaskSetManager: Finished task 1.0 in stage 3.0 (TID ) in ms on localhost (executor driver) (/)
// :: INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool
// :: INFO scheduler.DAGScheduler: ResultStage (saveAsTable at SparkTest.java:) finished in 0.466 s
自定义监听器jobEnd jobResult:JobSucceeded
// :: INFO scheduler.DAGScheduler: Job finished: saveAsTable at SparkTest.java:, took 0.616115 s
// :: INFO datasources.FileFormatWriter: Job null committed.
// :: INFO datasources.FileFormatWriter: Finished processing stats for job null.
// :: INFO hive.HiveExternalCatalog: Persisting file based data source table `coveroptimize`.`test_listener_table` into Hive metastore in Hive compatible format.
Application结束，时间：
// :: INFO server.AbstractConnector: Stopped Spark@5cc5b667{HTTP/1.1,[http/1.1]}{0.0.0.0:}
// :: INFO ui.SparkUI: Stopped Spark web UI at http://sl1-43087-b01.BJ.DATANGMOBILE.com:4040
// :: INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
// :: INFO memory.MemoryStore: MemoryStore cleared
// :: INFO storage.BlockManager: BlockManager stopped
// :: INFO storage.BlockManagerMaster: BlockManagerMaster stopped
// :: INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
// :: INFO spark.SparkContext: Successfully stopped SparkContext
// :: INFO util.ShutdownHookManager: Shutdown hook called
// :: INFO util.ShutdownHookManager: Deleting directory C:\Users\kongshuaiwei\AppData\Local\Temp\spark-547cf37e-2d1e-433c-a584-6c5b7365909f Process finished with exit code
通过SparkListener监控spark应用的更多相关文章
- 监控Spark应用方法简介
监控Spark应用有很多种方法. Web接口每一个SparkContext启动一个web UI用来展示应用相关的一些非常有用的信息,默认在4040端口.这些信息包括: 任务和调度状态的列表RDD大小和 ...
- Ubuntu 14.10 下Ganglia监控Spark集群
由于Licene的限制,没有放到默认的build里面,所以在官方网站下载的二进制文件中并不包含Gangla模块,如果需要使用,需要自己编译.在使用Maven编译Spark的时候,我们可以加上-Pspa ...
- Spark2.2(三十九)：如何根据appName监控spark任务,当任务不存在则启动（任务存在当超过多久没有活动状态则kill，等待下次启动）
业务需求 实现一个根据spark任务的appName来监控任务是否存在,及任务是否卡死的监控. 1)给定一个appName,根据appName从yarn application -list中验证任务是 ...
- Spark应用监控解决方案--使用Prometheus和Grafana监控Spark应用
Spark任务启动后,我们通常都是通过跳板机去Spark UI界面查看对应任务的信息,一旦任务多了之后,这将会是让人头疼的问题.如果能将所有任务信息集中起来监控,那将会是很完美的事情. 通过Spark ...
- Spark（五十）：使用JvisualVM监控Spark Executor JVM
引导 Windows环境下JvisulaVM一般存在于安装了JDK的目录${JAVA_HOME}/bin/JvisualVM.exe,它支持(本地和远程)jstatd和JMX两种方式连接远程JVM. ...
- 使用 JvisualVM 监控 spark executor
使用 JvisualVM,需要先配置 java 的启动参数 jmx 正常情况下,如下配置 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmx ...
- spark的运行指标监控
sparkUi的4040界面已经有了运行监控指标,为什么我们还要自定义存入redis? 1.结合自己的业务,可以将监控页面集成到自己的数据平台内,方便问题查找,邮件告警 2.可以在sparkUi的基础 ...
- Spark的Straggler深入学习（1）：如何在本地图形监控远程Spark的GC情况——使用java自带的jvisualvm
一.本文的目的 Straggler是目前研究的热点,Spark中也存在Straggler的问题.GC问题是总所周知的导致Straggler的重要因素之一,为了了解GC导致的Straggle ...
- Spark监控官方文档学习笔记
任务的监控和使用 有几种方式监控spark应用:Web UI,指标和外部方法 Web接口 每个SparkContext都会启动一个web UI,默认是4040端口,用来展示一些信息: 一系列调度的st ...
随机推荐
- 洛谷P2758编辑距离（线性DP)
题目描述 设A和B是两个字符串.我们要用最少的字符操作次数,将字符串A转换为字符串B.这里所说的字符操作共有三种: 1.删除一个字符: 2.插入一个字符: 3.将一个字符改为另一个字符: !皆为小写字 ...
- 【原】Django问题总结
一:python manage.py makemigrations No changes detected 在修改了models.py后,有些用户会喜欢用python manage.py makemi ...
- 「CF383C Propagating tree」
这应该属于一个比较麻烦的数据结构处理树上问题. 题目大意 给出一颗根节点编号为 \(1\) 的树,对于一个节点修改时在它的子树中对于深度奇偶性相同的节点加上这个权值,不同则减去这个值,单点查询. 分析 ...
- 洛谷P1301 魔鬼之城 题解
想找原题请点击这里:传送门 题目描述 在一个被分割为N*M个正方形房间的矩形魔鬼之城中,一个探险者必须遵循下列规则才能跳跃行动.他必须从(, )进入,从(N, M)走出:在每一房间的墙壁上都写了一个魔 ...
- MySQL - 设置UTF-8编码
1. 在Windows上,安装时请选择UTF-8编码,以便正确地处理中文. 在Mac或Linux上,需要编辑MySQL的配置文件,把数据库默认的编码全部改为UTF-8.MySQL的配置文件默认存放在/ ...
- [Write-up]-Trollcave: 1.2
关于 下载地址:点我 Flag:root/flag.txt 哔哩哔哩:视频 信息收集 不知道VM虚拟机怎么啦,导入镜像后,用Nmap扫了,发现不了主机.所以这次用了VBox. vboxnet0的IP为 ...
- IDEA下的SVN设置以及TortoiseSVN安装后bin目录下没有svn.exe如何解决？
1.首先,我们要确保电脑上已经安装了TortoiseSVN. 2.打开IDEA-File-Settings-Version Control-Subversion,在右边的界面上选择TortoiseSv ...
- Java基础 -1.4
标识符与关键字 对于标识符的组成在Java之中的定义如下:由字母.数字._.$ 组成 其中不能使用Java的保留字(关键字) 其中 $ 一般都有特殊的含义 不建议出现在自己所编写的代码上 关键字 是系 ...
- vmware 因误删Linux 虚拟机磁盘，无法启动处理方法
有可能我们在做了以下误操作,导致Linux系统无法启动: 1). 磁盘损坏或虚拟机磁盘被我们删除了,而fstab文件没有更新: 2). 由于误操作或其它原因使动态库错误. 1. 首先准备好系统安装盘, ...
- nacos作为配置中心动态刷新@RefreshScope添加后取值为null的一个问题
之前springboot项目常量类如下形式: @Component @RefreshScope//nacos配置中心时添加上 public class Constants { @Value(" ...