org.apache.spark.logging类报错
一,1 在使用spark读取kafka数据时,当spark升级到2.0之后,出现如上问题:之前遇到了,当时在工程里面添加了org.apache.spark.Logging类,能够运行。
但是在后期使用过程中,又遇到了相同的问题,为了一劳永逸,今天彻底把问题解决。
在项目下创建org.apache.spark.logging类
将源码考入类中
package org.apache.spark import org.apache.log4j.{LogManager, PropertyConfigurator}
import org.slf4j.{Logger, LoggerFactory}
import org.slf4j.impl.StaticLoggerBinder import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.util.Utils
/**
* Created by Administrator on 2017/8/11.
*/ /**
* :: DeveloperApi ::
* Utility trait for classes that want to log data. Creates a SLF4J logger for the class and allows
* logging messages at different levels using methods that only evaluate parameters lazily if the
* log level is enabled.
*
* NOTE: DO NOT USE this class outside of Spark. It is intended as an internal utility.
* This will likely be changed or removed in future releases.
*/
@DeveloperApi
trait Logging {
// Make the log field transient so that objects with Logging can
// be serialized and used on another machine
@transient private var log_ : Logger = null // Method to get the logger name for this object
protected def logName = {
// Ignore trailing $'s in the class names for Scala objects
this.getClass.getName.stripSuffix("$")
} // Method to get or create the logger for this object
protected def log: Logger = {
if (log_ == null) {
initializeIfNecessary()
log_ = LoggerFactory.getLogger(logName)
}
log_
} // Log methods that take only a String
protected def logInfo(msg: => String) {
if (log.isInfoEnabled) log.info(msg)
} protected def logDebug(msg: => String) {
if (log.isDebugEnabled) log.debug(msg)
} protected def logTrace(msg: => String) {
if (log.isTraceEnabled) log.trace(msg)
} protected def logWarning(msg: => String) {
if (log.isWarnEnabled) log.warn(msg)
} protected def logError(msg: => String) {
if (log.isErrorEnabled) log.error(msg)
} // Log methods that take Throwables (Exceptions/Errors) too
protected def logInfo(msg: => String, throwable: Throwable) {
if (log.isInfoEnabled) log.info(msg, throwable)
} protected def logDebug(msg: => String, throwable: Throwable) {
if (log.isDebugEnabled) log.debug(msg, throwable)
} protected def logTrace(msg: => String, throwable: Throwable) {
if (log.isTraceEnabled) log.trace(msg, throwable)
} protected def logWarning(msg: => String, throwable: Throwable) {
if (log.isWarnEnabled) log.warn(msg, throwable)
} protected def logError(msg: => String, throwable: Throwable) {
if (log.isErrorEnabled) log.error(msg, throwable)
} protected def isTraceEnabled(): Boolean = {
log.isTraceEnabled
} private def initializeIfNecessary() {
if (!Logging.initialized) {
Logging.initLock.synchronized {
if (!Logging.initialized) {
initializeLogging()
}
}
}
} private def initializeLogging() {
// Don't use a logger in here, as this is itself occurring during initialization of a logger
// If Log4j 1.2 is being used, but is not initialized, load a default properties file
val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr
// This distinguishes the log4j 1.2 binding, currently
// org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently
// org.apache.logging.slf4j.Log4jLoggerFactory
val usingLog4j12 = "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) lazy val isInInterpreter: Boolean = {
try {
val interpClass = classForName("org.apache.spark.repl.Main")
interpClass.getMethod("interp").invoke(null) != null
} catch {
case _: ClassNotFoundException => false
}
}
def classForName(className: String): Class[_] = {
Class.forName(className, true, getContextOrSparkClassLoader)
// scalastyle:on classforname
}
def getContextOrSparkClassLoader: ClassLoader =
Option(Thread.currentThread().getContextClassLoader).getOrElse(getSparkClassLoader)
def getSparkClassLoader: ClassLoader = getClass.getClassLoader if (usingLog4j12) {
val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
if (!log4j12Initialized) {
// scalastyle:off println
if (isInInterpreter) {
val replDefaultLogProps = "org/apache/spark/log4j-defaults-repl.properties"
Option(Utils.getSparkClassLoader.getResource(replDefaultLogProps)) match {
case Some(url) =>
PropertyConfigurator.configure(url)
System.err.println(s"Using Spark's repl log4j profile: $replDefaultLogProps")
System.err.println("To adjust logging level use sc.setLogLevel(\"INFO\")")
case None =>
System.err.println(s"Spark was unable to load $replDefaultLogProps")
}
} else {
val defaultLogProps = "org/apache/spark/log4j-defaults.properties"
Option(Utils.getSparkClassLoader.getResource(defaultLogProps)) match {
case Some(url) =>
PropertyConfigurator.configure(url)
System.err.println(s"Using Spark's default log4j profile: $defaultLogProps")
case None =>
System.err.println(s"Spark was unable to load $defaultLogProps")
}
}
// scalastyle:on println
}
}
Logging.initialized = true // Force a call into slf4j to initialize it. Avoids this happening from multiple threads
// and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html
log
}
} private object Logging {
@volatile private var initialized = false
val initLock = new Object()
try {
// We use reflection here to handle the case where users remove the
// slf4j-to-jul bridge order to route their logs to JUL.
val bridgeClass = Utils.classForName("org.slf4j.bridge.SLF4JBridgeHandler")
bridgeClass.getMethod("removeHandlersForRootLogger").invoke(null)
val installed = bridgeClass.getMethod("isInstalled").invoke(null).asInstanceOf[Boolean]
if (!installed) {
bridgeClass.getMethod("install").invoke(null)
}
} catch {
case e: ClassNotFoundException => // can't log anything yet so just fail silently
}
}
无需更改源码
2。现在重新运行下项目看适口可以运行
3,如果还是不行,进入cmd中进入项目下刚才建的org.apache.spark下,执行jar -cvf logging.jar 意思是将刚才拷贝进去的源码打成jar包然后引入当前项目依赖中
4,运行即会成功然后将此包在放入你的spark集群中的lib目录下 如果还出现问题
5
6 这两个包已经是完整的了 在我的百度网盘点击下面下载就好,按照上诉步骤 希望可以帮助到你
链接:https://pan.baidu.com/s/1wOL29M5vQVqmSG6ywxLxNg
提取码:61td
org.apache.spark.logging类报错的更多相关文章
- Spark程序编译报错error: object apache is not a member of package org
Spark程序编译报错: [INFO] Compiling 2 source files to E:\Develop\IDEAWorkspace\spark\target\classes at 156 ...
- 往sde中导入要素类报错000732
sde可以成功连接,可以在Server中注册. 但是向sde中导入要素类报错000732,如图所示. 点击红色圆圈提示 ERROR 000732. 将路径修改为绝对路径即可,如下图所示.
- apache ab压力测试报错(apr_socket_recv: Connection reset by peer (104))
apache ab压力测试报错(apr_socket_recv: Connection reset by peer (104)) 今天用apache 自带的ab工具测试,当并发量达到1000多的时 ...
- maven项目引用时,导入类报错,选择两个项目同时执行Maven update
maven项目引用时,导入类报错,选择两个项目同时执行Maven update springboot引入第三方jar,需要扫描时加@ComponentScan("第三方的包名") ...
- 继承ActionSupper类报错 --Struts2
如下图所示,继承ActionSupper类报错: 原因:缺少Struts2中JAR包,具体是:
- apache ab压力测试报错apr_socket_recv
apache ab压力测试报错(apr_socket_recv: Connection reset by peer (104)) apache 自带的ab工具测试,当并发量达到1000多的时候报错如下 ...
- 启动Tomcat服务时,出现org.apache.catalina.startup.VersionLoggerListener报错
启动Tomcat服务时,出现org.apache.catalina.startup.VersionLoggerListener报错解决办法:打开Tomcat安装后目录,进入conf文件夹,找到配置文件 ...
- php通过JavaBridge调用Java类库和不带包的自定义java类成功 但是调用带包的自定义Java类报错,该怎么解决
php通过JavaBridge调用Java类库和不带包的自定义java类成功 但是调用带包的自定义Java类报错,Class.forName("com.mysql.jdbc.Driver&q ...
- 【IntellJ IDEA】idea启动测试类报错Error running 'Test1.test': Command line is too long. Shorten command line for Test1.test or also for JUnit default configuration.
idea启动测试类报错 Error running 'Test1.test': Command line is too long. Shorten command line for Test1.tes ...
随机推荐
- 使用PowerShell比较本地文本文件与Web上的文本文件是否相同
使用PowerShell比较本地文本文件是否相同通常有两种方式:1.通过Get-FileHash这个命令,比较两个文件的哈希是否相同:2.通过Compare-Object这个命令,逐行比较两个文件的内 ...
- Centos7 防护墙 设置端口
Centos7中的防火墙调整为firewalld,试一下systemctl stop firewalld关闭防火墙. 命令:systemctl stop firewalld 命令:systemctl ...
- Java设计模式学习笔记(三) 工厂方法模式
前言 本篇是设计模式学习笔记的其中一篇文章,如对其他模式有兴趣,可从该地址查找设计模式学习笔记汇总地址 1. 简介 上一篇博客介绍了简单工厂模式,简单工厂模式存在一个很严重的问题: 就是当系统需要引入 ...
- samba搭建与配置说明
1. 环境检查getenforce 检查selinux service iptables stop 2.安装samba yum -y install samba 3.配置samba /etc/samb ...
- spark 源码分析之十六 -- Spark内存存储剖析
上篇spark 源码分析之十五 -- Spark内存管理剖析 讲解了Spark的内存管理机制,主要是MemoryManager的内容.跟Spark的内存管理机制最密切相关的就是内存存储,本篇文章主要介 ...
- py+selenium 明明定位不到元素,但却不报错或是报错AttributeError: 'list' object has no attribute 'click'【已解决】
问题:定位不到元素,但却不报错或者出现报错AttributeError: 'list' object has no attribute 'click' 如图 或者 解决方法: 将”driver ...
- Cocos2d-x 3.x 学习笔记(三):Scheduler Timer 调度与定时
1. 概述 Cocos2d-x 的 Scheduler 离不开 Timer.Timer 类是定时器,用来规定一个回调函数应该在何时被触发.Timer 封装了已运行时间.重复次数.已执行次数.延迟秒数 ...
- Linux 文件编程、时间编程基本函数
文件编程 文件描述符 fd --->>>数字(文件的身份证,代表文件身份),通过 fd 可找到正在操作或需要打开的文件. 基本函数操作: 1)打开/创建文件 int open (co ...
- 通过代数,数字,欧几里得平面和分形讨论JavaScript中的函数式编程
本文是对函数式编程范式的系列文章从而拉开了与以下延续一个. 介绍 在JavaScript中,函数只是对象.因此,可以构造函数,作为参数传递,从函数返回或分配给变量.因此,JavaScript具有一流的 ...
- 洛谷P2790 ccj与zrz之积木问题 题解
题目链接:https://www.luogu.org/problemnew/show/P2790 这题码量稍有点大... 分析: 这道题模拟即可.因为考虑到所有的操作vector可最快捷的实现,所以数 ...