P3
package p3.hadoop.mapred; import java.io.IOException;
import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import p3.common.lib.BinaryUtils;
import p3.common.lib.Bytes; public class PcapLineReader
{
private static final int DEFAULT_BUFFER_SIZE = 2048;
private int bufferSize;
private static final int PCAP_FILE_HEADER_LENGTH = 24;
private static final int PCAP_PACKET_HEADER_LENGTH = 16;
private static final int PCAP_PACKET_HEADER_CAPLEN_POS = 8;
private static final int PCAP_PACKET_HEADER_WIREDLEN_POS = 12;
private static final int PCAP_PACKET_HEADER_CAPLEN_LEN = 4;
private static final int PCAP_PACKET_HEADER_TIMESTAMP_LEN = 4;
private static final int PCAP_PACKET_MIN_LEN = 53;
private static final int PCAP_PACKET_MAX_LEN = 1519;
private static final int MAGIC_NUMBER = -725372255;
private static final int MIN_PKT_SIZE = 42;
private long min_captime;
private long max_captime;
private InputStream in;
private byte[] buffer;
byte[] pcap_header;
private int bufferLength;
int consumed; public PcapLineReader(InputStream in, int bufferSize, long min_captime, long max_captime)
{
this.bufferSize = 2048; this.bufferLength = 0;
this.consumed = 0; this.in = in;
this.bufferSize = bufferSize;
this.buffer = new byte[this.bufferSize];
this.min_captime = min_captime;
this.max_captime = max_captime;
} public PcapLineReader(InputStream in, Configuration conf)
throws IOException
{
this(in, 2048,
conf.getLong("pcap.file.captime.min", 1309412600L),
conf.getLong("pcap.file.captime.max", conf.getLong("pcap.file.captime.max", 1309412600L) + 172800L));
} public void close()
throws IOException
{
this.in.close();
} int skipPartialRecord(int fraction)
throws IOException
{
int pos = 0;
byte[] captured = new byte[fraction];
byte[] tmpTimestamp1 = new byte[4];
byte[] tmpTimestamp2 = new byte[4];
byte[] tmpCapturedLen1 = new byte[4];
byte[] tmpWiredLen1 = new byte[4];
byte[] tmpCapturedLen2 = new byte[4];
byte[] tmpWiredLen2 = new byte[4];
int caplen1 = 0;
int wiredlen1 = 0;
int caplen2 = 0;
int wiredlen2 = 0;
long timestamp2 = 0L; int size = 0;
long endureTime = 100L; if ((size = this.in.read(captured)) < 42) return 0; do
{
if ((size - pos < 32) || (size - pos < 53)) {
pos = size;
break;
} System.arraycopy(captured, pos, tmpTimestamp1, 0, 4);
long timestamp1 = Bytes.toLong(BinaryUtils.flipBO(tmpTimestamp1, 4)); System.arraycopy(captured, pos + 8, tmpCapturedLen1, 0, 4);
caplen1 = Bytes.toInt(BinaryUtils.flipBO(tmpCapturedLen1, 4)); System.arraycopy(captured, pos + 12, tmpWiredLen1, 0, 4);
wiredlen1 = Bytes.toInt(BinaryUtils.flipBO(tmpWiredLen1, 4)); if ((caplen1 > 53) && (caplen1 < 1519) && (size - pos - 32 - caplen1 > 0))
{
System.arraycopy(captured, pos + 16 + caplen1 + 8, tmpCapturedLen2, 0, 4);
caplen2 = Bytes.toInt(BinaryUtils.flipBO(tmpCapturedLen2, 4)); System.arraycopy(captured, pos + 16 + caplen1 + 12, tmpWiredLen2, 0, 4);
wiredlen2 = Bytes.toInt(BinaryUtils.flipBO(tmpWiredLen2, 4)); System.arraycopy(captured, pos + 16 + caplen1, tmpTimestamp2, 0, 4);
timestamp2 = Bytes.toLong(BinaryUtils.flipBO(tmpTimestamp2, 4)); if ((timestamp1 >= this.min_captime) && (timestamp1 < this.max_captime) && (this.min_captime <= timestamp2) && (timestamp2 < this.max_captime) &&
(wiredlen1 > 53) && (wiredlen1 < 1519) && (wiredlen2 > 53) && (wiredlen2 < 1519) &&
(caplen1 > 0) && (caplen1 <= wiredlen1) && (caplen2 > 0) && (caplen2 <= wiredlen2) &&
(timestamp2 >= timestamp1) && (timestamp2 - timestamp1 < endureTime)) {
return pos;
} } ++pos;
}
while (pos < size); return pos;
} int readPacket(int packetLen)
throws IOException
{
int bufferPosn = 16;
byte[] tmp_buffer = new byte[packetLen]; if ((this.bufferLength = this.in.read(tmp_buffer)) < packetLen) {
System.arraycopy(tmp_buffer, 0, this.buffer, bufferPosn, this.bufferLength);
bufferPosn += this.bufferLength; byte[] newpacket = new byte[packetLen - this.bufferLength]; if ((this.bufferLength = this.in.read(newpacket)) < 0) return bufferPosn;
System.arraycopy(newpacket, 0, this.buffer, bufferPosn, this.bufferLength);
}
else
{
System.arraycopy(tmp_buffer, 0, this.buffer, bufferPosn, this.bufferLength);
}
bufferPosn += this.bufferLength; return bufferPosn;
} int readPacketHeader()
{
int headerLength = 0;
int headerPosn = 0;
this.pcap_header = new byte[16]; byte[] tmp_header = new byte[16];
BytesWritable capturedLen = new BytesWritable();
try
{
if ((headerLength = this.in.read(this.pcap_header)) < 16)
{
if (headerLength == -1) return 0;
headerPosn += headerLength; byte[] newheader = new byte[16 - headerLength]; if ((headerLength = this.in.read(newheader)) < 0) {
this.consumed = headerPosn;
return -1;
}
System.arraycopy(newheader, 0, this.pcap_header, headerPosn, headerLength);
}
capturedLen.set(this.pcap_header, 8, 4);
System.arraycopy(this.pcap_header, 0, this.buffer, 0, 16);
headerPosn = 0;
}
catch (IOException e)
{
e.printStackTrace();
}
return Bytes.toInt(BinaryUtils.flipBO(capturedLen.getBytes(), 4));
} public int readFileHeader()
{
try {
byte[] magic = new byte[4];
this.bufferLength = this.in.read(this.buffer, 0, 24);
System.arraycopy(this.buffer, 0, magic, 0, magic.length); if (Bytes.toInt(magic) == -725372255) break label50;
return 0;
}
catch (IOException e) {
e.printStackTrace();
}
label50: return this.bufferLength;
} public int readLine(BytesWritable bytes, int maxLineLength, int maxBytesToConsume)
throws IOException
{
bytes.set(new BytesWritable());
boolean hitEndOfFile = false;
long bytesConsumed = 0L; int caplen = readPacketHeader(); if (caplen == 0) {
bytesConsumed = 0L;
} else if (caplen == -1) {
bytesConsumed += this.consumed;
}
else if ((caplen > 0) && (caplen < 1519)) {
if ((this.bufferLength = readPacket(caplen)) < caplen + 16) {
hitEndOfFile = true;
}
bytesConsumed += this.bufferLength; if (!(hitEndOfFile)) {
bytes.set(this.buffer, 0, caplen + 16);
}
} return (int)Math.min(bytesConsumed, 2147483647L);
} public int readLine(BytesWritable str, int maxLineLength)
throws IOException
{
return readLine(str, maxLineLength, 2147483647);
} public int readLine(BytesWritable str)
throws IOException
{
return readLine(str, 2147483647, 2147483647);
}
}
// Decompiled by Jad v1.5.8e2. Copyright 2001 Pavel Kouznetsov.
// Jad home page: http://kpdus.tripod.com/jad.html
// Decompiler options: packimports(3) fieldsfirst ansi space
// Source File Name: PcapVlenRecordReader.java package p3.hadoop.mapred; import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader; // Referenced classes of package p3.hadoop.mapred:
// PcapLineReader public class PcapVlenRecordReader
implements RecordReader
{ private CompressionCodecFactory compressionCodecs;
private long start;
private long pos;
private long end;
private PcapLineReader in;
int maxLineLength;
private boolean fileheader_skip; public PcapVlenRecordReader(Configuration job, FileSplit split)
throws IOException
{
compressionCodecs = null;
fileheader_skip = true;
maxLineLength = job.getInt("mapred.linerecordreader.maxlength", 0x7fffffff);
fileheader_skip = job.getBoolean("pcap.file.header.skip", true);
start = split.getStart();
end = start + split.getLength();
Path file = split.getPath();
compressionCodecs = new CompressionCodecFactory(job);
CompressionCodec codec = compressionCodecs.getCodec(file);
FileSystem fs = file.getFileSystem(job);
FSDataInputStream fileIn = fs.open(split.getPath());
boolean skipFileHeader = false;
boolean skipPartialRecord = false;
int fraction = 4000;
if (codec != null)
{
in = new PcapLineReader(codec.createInputStream(fileIn), job);
end = 0x7fffffffffffffffL;
skipFileHeader = true;
} else
{
if (start == 0L)
{
skipFileHeader = true;
} else
{
skipPartialRecord = true;
fileIn.seek(start);
}
in = new PcapLineReader(fileIn, job);
}
if (skipFileHeader)
start += in.readFileHeader();
if (skipPartialRecord)
{
int skip;
for (skip = in.skipPartialRecord(fraction); skip == fraction; skip = in.skipPartialRecord(fraction))
start += skip; start += skip;
fileIn.seek(start);
in = new PcapLineReader(fileIn, job);
}
pos = start;
} public LongWritable createKey()
{
return new LongWritable();
} public BytesWritable createValue()
{
return new BytesWritable();
} public synchronized boolean next(LongWritable key, BytesWritable value)
throws IOException
{
while (pos < end)
{
key.set(pos);
int newSize = in.readLine(value, maxLineLength, Math.max((int)Math.min(0x7fffffffL, end - pos), maxLineLength));
if (newSize == 0)
{
pos = end;
return false;
}
pos += newSize;
if (newSize < maxLineLength)
return true;
}
return false;
} public float getProgress()
{
if (start == end)
return 0.0F;
else
return Math.min(1.0F, (float)(pos - start) / (float)(end - start));
} public synchronized long getPos()
throws IOException
{
return pos;
} public synchronized void close()
throws IOException
{
if (in != null)
in.close();
} public volatile boolean next(Object obj, Object obj1)
throws IOException
{
return next((LongWritable)obj, (BytesWritable)obj1);
} public volatile Object createValue()
{
return createValue();
} public volatile Object createKey()
{
return createKey();
}
}
P3的更多相关文章
- ERROR ITMS-90682: Invalid Bundle - The asset catalog at 'Payload/XXXXX/Assets.car' can't contain 16-bit or P3 assets if the app supports iOS 9.3 or earlier.
刚升级Xcode 8, 幺蛾子又出现了.提交的时候出了这个问题. BTW,感谢google.以下为解决方案:‘ 在 Xcode 8 中,当你资源文件中[含有16位图]或者[图片显示模式γ值为'P3'] ...
- 单片微机原理P3:80C51外部拓展系统
外部拓展其实是个相对来说很好玩的章节,可以真正开始用单片机写程序了,比较重要的是外部存储器拓展,81C55拓展,矩阵键盘,动态显示,DAC和ADC. 0. IO接口电路概念与存储器拓展 1. 为什 ...
- enq: TX - row lock contention 参数P1,P2,P3说明
enq: TX - row lock contention三个参数,例如,下面的等待事件 * P1 = name|mode <<<<<<< ...
- 在Cenos系统安装Python3.5版本,使P2和P3共存
首先Cenos安装好后,系统自带python2.6版本 输入>>>exit() 退出 使用迅雷下载python3.5 链接:https://www.python.org/ft ...
- 延时、输入输出接口P0~P3
1.寄存器 为了知道延时程序是如何工作的,我们必需首先了解延时程序中出现的一些符号,就从R1开始,R1被称之为工作寄存器.什么是工作寄存器呢?让我们从现实生活中来找找答案.如果出一道数学题:123+5 ...
- 等待事件P1 P2 P3含义
从以下两个视图中查到的session中,有P1,P2,P3参数select * from v$sessionselect * from v$session_waitselect * from v$se ...
- 等待事件对应的p1,p2,p3含义
Oracle 10g v$session视图中不同等待事件对应的p1,p2,p3的含义也不同,我们不可能记住所有等待事件对应的p1,p2,p3的含义. 可以通过查询V$EVENT_NAME知道每个等待 ...
- [raspberry p3] [suse] 安装maven
[raspberry p3] [suse] 安装maven 配置package repositroy, 添加devel:tools:building vim /etc/zypp/repos.d/ope ...
- iOS检测项目图片资源是否包含P3图片
1.问题描述 我们需要知道的是在iOS9.3以下系统上,.ipa包内如果含有p3图片,将会导致严重的闪退问题,具体原因还请google,非本文的重点. 2.问题解决 拿到的如果是ipa包(不是则跳过) ...
随机推荐
- android软键盘的显示和隐藏
显示: InputMethodManager imm = (InputMethodManager) getSystemService(INPUT_METHOD_SERVICE); imm.toggle ...
- CSS常用字体名称
CSS样式中常用的字体名称 css中引入字体: @font-face { font-family: "AncientWar"; src: url('style/css/font ...
- linux僵尸进程
什么是僵尸进程? 在UNIX 系统中,一个进程结束了,但是他的父进程没有等待(调用wait / waitpid)他, 那么他将变成一个僵尸进程. 在fork()/execve()过程中,假设子 ...
- 浅谈计算机中的IO模型
IO模型一共有5种: blocking IO #阻塞IO nonblocking IO #非阻塞IO IO myltiplexing #IO多路复用 signal driven IO #信号驱动IO ...
- 玩转spring boot——简单登录认证
前言 在一个web项目中,某些页面是可以匿名访问的,但有些页面则不能.spring mvc提供了HandlerInterceptor接口来应对,只需要重写preHandle方法便可以实现此功能.那么使 ...
- BZOJ 2142: 礼物 [Lucas定理]
2142: 礼物 Time Limit: 10 Sec Memory Limit: 259 MBSubmit: 1294 Solved: 534[Submit][Status][Discuss] ...
- vue2.0之render函数
虽然vue推荐用template来创建你的html,但是在某些时候你也会用到render函数. 虚拟DOM Vue 通过建立一个虚拟 DOM 对真实 DOM 发生的变化保持追踪.请近距离看一下这行代码 ...
- 在ASP.NET MVC中使用Web API和EntityFramework构建应用程序
最近做了一个项目技术预研:在ASP.NET MVC框架中使用Web API和EntityFramework,构建一个基础的架构,并在此基础上实现基本的CRUD应用. 以下是详细的步骤. 第一步 在数据 ...
- Ubuntu的Java环境变量
新架构要上线了,这两天开始准备分析一下了,今天是直接进到JAVA_HOME的lib目录执行的java -cp sa-jdi.jar sun.jvm.hotspot.HSDB,然后报了个错: 这是哪来的 ...
- Redis进阶实践之十一 Redis的Cluster集群搭建
一.引言 本文档只对Redis的Cluster集群做简单的介绍,并没有对分布式系统的详细概念做深入的探讨.本文只是提供了有关如何设置集群.测试和操作集群的说明,而不涉及Redis集群规范中涵 ...