zookeeper barrier和queue应用实例
zookeeper工具类：
获取连接实例；创建节点；获取子节点；设置节点数据；获取节点数据；访问控制等。
package org.windwant.zookeeper; import org.apache.zookeeper.*;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.List; /**
* zookeeper util
*/
public class ZookeeperUtil {
private static final int SESSION_TIMEOUT = 30000; /**
* 使用连接串创建连接
* @param domain
* @param w
* @return
*/
public static ZooKeeper getInstance(String domain, Watcher w){
try {
return new ZooKeeper(domain,SESSION_TIMEOUT, w);
} catch (IOException e) {
e.printStackTrace();
}
return null;
} public static String createNode(ZooKeeper zk, String path, byte[] data){
try {
return zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
} public static List<String> getChildrenNode(ZooKeeper zk, String path){
try {
return zk.getChildren(path, false);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
} public static Stat setNodeData(ZooKeeper zk, String path, byte[] data, int version){
try {
return zk.setData(path, data, version);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
} public static byte[] getNodeData(ZooKeeper zk, String path){
try {
return zk.getData(path, false, null);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
} public static void deleteNode(ZooKeeper zk, String path, int version){
try {
zk.delete(path, version);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (KeeperException e) {
e.printStackTrace();
}
} public static void closeZk(ZooKeeper zk){
try {
if(zk != null) {
zk.close();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
} public static void addAuth(ZooKeeper zk, String userName, String passwd){
try {
zk.addAuthInfo(String.valueOf(Ids.AUTH_IDS), DigestAuthenticationProvider.generateDigest(userName + ":" + passwd).getBytes());
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
} public static void main(String[] args) {
// try {
// ZooKeeper zk = new ZooKeeper("localhost", 2181, null);
// addAuth(zk, "roger", "123456");
// } catch (IOException e) {
// e.printStackTrace();
// }
try {
System.out.println(DigestAuthenticationProvider.generateDigest("roger:123456"));
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
}
}
继承父类：
SyncPrimitive 负责zookeeper连接及根节点的初始化。 实现zookeeper的Watcher
package org.windwant.zookeeper; import org.apache.zookeeper.*; import java.io.IOException;
import java.util.ArrayList;
import java.util.List; public class SyncPrimitive implements Watcher { protected ZooKeeper zk = null;
protected Integer mutex; SyncPrimitive(Integer mutex) {
this.mutex = mutex;
} /**
* 初始化zookeeper
* @param domain
*/
protected void initZK(String domain){
System.out.println(Thread.currentThread().getName() + ": init zookeeper...");
try {
zk = new ZooKeeper(domain, 30000, this);
System.out.println(Thread.currentThread().getName() + ": zookeeper connected " + zk);
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 初始化应用根节点 并发处理
* @param root
*/
protected void initZKRootNode(String root){
//并发控制
synchronized (mutex) {
try {
if (zk != null) {
if (zk.exists(root, false) != null) {
List<String> child = zk.getChildren(root, false);
if (child != null && !child.isEmpty()) {
//zookeeper multi操作；或者 Transaction（multi封装） commit操作；
List<Op> ops = new ArrayList<>();
child.forEach(c -> {
ops.add(Op.delete(root + "/" + c, -1));
});
List<OpResult> opRsts = zk.multi(ops);
System.out.println(Thread.currentThread().getName() + ": deleted child node success!");
}
zk.setData(root, String.valueOf(0).getBytes(), -1);
System.out.println(Thread.currentThread().getName() + ": app root node " + root + " init success! ");
} else {
zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
System.out.println(Thread.currentThread().getName() + ": app root node " + root + " create success! ");
}
}
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} public void process(WatchedEvent event) {
if(event.getState().equals(Event.KeeperState.SyncConnected)) {
}
}
}
zookeeper 屏障 Barrier：
SyncPrimitiveBarrier
enter()：加入屏障队列
leave()：离开屏障队列
package org.windwant.zookeeper; import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat; import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom; public class SyncPrimitiveBarrier extends SyncPrimitive { private String root;
//屏障阈值
private int size;
private String name; /**
* Barrier constructor
*
* @param domain
* @param root
* @param size
*/
public SyncPrimitiveBarrier(String domain, String root, Integer size) {
super(size);
this.root = root;
this.size = size; initZK(domain);
initZKRootNode(root); // My node name
try {
name = new String(InetAddress.getLocalHost().getCanonicalHostName().toString());
} catch (UnknownHostException e) {
System.out.println(e.toString());
} } /**
* Join barrier
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/ boolean enter() throws KeeperException, InterruptedException{
synchronized (mutex) {
List<String> list = zk.getChildren(root, true);
//当前节点数小于阈值，则创建节点，进入barrier
if (list.size() < size) {
System.out.println("node: " + list.size());
this.name = zk.create(root + "/" + name, new byte[0], Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
if (list.size() + 1 == size) {
System.out.println("set data node size" + list.size());
zk.setData(root, String.valueOf(list.size() + 1).getBytes(), -1);
}
System.out.println(Thread.currentThread().getName() + ": " + name + " enter barrier!");
return true;
} else {
//否则不进入
System.out.println(Thread.currentThread().getName() + ": " + name + " barrier full, inaccessible!");
return false;
}
}
} /**
* Wait until all reach barrier
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/ boolean leave() throws KeeperException, InterruptedException, UnsupportedEncodingException {
while (true) {
int data = Integer.parseInt(new String(zk.getData(root, false, new Stat()), "UTF-8"));
if (data == size) {
System.out.println("leave size: " + data);
//离开
zk.delete(name, -1);
System.out.println(Thread.currentThread().getName() + ": " + name + " left barrier!");
return true;
} else {
System.out.println(Thread.currentThread().getName() + ": " + name + " waitting for others!");
Thread.sleep(1000);//每秒检查一次
}
}
} public static void main(String[] args) {
for (int i = 0; i < 3; i++) {
new Thread(() -> {
SyncPrimitiveBarrier syncPrimitiveBarrier = new SyncPrimitiveBarrier("localhost:2181", "/barrier_test", 3);
boolean flag = false;
try {
//模拟需要到达barrier的时间
Thread.sleep(ThreadLocalRandom.current().nextInt(1,5)*1000);
flag = syncPrimitiveBarrier.enter(); //尝试离开barrier
syncPrimitiveBarrier.leave();
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}).start(); }
}
}
zookeeper 消息队列：
SyncPrimitiveQueue
produce(int i)：生成消息放入队列
consume()：消费队列消息
package org.windwant.zookeeper; import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat; import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom; /**
* Producer-Consumer queue
*/
public class SyncPrimitiveQueue extends SyncPrimitive { private String root;
private int queueSize; /**
* Constructor of producer-consumer queue
*
* @param domain
* @param name
*/ SyncPrimitiveQueue(String domain, String name, Integer queueSize) {
super(queueSize);
this.root = name;
this.queueSize = queueSize;
initZK(domain);
initZKRootNode(root);
} /**
* Add element to the queue.
*
* @param i
* @return
*/ public boolean produce(int i) throws KeeperException, InterruptedException{
synchronized (mutex) {
List<String> children = zk.getChildren(root, false);
if(children != null && children.size()>=mutex){
System.out.println(Thread.currentThread().getName() + ": producer queue full, waiting for consuming");
mutex.wait();
}
ByteBuffer b = ByteBuffer.allocate(4);
byte[] value; b.putInt(i);
value = b.array();
String node = zk.create(root + "/element", value, Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT_SEQUENTIAL);
System.out.println(Thread.currentThread().getName() + ": produce value: " + node);
mutex.notifyAll();//通知消费
return true;
}
} /**
* Remove first element from the queue.
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/
int consume() throws KeeperException, InterruptedException{
int retvalue = -1;
Stat stat = null; // Get the first element available
while (true) {
synchronized (mutex) {
List<String> list = zk.getChildren(root, true);
if (list.size() == 0) {
System.out.println(Thread.currentThread().getName() + ": resource not awailable, waitting for produce!");
mutex.wait();
} else {
list.sort((String s1, String s2) -> s1.compareTo(s2)); //消费序号最小的节点
String dest = list.get(0);
System.out.println(Thread.currentThread().getName() + ": cosume value: " + root + "/" + dest);
byte[] b = zk.getData(root + "/" + dest,
false, stat);
zk.delete(root + "/" + dest, 0); //消费后删除
ByteBuffer buffer = ByteBuffer.wrap(b);
retvalue = buffer.getInt();
mutex.notifyAll();
return retvalue;
}
}
}
} public static void main(String[] args) {
SyncPrimitiveQueue syncPrimitiveQueue = new SyncPrimitiveQueue("localhost:2181", "/queue_test", 10);
//生产 每隔三秒 模拟慢生产
new Thread(() -> {
for (int i = 0; i < Integer.MAX_VALUE; i++) {
try {
syncPrimitiveQueue.produce(i);
Thread.sleep(ThreadLocalRandom.current().nextInt(0, 3)*1000);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}).start(); try {
Thread.sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
} //消费 每隔一秒 模拟快消费
new Thread(() -> {
for (int i = 0; i < Integer.MAX_VALUE ; i++) {
try {
syncPrimitiveQueue.consume();
Thread.sleep(ThreadLocalRandom.current().nextInt(0, 3)*1000);
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}).start();
}
}
zookeeper分布式锁：
SynZookeeperLock
getInstance(String domain)：获取zookeeper实例
tryLock(String domain, String path, byte[] data, CountDownLatch c)：尝试获取分布式锁。
package org.windwant.zookeeper; import org.apache.zookeeper.*;
import org.apache.zookeeper.ZooDefs.Ids; import java.io.IOException;
import java.util.concurrent.*; /**
* zookeeper 分布式锁
*/
public class SynZookeeperLock {
private static final int SESSION_TIMEOUT = 30000; public static ZooKeeper getInstance(String domain){
try {
CountDownLatch c = new CountDownLatch(1);
ZooKeeper zk = new ZooKeeper(domain, SESSION_TIMEOUT, new Watcher() {
@Override
public void process(WatchedEvent event) {
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) {
c.countDown(); // 唤醒当前正在执行的线程
}
}
});
//阻塞直到连接完成
c.await();
return zk;
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
}
/**
* 获取分布式锁
* 使用临时节点，避免进程获取锁后，down掉未释放锁问题
* @param domain
* @param path
* @param data
* @param c
*/
public static void tryLock(String domain, String path, byte[] data, CountDownLatch c){
//每次获取锁使用新的会话连接
ZooKeeper zk = getInstance(domain);
zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, (rc, path1, ctx, name) -> {
//节点创建成功，获取锁
if (rc == 0) {
System.out.println(Thread.currentThread().getName() + "：result " + rc + " lock " + path + ", created!");
try {
//模拟获取锁后3秒释放
Thread.sleep(3000);
System.out.println(Thread.currentThread().getName() + "：task complete，try release lock！");
zk.delete(path, -1, (rc1, path2, ctx1) -> {
if(rc1 == 0){
System.out.println(Thread.currentThread().getName() + "：lock released！");
}
}, null); } catch (InterruptedException e) {
e.printStackTrace();
}finally {
//释放等待
c.countDown();
}
} else if(rc == -110) {//节点已存在，则说明锁已被其它进程获取，则创建watch，并阻塞等待
System.out.println(Thread.currentThread().getName() + "：result " + rc + " lock " + path + " already created, waiting!");
try {
zk.exists(path, event -> {
//watch 到锁删除事件，则触发重新获取锁
if (event.getType().equals(Watcher.Event.EventType.NodeDeleted)) {
System.out.println(Thread.currentThread().getName() + "：get node deleted event! try lock!");
//释放连接，避免服务器因为连接数限制
try {
zk.close();
} catch (InterruptedException e) {
e.printStackTrace();
}
SynZookeeperLock.tryLock(domain, path, data, c);
c.countDown();
}
});
} catch (KeeperException e) {
//包括ConnectionLossException（网络，服务器故障） 需要确认客户端重连执行情况 之前的请求是否需要重新执行
e.printStackTrace();
c.countDown();
} catch (InterruptedException e) {
//线程中断，打断请求
e.printStackTrace();
c.countDown();
}
}else {
//-4 -112
System.out.println(Thread.currentThread().getName() + ": connection lose or session invalid");
c.countDown();
// tryLock(domain, path, data, c);
}
}, new Object());
try {
//阻塞等待结果
c.await();
} catch (InterruptedException e) {
e.printStackTrace();
}finally {
c.countDown();
}
} public static void main(String[] args) {
String lockPath = "/testlock";
byte[] lock = "lock".getBytes();
String domain = "127.0.0.1:2181";
//测试获取锁线程 注意服务器最大连接数限制
for (int i = 0; i < 20; i++) {
Thread tmp = new Thread( () -> tryLock(domain, lockPath, lock, new CountDownLatch(1)));
tmp.start();
}
}
}
项目地址：https://github.com/windwant/windwant-demo/tree/master/zookeeper-service
zookeeper barrier和queue应用实例的更多相关文章
- ZooKeeper系列(9)：ZooKeeper实现分布式Barrier和Queue
1. 快速开始 1.1概述: Zookeeper是Hadoop的一个子项目,它是分布式系统中的协调系统,可提供的服务主要有:配置服务.名字服务.分布式同步.组服务等. 1.2 使用常见 1.2.1 统 ...
- 初始zookeeper与集群搭建实例
zookeeper是什么 Zookeeper,一种分布式应用的协作服务,是Google的Chubby一个开源的实现,是Hadoop的分布式协调服务,它包含一个简单的原语集,应用于分布式应用的协作服务, ...
- 初识ZooKeeper与集群搭建实例
原文链接:http://www.linuxidc.com/Linux/2015-02/114230.htm zookeeper是什么 Zookeeper,一种分布式应用的协作服务,是Google的Ch ...
- Zookeeper：fsync超时导致实例异常
一.问题描述 2019-02-19 08:44左右,实时计算服务重启,报错显示找不到zk集群的leader节点,同时ZooKeeper集群有告警显示连接超时: 指标[连接耗时(ms)=18221]符合 ...
- spring整合activemq发送MQ消息[queue模式]实例
queue类型消息 pom依赖 <dependency> <groupId>junit</groupId> <artifactId>junit</ ...
- Delphi下MSMQ（Mircosoft Message Queue）实例（私有队列）
网上关于消息队列技术原理说明的详细文档很多,但涉及到Delphi的具体实现很少,这是我从网上找了一上午的资料,自己整合和尝试的能运行的程序. 打开控制面板->程序->添加组件,添加消息队列 ...
- ZooKeeper实现配置中心的实例（原生API实现）（转）
说明:要实现配置中心的例子,可以选择的SDK有很多,原生自带的SDK也是不错的选择.比如使用I0Itec,Spring Boot集成等. 大型应用通常会按业务拆分成一个个业务子系统,这些大大小小的子应 ...
- Zookeeper应用之——栅栏（barrier）
Zookeeper应用之——栅栏(barrier) 栅栏(barrier)简介 barrier的作用是所有的线程等待,知道某一时刻,锁释放,所有的线程同时执行.举一个生动的例子,比如跑步比赛,所有 运 ...
- ZooKeeper实现同步屏障(Barrier)
按照维基百科的解释:同步屏障(Barrier)是并行计算中的一种同步方法.对于一群进程或线程,程序中的一个同步屏障意味着任何线程/进程执行到此后必须等待,直到所有线程/进程都到达此点才可继续执行下文. ...
随机推荐
- 【JSP手记】--jsp里面session.getAttribute("×××")在java中的表示
JSP里面的 <%=session.getAttribute("×××")%> 与java等价于 request.getSession().get ...
- [小北De编程手记] : Lesson 03 玩转 xUnit.Net 之 Fixture（上）
在使用xUnit.Net Framework构建单元测试或自动化测试项目的时候,无论是针对一些比较耗费资源的对象亦或是为了支持Test case预设数据的能力,我们都需要有一些初始化或是清理相关的动作 ...
- Java实现多线程并发
import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util ...
- @media自适应宽度
<!DOCTYPE HTML> <html> <head> <meta charset="utf-8"> <meta name ...
- 通过 imagick 让 PHP 生成 PSD 文件缩略图
第一步.安装ImageMagick 首先需要安装 ImageMagick ,因为 imagick 是一个可以供 PHP 调用 ImageMagick 功能的PHP扩展.(目前最新版本是:ImageMa ...
- jQuery Label Better – 友好的表单输入框提示插件
jQuery Label Better 帮助你标记您的表单输入域,带有美丽的动画效果而且不占用空间.这个插件的独特之处在于所有你需要做的就是添加一个占位符文本,只有当用户需要它的时候才显示标签. 您可 ...
- HTML5 表单新增属性
1. 表单内元素的form属性 在H5中可以把form放到页面的任何地方,然后为该元素指定一个form属性,属性值为该表单的id,这样就可以声明该元素从属于指定表单了 <form id=&quo ...
- 【web前端面试题整理04】阿里一行之大神面对面
前言 这段时间我在河南一家公司当了一段时间的前端主管,最后可耻的匿了,原因各种各样,最主要的就是不想呆在郑州了. 其实这里的同事还是很不错的,面对老总最后的挽留我不是没有动心,而是这个地方确实不太好, ...
- JS常用的设计模式
单例模式 只创建类的唯一一个实例.我们看了好几种可以不通过构造函数和类Java语法达成单例的方法.从另一方面来说,JavaScript中所有的对象都是单例.有时候开发者说的单例是指通过模块化模式创建的 ...
- 好文要顶之 --- 简单粗暴地理解 JavaScript 原型链
原型链理解起来有点绕了,网上资料也是很多,每次晚上睡不着的时候总喜欢在网上找点原型链和闭包的文章看,效果极好. 不要纠结于那一堆术语了,那除了让你脑筋拧成麻花,真的不能帮你什么.简单粗暴点看原型链吧, ...