cat /etc/ecm/hadoop-conf/fair-scheduler.xml

<?xml version="1.0"?>

<allocations>

<aclSubmitApps>*</aclSubmitApps>

<weight>2</weight>

<minResources>10000 mb, 10vcores</minResources>

<maxChildResources>34000 mb,24 vcores</maxChildResources>

<maxRunningApps>50</maxRunningApps>

<maxAMShare>1</maxAMShare>

<maxResources>400000 mb, 200vcores</maxResources> #限制队列最大使用资源

<aclAdministerApps>*</aclAdministerApps>

<schedulingPolicy>fair</schedulingPolicy>

<queue name="default">

<aclSubmitApps>*</aclSubmitApps>

<minResources>10000 mb, 10vcores</minResources>

<aclAdministerApps>*</aclAdministerApps>

<weight>1</weight>

<maxRunningApps>10</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<maxResources>200000 mb, 100vcores</maxResources>

</queue>

<queue name="collects">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<maxAMShare>0.8</maxAMShare>

<minResources>50 mb, 2vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>50</maxRunningApps>

</queue>

<queue name="data_bi">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>4</weight>

<minResources>100 mb, 1vcores</minResources>

<maxResources>30000 mb, 50vcores</maxResources>

<maxRunningApps>5</maxRunningApps>

</queue>

<queue name="opay_collects">

<weight>20</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>20</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="opos_collects">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>80000 mb, 50vcores</maxResources>

<maxRunningApps>10</maxRunningApps>

</queue>

<queue name="users" type="parent">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>10000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="airflow">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<minResources>10 mb, 2vcores</minResources>

<maxResources>200000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

</queue>

<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>

<userMaxAppsDefault>50</userMaxAppsDefault>

<queueMaxAppsDefault>50</queueMaxAppsDefault>

<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>

<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>

<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>

<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>

<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>

</allocations>

#新的xml, 不带root限制: 放emr的yarn-配置-fair-scheduler

<?xml version="1.0" encoding="utf-8"?>
<allocations>
<queue name="root">
<queue name="default">
<aclSubmitApps>*</aclSubmitApps>
<minResources>10000 mb, 10vcores</minResources>
<aclAdministerApps>*</aclAdministerApps>
<weight>1</weight>
<maxRunningApps>10</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<maxResources>200000 mb, 100vcores</maxResources>
</queue>
<queue name="collects">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<maxAMShare>0.8</maxAMShare>
<minResources>50 mb, 2vcores</minResources>
<maxResources>400000 mb, 200vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
</queue>
<queue name="data_bi">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>4</weight>
<minResources>100 mb, 1vcores</minResources>
<maxResources>30000 mb, 50vcores</maxResources>
<maxRunningApps>5</maxRunningApps>
</queue>
<queue name="opay_collects">
<weight>20</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>400000 mb, 200vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="opos_collects">
<weight>5</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>80000 mb, 50vcores</maxResources>
<maxRunningApps>10</maxRunningApps>
</queue>
<queue name="users" type="parent">
<weight>5</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>10000 mb, 150vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="airflow">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<minResources>10 mb, 2vcores</minResources>
<maxResources>200000 mb, 150vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
</queue>
</queue>
<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
<userMaxAppsDefault>50</userMaxAppsDefault>
<queueMaxAppsDefault>50</queueMaxAppsDefault>
<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>
<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>
<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>
<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>
</allocations>

EMR的fair-scheduler.xml的更多相关文章

  1. Fair Scheduler 队列设置经验总结

    Fair Scheduler 队列设置经验总结 由于公司的hadoop集群的计算资源不是很充足,需要开启yarn资源队列的资源抢占.在使用过程中,才明白资源抢占的一些特点.在这里总结一下. 只有一个队 ...

  2. 三：Fair Scheduler 公平调度器

    参考资料: http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html http://h ...

  3. Fair Scheduler中的Delay Schedule分析

    延迟调度的主要目的是提高数据本地性(data locality),减少数据在网络中的传输.对于那些输入数据不在本地的MapTask,调度器将会延迟调度他们,而把slot分配给那些具备本地性的MapTa ...

  4. Hadoop学习之--Fair Scheduler作业调度分析

    Fair Scheduler调度器同步心跳分配任务的过程简单来讲会经历以下环节: 1. 对map/reduce是否已经达到资源上限的循环判断 2. 对pool队列根据Fair算法排序 3.然后循环po ...

  5. YARN的Fair Scheduler和Capacity Scheduler

    关于Scheduler YARN有四种调度机制:Fair Schedule,Capacity Schedule,FIFO以及Priority: 其中Fair Scheduler是资源池机制,进入到里面 ...

  6. Hadoop的三种调度器FIFO、Capacity Scheduler、Fair Scheduler（转载）

    目前Hadoop有三种比较流行的资源调度器:FIFO .Capacity Scheduler.Fair Scheduler.目前Hadoop2.7默认使用的是Capacity Scheduler容量调 ...

  7. fair scheduler配置

    <property>    <name>yarn.resourcemanager.scheduler.class</name>    <value>or ...

  8. Yarn参数优化(Fair Scheduler版本)

    YARN 自从hadoop2.0之后, 我们可以使用apache yarn 来对集群资源进行管理.yarn把可以把资源(内存,CPU)以Container的方式进行划分隔离.YARN会管理集群中所有机 ...

  9. Linux 2.6 完全公平调度算法CFS（Completely Fair Scheduler） 分析

    转会http://www.ibm.com/developerworks/cn/linux/l-completely-fair-scheduler/index.html? ca=drs-cn-0125 ...

  10. 利用yarn capacity scheduler在EMR集群上实现大集群的多租户的集群资源隔离和quota限制

    转自:https://m.aliyun.com/yunqi/articles/79700 背景 使用过hadoop的人基本都会考虑集群里面资源的调度和优先级的问题,假设你现在所在的公司有一个大hado ...

随机推荐

  1. CF369E Valera and Queries kdtree

    给你一堆线段,求:一个区间内包含的本质不同线段种类数(只要线段有一部分在区间中就算是包含) 考虑容斥:总线段数-被那些没有询问的区间完全覆盖的数量. 用离线+树状数组数点或者 KDtree 数点即可. ...

  2. jQuery相关方法10

    一.链式编程的原理 <script> //构造函数 function Person(age){ this.age=age; this.sayHi=function(txt){ if(txt ...

  3. unbuntu16.04安装geoserver运行环境

    1.下载并上传 在windows下载geoserver 2.15.1Platform Independent Binary版本, 是zip文件,然后使用xfile将zip上传到/usr/geoserv ...

  4. 支持utf8的str_split函数

    <?php header("Content-type: text/html; charset=utf-8"); /** * 按字节数对字符串进行分片 * @param $st ...

  5. python3 系统监控脚本（CPU，memory，网络，disk等）

    #!/usr/bin/env python3 #create at 2018-11-30 'this is a system monitor scripts' __author__="yjt ...

  6. 小程序wx.showLoading的使用

    比如说在用户点击登录的时候,为了防止用户点击点第二次,可以加一个loading,在请求结束之后就关闭

  7. 页面tr和td的的隐藏与显示

    <view:qrytr attributes="class=zcrzs"> </view:qrytr>     var bd11 = $("tr[ ...

  8. Vue源码分析(一) : new Vue() 做了什么

    Vue源码分析(一) : new Vue() 做了什么 author: @TiffanysBear 在了解new Vue做了什么之前,我们先对Vue源码做一些基础的了解,如果你已经对基础的源码目录设计 ...

  9. Android 查看和修改网络mtu

    CPU:RK3288 系统:Android 5.1 MTU:通信术语 最大传输单元(Maximum Transmission Unit,MTU)是指一种通信协议的某一层上面所能通过的最大数据包大小(以 ...

  10. ArcGIS超级工具SPTOOLS1.7升级说明

    ArcGIS超级工具1.7升级说明:多了:5个工具,总87工具. 5.11   数据打包 44 5.11.1.  mxd批量打包MPK:对一个文件夹所有MXD打包MPK 5.11.2.  mxd文档发 ...