1-监控界面sql保存
1, BufferSize_machine
1), template
主要用来监控buffersize的状态的
name: 模块名字, 用于后续调取使用,
label: 模块显示名字, 在页面显示的
includeAll: 是否包含 all 按钮
query: 查询的sql语句, 由于模版一致, 所以后续只保留sql
classify:
select distinct(classify) from host_dict
model_name
select distinct(model_name) from host_dict where classify in ($classify)
2) KafkaSinkNetword
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "BufferSize"
AND object = 'KafkaSinkNetwork'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, host
3), KafkaSinkFile
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "BufferSize"
AND object = 'KafkaSinkFile'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, host
4), FileSink
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "BufferSize"
AND object = 'FileSink'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, host
5), MessageCopy
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "BufferSize"
AND object = 'MessageCopy'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, host
2, BufferSizeTopic
1) BufferSize长度
SELECT
UNIX_TIMESTAMP(time) as time_sec,
sum(value) as value,
object as metric
FROM jmx_status
WHERE $__timeFilter(time)
AND attribute = 'BufferSize'
GROUP BY object, time
ORDER BY time ASC, metric
3, Metric_machine
1), template
Classify
select distinct(classify) from host_dict
model_name
select distinct(model_name) from host_dict where classify in ($classify)
source_file
select distinct(component) from topic_count where component like '%Source'
kafka_file
select distinct(component) from topic_count where component like 'Kafka%';
2), 接受消息总量
SELECT
UNIX_TIMESTAMP(tc.time) as time_sec,
tc.host as metric,
SUM(out_num) as value
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
left join host_dict as hd
on hd.innet_ip = tc.host
WHERE $__timeFilter(time)
AND tc.component in ($source_file)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
GROUP BY tc.host, tc.time
ORDER BY tc.time ASC, tc.host
3), 发送kafka消息总量
SELECT
UNIX_TIMESTAMP(tc.time) as time_sec,
tc.host as metric,
SUM(in_num - out_num) as value
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
left join host_dict as hd
on hd.innet_ip = tc.host
WHERE $__timeFilter(time)
AND tc.component in ($kafka_file)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
GROUP BY tc.host, tc.time
ORDER BY tc.time ASC, tc.host
4), 发送kafka消息失败量
SELECT
UNIX_TIMESTAMP(tc.time) as time_sec,
tc.host as metric,
SUM(out_num) as value
FROM topic_count as tc
left join host_dict as hd
on hd.innet_ip = tc.host
WHERE $__timeFilter(time)
AND (tc.component = 'KafkaSinkNetwork' OR tc.component = 'KafkaSinkFile')
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
GROUP BY tc.host, tc.time
ORDER BY tc.time ASC, tc.host
4, Metric_topic
1), template
component_name
select distinct(component_name) from topic_dict
topic
select distinct(topic) from topic_dict where component_name in ($component_name)
source_file
select distinct(component) from topic_count where component like '%Source'
kafka_file
select distinct(component) from topic_count where component like 'Kafka%';
2), iris接收总量
SELECT
UNIX_TIMESTAMP(tc.time) as time_sec,
tc.topic as metric,
SUM(out_num) as value
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
WHERE $__timeFilter(time)
AND tc.topic in ($topic)
AND tc.component in ($source_file)
GROUP BY tc.topic, tc.time
ORDER BY tc.time ASC, tc.topic
3), 发送kafka消息总量
SELECT
UNIX_TIMESTAMP(time) as time_sec,
tc.topic as metric,
SUM(in_num - out_num) as value
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
WHERE $__timeFilter(time)
AND tc.topic in ($topic)
AND tc.component in ($kafka_file)
GROUP BY tc.topic, tc.time
ORDER BY tc.time ASC, tc.topic
4), 发送kafka消息失败量
SELECT
UNIX_TIMESTAMP(time) as time_sec,
sum(out_num) as value,
topic as metric
FROM topic_count
WHERE $__timeFilter(time)
AND (component = 'KafkaSinkNetwork' OR component = 'KafkaSinkFile')
AND topic in ($topic)
GROUP BY topic, time
ORDER BY time ASC, metric
5), 消息丢失数
SELECT
UNIX_TIMESTAMP(time) as time_sec,
(temp.value - SUM(tc.in_num - tc.out_num)) as value,
tc.topic as metric
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
right join (
SELECT
tc2.time as calen,
tc2.topic,
SUM(out_num) as value
FROM topic_count as tc2
left join topic_dict as td2
on tc2.topic = td2.topic
WHERE $__timeFilter(tc2.time)
AND tc2.topic in ($topic)
AND tc2.component in ($source_file)
AND tc2.component <> 'FileSource'
GROUP BY tc2.topic, tc2.time
) as temp
on temp.calen = tc.time
and temp.topic = tc.topic
WHERE $__timeFilter(time)
AND tc.topic in ($topic)
AND tc.component in ($kafka_file)
GROUP BY tc.time, tc.topic
ORDER BY tc.topic, tc.time asc
6), 验平汇总, 此为表格
SELECT
date_format(time, '%Y-%m-%d %H:%i:%s') as time,
tc.topic,
temp.value as iris总接受量,
SUM(tc.in_num - tc.out_num) as kafka发送成功,
SUM(tc.out_num) as kafka发送失败,
(temp.value - SUM(tc.in_num - tc.out_num)) as 消息丢失数
FROM topic_count as tc
left join topic_dict as td
on tc.topic = td.topic
right join (
SELECT
tc2.time as calen,
tc2.topic,
SUM(out_num) as value
FROM topic_count as tc2
left join topic_dict as td2
on tc2.topic = td2.topic
WHERE $__timeFilter(tc2.time)
AND tc2.topic in ($topic)
AND tc2.component in ($source_file)
AND tc2.component <> 'FileSource'
GROUP BY tc2.topic, tc2.time
) as temp
on temp.calen = tc.time
and temp.topic = tc.topic
WHERE $__timeFilter(time)
AND tc.topic in ($topic)
AND tc.component in ($kafka_file)
GROUP BY tc.time, tc.topic
ORDER BY tc.topic, tc.time asc
5, QPS_Component
qps
SELECT
UNIX_TIMESTAMP(time) as time_sec,
sum(value) as value,
object as metric
FROM jmx_status
WHERE $__timeFilter(time)
AND attribute = 'QPS'
GROUP BY object, time
ORDER BY time ASC, metric
6, QPS_machine
1), template
classify
select distinct(classify) from host_dict
model_name
select distinct(model_name) from host_dict where classify in ($classify)
2), topic_source
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "QPS"
AND object = 'TcpSource'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC, metric
limit
3), js_source
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "QPS"
AND object = 'JsSource'
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC
limit
4), legency_source
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE object = 'LegacyJsSource'
AND attribute = "QPS"
AND $__timeFilter(time)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC
limit
5), webSource
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE object = 'WebSource'
AND attribute = "QPS"
AND $__timeFilter(time)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC
limit
6), zhixinSource
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
concat('KSF-', host) as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE object = 'ZhixinSource'
AND attribute = "QPS"
AND $__timeFilter(time)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC
limit
7) cdn_httpsource
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE object = 'CdnHttpSource'
AND attribute = "QPS"
AND $__timeFilter(time)
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time DESC
limit
8), qps_everyhost
SELECT
UNIX_TIMESTAMP(time) as time_sec,
sum(value) as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "QPS"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
group by host, time
ORDER BY time DESC, metric
limit
9), qps_hostnum
SELECT
UNIX_TIMESTAMP(js.time) as time_sec,
count(distinct(js.host)) as value,
hd.classify as metric
FROM jmx_status as js
left join host_dict as hd
on js.host = hd.innet_ip
WHERE $__timeFilter(time)
AND attribute = "QPS"
group by time, hd.classify
ORDER BY time DESC, metric
limit
7, Resource_machine
1), template
classify
select distinct(classify) from host_dict
model_name
select distinct(model_name) from host_dict where classify in ($classify)
2), cpu_used
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "SystemCpuLoad"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, metric
3), memory_used
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "HeapMemoryUsage.used"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, metric
4), thread_count
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "ThreadCount"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, metric
5), openfile_script
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "OpenFileDescriptorCount"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, metric
8, alter 报警用, 需要配合设置
iris第一次尝试入kafka失败百分比
SELECT
UNIX_TIMESTAMP(time) as time_sec,
value as value,
host as metric
FROM jmx_status as jmx
left join host_dict as hd
on hd.innet_ip = jmx.host
WHERE $__timeFilter(time)
AND attribute = "OpenFileDescriptorCount"
AND hd.classify in ($classify)
and hd.model_name in ($model_name)
ORDER BY time ASC, metric
主要在alter标签中
然后在alter标签中进行配置
9, 需要用到的sql保存
1), host_dict
CREATE TABLE `host_dict` (
`id` bigint() NOT NULL AUTO_INCREMENT COMMENT '主键id',
`classify` varchar() DEFAULT NULL COMMENT '类型',
`model_name` varchar() DEFAULT NULL COMMENT '模块名',
`innet_ip` varchar() DEFAULT NULL COMMENT '内网ip',
`outnet_ip` varchar() DEFAULT NULL COMMENT '外网ip',
`cpu_core` int() DEFAULT NULL COMMENT 'cpu核心',
`memory_size` int() DEFAULT NULL COMMENT '内存',
`address` varchar() DEFAULT NULL COMMENT '机房',
`status` varchar() DEFAULT NULL COMMENT '状态',
`plan` varchar() DEFAULT NULL COMMENT '规划',
PRIMARY KEY (`id`),
KEY `classify` (`classify`,`model_name`,`innet_ip`),
KEY `idx_innet_ip` (`innet_ip`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT= DEFAULT CHARSET=utf8
2), jmx_status
CREATE TABLE `jmx_status` (
`host` varchar() NOT NULL DEFAULT '',
`time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`report` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`object` varchar() NOT NULL DEFAULT '',
`attribute` varchar() NOT NULL DEFAULT '',
`value` double DEFAULT NULL,
PRIMARY KEY (`host`,`time`,`object`,`attribute`),
KEY `idx_host_time_attribute_object` (`host`,`time`,`attribute`,`object`) USING BTREE,
KEY `idx_time_attribute` (`time`,`attribute`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
/*!50100 PARTITION BY RANGE (unix_timestamp(time))
(PARTITION p20180811 VALUES LESS THAN (1534003199) ENGINE = InnoDB,
PARTITION p20180812 VALUES LESS THAN (1534089599) ENGINE = InnoDB,
PARTITION p20180813 VALUES LESS THAN (1534175999) ENGINE = InnoDB,
PARTITION p20180814 VALUES LESS THAN (1534262399) ENGINE = InnoDB,
PARTITION p20180815 VALUES LESS THAN (1534348799) ENGINE = InnoDB,
PARTITION p20180816 VALUES LESS THAN (1534435199) ENGINE = InnoDB,
PARTITION p20180817 VALUES LESS THAN (1534521599) ENGINE = InnoDB,
PARTITION p20180818 VALUES LESS THAN (1534607999) ENGINE = InnoDB,
PARTITION p20180819 VALUES LESS THAN (1534694399) ENGINE = InnoDB,
PARTITION p20180820 VALUES LESS THAN (1534780799) ENGINE = InnoDB,
PARTITION p20180821 VALUES LESS THAN (1534867199) ENGINE = InnoDB,
PARTITION p20180822 VALUES LESS THAN (1534953599) ENGINE = InnoDB) */
3), topic_count
CREATE TABLE `topic_count` (
`host` varchar() NOT NULL DEFAULT '',
`time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`report` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`component` varchar() NOT NULL DEFAULT '',
`topic` varchar() NOT NULL DEFAULT '',
`in_num` bigint() DEFAULT NULL,
`out_num` bigint() DEFAULT NULL,
PRIMARY KEY (`host`,`time`,`topic`,`component`),
KEY `component` (`component`,`topic`,`time`),
KEY `idx_topic` (`topic`) USING BTREE,
KEY `idx_time_topic` (`time`,`topic`) USING BTREE,
KEY `idx_compnent` (`component`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
4), topic_dict
CREATE TABLE `topic_dict` (
`id` bigint() NOT NULL AUTO_INCREMENT COMMENT '主键id',
`model` varchar() DEFAULT NULL COMMENT '模式',
`component` varchar() DEFAULT NULL COMMENT '组件',
`component_name` varchar() DEFAULT NULL COMMENT '组件名称',
`component_type` varchar() DEFAULT NULL COMMENT '组件类型',
`topic` varchar() DEFAULT NULL COMMENT 'topic',
`topic_type` varchar() DEFAULT NULL COMMENT 'topic类型',
`status` varchar() DEFAULT 'ON' COMMENT '状态',
PRIMARY KEY (`id`),
KEY `component` (`component`,`topic`),
KEY `idx_topic` (`topic`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT= DEFAULT CHARSET=utf8
1-监控界面sql保存的更多相关文章
- kafka-eagle监控界面搭建
kafka-eagle监控界面搭建 一.背景 二 .mac上安装kafka-eagle 1.安装JDK 2.安装eagle 1.下载eagle 2.解压并配置环境变量 3.启用kafka的JMX 4. ...
- PLSQL_监控有些SQL的执行次数和频率
原文:PLSQL_监控有些SQL的执行次数和频率 2014-12-25 Created By 鲍新建
- 【DB2】监控动态SQL语句
一.db2监控动态SQL(快照监控) db2示例用户登陆后,使用脚本语句db2 get snapshot for all on dbname>snap.out 也可以使用db2 get snap ...
- SpringBoot2.0 基础案例(07)：集成Druid连接池，配置监控界面
一.Druid连接池 1.druid简介 Druid连接池是阿里巴巴开源的数据库连接池项目.Druid连接池为监控而生,内置强大的监控功能,监控特性不影响性能.功能强大,能防SQL注入,内置Login ...
- 通过本地Agent监控Azure sql database
背景: 虽然Azure sql database有DMVs可以查看DTU等使用情况,但记录有时间限制,不会一直保留.为了更好监控Azure_sql_database上各个库的DTU使用情况.数据库磁盘 ...
- 六：SpringBoot-集成Druid连接池，配置监控界面
SpringBoot-集成Druid连接池,配置监控界面 1.Druid连接池 1.1 Druid特点 2.SpringBoot整合Druid 2.1 引入核心依赖 2.2 数据源配置文件 2.3 核 ...
- C#使用Oxyplot绘制监控界面
C#中可选的绘图工具有很多,除了Oxyplot还有DynamicDataDisplay(已经改名为InteractiveDataDisplay)等等.不过由于笔者这里存在一些环境上的特殊要求,.Net ...
- 微服务监控druid sql
参考该文档 保存druid的监控记录 把日志保存的关系数据数据库(mysql,oracle等) 或者nosql数据库(redis,芒果db等) 保存的时候可以增加微服务名称标识好知道是哪个微服务的sq ...
- Unity 编辑器的 界面布局 保存方法
在软件界面的右上角(关闭按钮的下方),点击 layout (界面)的下拉箭头. 弹出选项中的 save layout....(保存界面选项),输入命名,就可以生成这个界面的布局. (软件本身也有 ...
随机推荐
- （转载）Ubuntu 安装GNU Scientific library（GSL）
背景: Blei的hlda的C语言实现需要使用C语言的科学计算包GSL,因此决定安装.由于在windows下安装极其繁琐,先在Linux上安装之. 系统环境: Linux version 2.6.35 ...
- vue 需求 data中的数据之间的调用
我遇到过这种情况 就是在我的data中 会有数据调用data中的其他数据 如图 我的alertInfoType需要拿到screeningCondition中type的值 用过vue的都知道 我是不 ...
- sas通过IMPORT过程读取外部文件数据
SAS通过IMPORT过程读取外部文件数据 使用IMPORT过程导入带分隔符的文件外,Microsoft Access数据库文件.Miscrosft Excel工作簿. dBase文件.JMP文件.S ...
- html样式板
一.bootstrap 二.element 三.iconfont图标 四.font awesome图标
- Python11/12--GIL/互斥锁/进程池
GIL1.全局解释器锁? 锁就是线程里面那个锁 锁是为了避免资源竞争造成数据的错乱 2.python程序的执行过程? 1.启动解释器进程 python.exe 2.解析你的py文件并执行它 每个py程 ...
- linux系统下载pycharm
如何下载pycharm安装包? 你可以通过访问 https://www.jetbrains.com/pycharm/download/#section=linux 获取安装包,如果是新手建议使用社区版 ...
- Django——用户认证和判断用户是否登录
用户认证 必须通过认证之后才能login(request,user)这样才能保存会话到request中,注销后会话结束 注意 自定义的用户登陆时只不止需要验证用户名和密码的需要写认证,就例如在线教育平 ...
- 请求转发(forward)和请求重定向(redirect)的区别(转)
作者:@haimishasha本文为作者原创,转载请注明出处:https://www.cnblogs.com/haimishasha/p/5952129.html 用户向服务器发送了一次HTTP请求, ...
- 《大型网站系统与Java中间件实践》
读了一下,个人认为最好的部分,就是第四章了. CH04 服务框架 4.2 服务设计与实现 // 获取可用服务地址列表 // 确定调用服务目标机器 // 建立连接(Socket) // 请求序列化 // ...
- spring boot开发笔记——mybatis
概述 mybatis框架的优点,就不用多说了,今天这边干货主要讲mybatis的逆向工程,以及springboot的集成技巧,和分页的使用 因为在日常的开发中,当碰到特殊需求之类会手动写一下s ...