四. sql上线平台

一.安装inception及使用
1安装基础环境:

　　inception是一个集审核、执行、备份及生成回滚语句于一身的MySQL自动化运维工具.

注意，Centos7一定要制定bison版本不高于2.6，否则Inception可能会出现编译错误，该下载速度十分缓慢，可以考虑长时间等待或者更换镜像源。

下载Inception安装包:

Inception源地址已没有源码，可以打开源地址，然后在右上角上面的fork处选择新的下载地址。

源码地址：https://github.com/mysql-inception/inception

https://github.com/bbotte/inception-mysql   下载地址

安装基础环境:
[root@CentOS ~]# yum -y install cmake libncurses5-dev libssl-dev g++ bison gcc gcc-c++ openssl-devel ncurses-devel mysql pymysql
[root@CentOS ]# yum  remove bison -y
安装bison:
[root@CentOS ~]#wget http://ftp.gnu.org/gnu/bison/bison-2.5.1.tar.gz
[root@CentOS ~]# tar -zxvf bison-2.5.1.tar.gz
[root@CentOS bison-2.5.1]# ./configure
[root@CentOS bison-2.5.1]# vi ./lib/stdio.h
删除  这一行  _GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
[root@CentOS bison-2.5.1]# ./configure
[root@CentOS bison-2.5.1]# make && make install
[root@CentOS bison-2.5.1]# bison -V
安装inception：
[root@CentOS bison-2.5.1]# cd /usr/local/
将上述下载的包传上来
[root@CentOS local]# unzip inception-mysql-master.zip
[root@CentOS local]# mv inception-mysql-master inception-master
[root@CentOS local]# cd inception-master
[root@CentOS inception-master]# sh inception_build.sh builddir  Linux
修改配置文件:
vim /etc/inc.cnf   创建此文件
[inception]
general_log=1
general_log_file=inc.log
port=6669
socket=/tmp/inc.socket
character-set-client-handshake=0
character-set-server=utf8# 这里的数据库是填inception的库，也用作备份库
inception_remote_system_password=123456
inception_remote_system_user=root
inception_remote_backup_port=3306
inception_remote_backup_host=127.0.0.1
inception_support_charset=utf8
inception_enable_nullable=0
inception_check_primary_key=1
inception_check_column_comment=1
inception_check_table_comment=1
inception_osc_min_table_size=1
inception_osc_bin_dir=/usr/bin
inception_osc_chunk_time=0.1
inception_ddl_support=1
inception_enable_blob_type=1
inception_check_column_default_value=1
启动服务:
[root@CentOS inception-master]# /usr/local/inception-master/builddir/mysql/bin/Inception --defaults-file=/etc/inc.cnf
(python36env) [vagrant@CentOS devops]$ netstat -nltp 可以看到有6669的端口跑起来了
测试是否安装成功：输入后123456，进入数据库命令行
(python36env) [vagrant@CentOS devops]$ mysql -uroot -h127.0.0.1 -P 6669   登进inception管理控制台
MySQL [(none)]> inception get variables;   查看它所有变量/属性

+------------------------------------------+---------------------------------------------+
| autocommit | OFF |
| bind_address | * |
| character_set_system | utf8 |
| character_sets_dir | /usr/local/inception-master/share/charsets/ |
| connect_timeout | 10 |
| date_format | %Y-%m-%d |
| datetime_format | %Y-%m-%d %H:%i:%s |
| general_log | ON |
| general_log_file | inc.log |
| inception_check_autoincrement_datatype | ON |
| inception_check_autoincrement_init_value | ON |
| inception_check_autoincrement_name | ON |
| inception_check_column_comment | ON |
| inception_check_column_default_value | ON |
| inception_check_dml_limit | ON |
| inception_check_dml_orderby | ON |
| inception_check_dml_where | ON |
| inception_check_identifier | ON |
| inception_check_index_prefix | ON |
| inception_check_insert_field | ON |
| inception_check_primary_key | ON |
| inception_check_table_comment | ON |
| inception_check_timestamp_default | ON |
| inception_ddl_support | ON |
| inception_enable_autoincrement_unsigned | ON |
| inception_enable_blob_type | ON |
| inception_enable_column_charset | OFF |
| inception_enable_enum_set_bit | OFF |
| inception_enable_foreign_key | OFF |
| inception_enable_identifer_keyword | OFF |
| inception_enable_not_innodb | OFF |
| inception_enable_nullable | OFF |
| inception_enable_orderby_rand | OFF |
| inception_enable_partition_table | OFF |
| inception_enable_pk_columns_only_int | OFF |
| inception_enable_select_star | OFF |
| inception_enable_sql_statistic | ON |
| inception_max_char_length | 16 |
| inception_max_key_parts | 5 |
| inception_max_keys | 16 |
| inception_max_primary_key_parts | 5 |
| inception_max_update_rows | 10000 |
| inception_merge_alter_table | ON

。。。。。。

(python36env) [vagrant@CentOS devops]$ sudo su - root
[root@CentOS ~]# netstat -nltp
tcp6 0 0 :::6669 :::* LISTEN 1150/Incep
[root@CentOS ~]# kill -9 1150
使用后台方式跑:(公司中一般用supervisior去管理它)
[root@CentOS inception-master]# nohup /usr/local/inception-master/builddir/mysql/bin/Inception --defaults-file=/etc/inc.cnf &
inception使用规范及说明文档:
https://inception-document.readthedocs.io/zh_CN/latest/

2.inception简单使用:
可通过如下脚本调inception(在执行sql时，把sql作为一参数传给此脚本，此脚本去连inception作了一检测和执行操作)。

[root@CentOS ~]# mysql -uroot -p

MariaDB [(none)]> create database pro1;

MariaDB [(none)]> use pro1;

建表:

MariaDB [pro1]> create table if not exists `mytable1`(
-> `id` int unsigned auto_increment,
-> `myname` varchar(10) not null,
-> primary key(`id`)
-> )engine=innodb default charset=utf8;

MariaDB [pro1]> show tables;
+----------------+
| Tables_in_pro1 |
+----------------+
| mytable1 |

[root@CentOS ~]# vi test.py

#!/usr/bin/python
#-\*-coding: utf-8-\*-

''' 建库建表语句
CREATE DATABASE pro1;
CREATE TABLE IF NOT EXISTS `mytable1`(
   `id` INT UNSIGNED AUTO_INCREMENT,
   `myname` VARCHAR(10) NOT NULL,
   PRIMARY KEY ( `id` )
)ENGINE=InnoDB DEFAULT CHARSET=utf8;
'''

import MySQLdb
# 待审核/执行的sql语句（需包含目标数据库的地址、端口 等参数）
sql='/* --user=root;--password=123456;--host=127.0.0.1;--port=3306;--enable-execute; */\
inception_magic_start;\
use pro1;\
insert into mytable1 (myname) values ("xianyu1"),("xianyu2");\
insert into mytable1 (myname) values ("xianyu1"),("xianyu2");\
inception_magic_commit;'
try:
    # inception的地址、端口等
    conn=MySQLdb.connect(host='127.0.0.1',user='root',passwd='123456',db='',port=6669)
    cur=conn.cursor()
    ret=cur.execute(sql)
    result=cur.fetchall()
    num_fields = len(cur.description)
    field_names = [i[0] for i in cur.description]
    print (result)
    '''
    for row in result:
        print row[0], "|",row[1],"|",row[2],"|",row[3],"|",row[4],"|",
        row[5],"|",row[6],"|",row[7],"|",row[8],"|",row[9],"|",row[10]
    '''
    cur.close()
    conn.close()
except Exception  as e:
     print (e)

[root@CentOS ~]# yum install -y epel-release

[root@CentOS ~]# yum install -y python-pip

[root@CentOS ~]# pip install MySQL-python 　　python3的话就用pymysql,那对应的如下test.py中也要改成pymysql

[root@CentOS ~]# python test.py  运行后返回如下结果了两条记录
((1L, 'CHECKED', 0L, 'Audit completed', 'None', 'use pro1', 0L, "'0_0_0'", 'None', '0', ''), (2L, 'CHECKED', 0L, 'Audit completed', 'None', 'insert into mytable1 (myname) values ("xianyu1"),("xianyu2")', 2L, "'0_0_1'", '127_0_0_1_3306_pro1', '0', ''), (3L, 'CHECKED', 0L, 'Audit completed', 'None', 'insert into mytable1 (myname) values ("xianyu1"),("xianyu2")', 2L, "'0_0_2'", '127_0_0_1_3306_pro1', '0', ''))

MariaDB [pro1]> select * from mytable1;表中也插入数据了
+----+---------+
| id | myname |
+----+---------+
| 1 | xianyu1 |
| 2 | xianyu2 |
| 3 | xianyu1 |
| 4 | xianyu2 |

inception自动备份功能需要:
1.mysql设置binlog需要如下配置
[root@CentOS ~]# vi /etc/my.cnf
[mysqld]
log-bin = mysql-bin
binlog_format = ROW
server-id = 1
[root@CentOS ~]# systemctl restart mariadb
MariaDB [(none)]> show databases;
+---------------------+
| Database |
+---------------------+
| information_schema |
| 127_0_0_1_3306_pro1 |   有备份库了.
| devops |
| inception |
| mysql |
| performance_schema |
| pro1 |
| test1 |

如图inception与djangog与mysql三个组件关系:

它们分别对应三个库:

 二.启动sql上线项目基础包启动

(1)如下图把基础包拷贝到项目根目录中:

(2)用pycharm打开此sql项目基础包

(3)sqlweb/settings.py配置:

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': 'sqlweb',
        'USER': 'root',
        'PASSWORD': '123456',
        'HOST':'127.0.0.1',
        'PORT':'3306',
    },
}

(5)数据迁移

(python36env) [vagrant@CentOS sqlweb]$ cd /vagrant/sqlweb
(python36env) [vagrant@CentOS sqlweb]$ pip install pymysql
(python36env) [vagrant@CentOS sqlweb]$ pip install django-rest-swagger
(python36env) [vagrant@CentOS devops]$ mysql -uroot -p123456
MariaDB [(none)]> create database sqlweb;
(python36env) [vagrant@CentOS sqlweb]$ python manage.py makemigrations
(python36env) [vagrant@CentOS sqlweb]$ python manage.py migrate
(python36env) [vagrant@CentOS sqlweb]$ python manage.py createsuperuser
admin
admin@123.com

(6)pycharm远程连接sql项目

此时启动sql项目如下图无法打开， 原因没做映射:

但是用命令行却可以:

[vagrant@CentOS ~]$ curl http://127.0.0.1:8081/

a.解决：配置映射F:\devops\data\vagrantfile

F:\devops\data>vagrant reload

效果如下图可以了，

且如下图中可登录的效果，这样基础包就跑起来了!

 三.iview初始化--让sql项目与前端结合起来

　　把iview基础包运行起来。此基础包用法与element一样，只需在官网下载此包，并做基本配置---npm stall 装node库，然后运行起来即可了。

(1)

F:\devops\data\frontend>npm install --registry=https://registry.npm.taobao.org

(2)用webstorm打开frontend基础包

跨域代理配置:我所有的后端接口都是以api开头的，所以把api开头的都转换到127.0.0.1:8081这个地址。

 (3)后端接口配置

 F:\devops\data\frontend>npm run dev

报错:npm run dev 报错 iview TypeError [ERR_INVALID_CALLBACK]: Callback must be a function

然后找到webpack.dev.config.js打开

将这一行代码：fs.write(fd, buf, 0, buf.length, 0, function(err, written, buffer) {});

替换为fs.write(fd, buf, 0, 'utf-8', function(err, written, buffer) {});

保存然后再次执行 npm run dev 启动跳转到如下图

(4)在页面创建用户如图

登录不上报如下错:net::ERR_NAME_NOT_RESOLVED

 解决:

(1)如下图，登录配置中告诉我们走的登录path是在utils中配置，所以改成如下

(2)C:\Windows\System32\drivers\etc\hosts做解析加上如下配置----我的没做

127.0.0.1 sqlweb

保存后如下图登成功了：

https://www.iviewui.com/   iviewui前端组件模版参考

四.sql 审核功能实现 

 1.写api接口,路由,视图--前端接口展示

(1)F:\devops\data\frontend\src\api\sql\check.js

import axios from '../../libs/http'

const autoSelects = '/api/sqlmng/autoselects/'
const inceptionCheck = '/api/sqlmng/inceptioncheck/'

export function GetSelectData(data) {
    return axios({
        url: autoSelects,
        method: 'post',
        data: data
    })
}

export function CheckSql(data) {
    return axios({
        url: inceptionCheck,
        method: 'post',
        data: data
    })
}

(2)F:\devops\data\frontend\src\views\sql\check.vue

<template>
  <div>
      <Card>
      <Row>
        <Col span="12">
            <Alert show-icon>输入要上线的SQL语句</Alert>
            </br>
            <div>
              <Form class="step-form" ref="checkContent" :model="checkData" :rules="ruleCheckData" :label-width="100">
                <FormItem label="SQL" prop="sql_content">
                  <Input v-model="checkData.sql_content" type="textarea" :autosize="{minRows: 10,maxRows: 20}" placeholder="请输入SQL" />
                </FormItem>
                <FormItem label="备注">
                  <Input v-model="checkData.remark" type="textarea" :autosize="{minRows: 2,maxRows: 5}" placeholder="请输入备注" />
                </FormItem>
                <FormItem label="操作">
                  <Row>
                    <Col span="12">
                      <center>
                        <Button type="primary" @click='handleCheckSql'>确定</Button>
                      </center>
                    </Col>
                    <Col span="12">
                      <center>
                        <Button>清空</Button>
                      </center>
                    </Col>
                  </Row>
                </FormItem>
              </Form>
            </div>
            </Col>

          <Col span="12">
            <Alert show-icon style='margin-left:12%'>
              <Icon type="ios-lightbulb-outline" slot="icon"></Icon>
                选择执行条件
            </Alert>
            <div style='margin-left:50px'>
              </br>
              <div>
                <Form class="step-form" ref="checkConf" :model="checkData" :rules="ruleCheckData" :label-width="100">
                  <FormItem label="环境">
                    <RadioGroup v-model="checkData.env" @on-change="handleSelect">
                      <Radio label="prd">生产</Radio>
                      <Radio label="test">测试</Radio>
                    </RadioGroup>
                  </FormItem>
                  <FormItem label="数据库" prop="db">
                    <Select v-model="checkData.db" style="width:200px; margin-left:10px">
                        <Option v-for="item in dbList" :value="item.value" :key="item.value">{{ item.label }}</Option>
                    </Select>
                  </FormItem>
                  <FormItem label="执行人" prop="treater">
                    <Select v-model="checkData.treater" style="width:200px; margin-left:10px">
                        <Option v-for="item in transactorList" :value="item.value" :key="item.value">{{ item.label }}</Option>
                    </Select>
                  </FormItem>
                </Form>
              </div>

            </div>
        </Col>
      </Row>

    </Card>

  </div>
</template>
<script>
  import { GetSelectData, CheckSql } from '@/api/sql/check'

  export default {
    data () {
      return {
        checkData: {
          sql_content: '',
          remark: '',
          env: 'prd',
          db: '',
          treater: '',
        },
        ruleCheckData: {
          sql_content: [{ required: true, message: '请填写SQL', trigger: 'blur' }],
          treater:  [{ required: true, message: '请选择执行人', trigger: 'change' }],
          db: [{ required: true, message: '请选择数据库', trigger: 'change', type: 'number' }],
        },
        dbList: [],
        transactorList: [],
        keyMap: {
          'sql_content':'SQL',
          'env':'环境',
          'db':'数据库',
          'treater':'执行人',
        },
      }
    },

    created () {
      this.handleSelect(this.checkData.env)
    },

    methods: {

      renderFunc (treater) {
          this.$Notice.success({
              title: 'SQL审核通过',
              desc: 'SQL审核通过...',
              render: h => {
                  return h('span', [
                      '请等待 ',
                      h('a', treater),
                      ' 执行'
                  ])
              }
          });
      },

      warning (title, msg) {
          this.$Notice.warning({
              title: title,
              duration: 0,
              desc: msg
          });
      },

      handleSelect (e) {
          GetSelectData({env:e})
          .then(response => {
            console.log(response)
            const dbs = response.data.data.dbs
            const managers = response.data.data.managers
            this.dbList = []
            dbs.map( (item) => {
              this.dbList.push({
                value:item.id,
                label:item.name
              })
            })
            this.transactorList = []
            managers.map( (item) => {
              this.transactorList.push({
                value:item,
                label:item,
              })
            })
          })
          .catch(error => {
            console.log(error)
          })
      },

      handleCheckSql () {
        this.$refs.checkContent.validate((valid) => {
          if (!valid) {
            return
          }
          this.$refs.checkConf.validate((valid) => {
            if (!valid) {
              return
            }
            CheckSql(this.checkData)
            .then(response => {
              console.log(response)
              let status = response.data.status
              let msg = response.data.msg
              if (status == 0){
                this.renderFunc(this.checkData.treater)
              } else if (status == -1 || status == -2){
                this.warning('SQL审核不通过', msg)
              }
            })
            .catch(error => {
              console.log(error)
            })
          })
        })

      },

    },

  }
</script>

(3)src\router\router.js加入此段代码

刷新后如图效果:

 2.sql实现sql语句简单审核--无判断

(1)sqlmgn/models.py:

#coding=utf-8
from django.db import models
from django.contrib.auth.models import User, Group

class Basemodel(models.Model):
    name = models.CharField(default='', null=True, blank=True, max_length=128, verbose_name='名字')
    createtime = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')
    updatetime = models.DateTimeField(auto_now=True, verbose_name='修改时间')
    remark = models.TextField(default='', null=True, blank=True, verbose_name='备注')

    def __unicode__(self):
        return self.name

    class Meta:
        abstract = True
        ordering = ['-id']

class Dbconf(Basemodel):
    sql_content = models.TextField()

(2)sqlmng/urls.py

#coding=utf-8
from django.conf.urls import include, url
from rest_framework.routers import DefaultRouter
from .views import *

# register的可选参数 base_name: 用来生成urls名字，如果viewset中没有包含queryset, base_name一定要有

router = DefaultRouter()
router.register(r'dbconfs', DbViewSet, base_name='dbviewset')
router.register(r'inceptioncheck', InceptionCheckView, base_name='Inceptioncheckview')

urlpatterns = [
    url(r'^', include(router.urls)),
]

(3)sqlmg/views.py

# Create your views here.
from rest_framework.response import Response
from .baseviews import BaseView
from .serializers import DbSerializer
from .models import *
from utils.inception import table_structure

class DbViewSet(BaseView):
    queryset = Dbconf.objects.all()
    serializer_class = DbSerializer
    search_fields = ['name', 'user', 'password','host', 'port', 'env']

class InceptionCheckView(BaseView):
    # 重写create--因为发任务是post发过来的,(注意view中的create就是对应前端请求的post方法(等价于post)。而序列化中的create方法是view中的create去写数据时调用它)。
    def create(self, request, *args, **kwargs):
        # 接收前端数据
        request_data = request.data
        print(request_data)
        sql_content = request_data.get('sql_content')
        db_addr = '--user=root; --password=123456; --host=127.0.0.1; --port=3306; --enable-check;'
        db_name = 'pro1'
        res = table_structure(db_addr, db_name, sql_content)
        print(res)
        return Response(self.ret)

(4)sqlmg/serializers.py

# -*- coding:utf-8 -*-
from rest_framework import serializers
from .models import *

class DbSerializer(serializers.ModelSerializer):
    class Meta:
        model = Dbconf
        fields = '__all__'

(5)新建根项目下utils包/inception.py:

#coding=utf-8

import pymysql

def table_structure(dbaddr, dbname, sqlcontent):
    status = 0
    sql = '/* {} */\
      inception_magic_start;\
      use {}; {} inception_magic_commit;'.format(dbaddr, dbname, sqlcontent)
    try:
        #conn = pymysql.connect(host='172.16.98.12',user='root',passwd='',db='',port=6669,use_unicode=True, charset="utf8")
        conn = pymysql.connect(host='127.0.0.1', user='root', passwd='', port=6669, db='', use_unicode=True, charset="utf8")  # 连接inception
        cur = conn.cursor()
        cur.execute(sql)
        result = cur.fetchall()
        cur.close()
        conn.close()
    except pymysql.Error as e:
        status = -1
        result = "Mysql Error {}: {}".format(e.args[0], e.args[1])
    return {'result':result, 'status':status}

def get_rollback(sql, dbname=''):
    conn = pymysql.connect(host = '127.0.0.1', port = 3306, user = 'root', passwd = '123456', db = dbname, charset = 'utf8')  # 连接回滚库
    conn.autocommit(True)
    cur = conn.cursor()
    cur.execute(sql)
    return cur.fetchall()

(5)src/view/sql/check.vue   ----删除检查form功能修改成右图，否则登录不上

            

(python36env) [vagrant@CentOS sqlweb]$ python manage.py runserver 0.0.0.0:8081

运行后报错:

if int(self.server_version.split('.', 1)[0]) >= 5:
ValueError: invalid literal for int() with base 10: 'Inception2'
解决:
(python36env) [vagrant@CentOS sqlweb]$ vi /home/vagrant/python36env/lib/python3.6/site-packages/pymysql/connections.py 加入如下代码

try:
    if int(self.server_version.split('.', 1)[0]) >= 5:
        self.client_flag |= CLIENT.MULTI_RESULTS
except:
    if self.server_version.split('.', 1)[0] == 'Inception2':
        self.client_flag |= CLIENT.MULTI_RESULTS

如下图效果:

如下图提交后出来：

上述效果中还没实现审核的功能，如有错误的sql语句它还不能处理 判断语句

3.sql审核判断功能

　　需求:dba跟我提需求，他需要在我们提交的sql中不允许有use,drop等这样的关键字出现！--实现:先获取sql内容，然后对它做正则匹配关键在内容中否。

我在让它调inception前匹配，先判断是否符合dba的规则，不符合则不让其调inception,

(1)sqlmng/views.py:

from django.shortcuts import render

# Create your views here.
from rest_framework.response import Response
from .baseviews import BaseView
from .serializers import DbSerializer
from .models import *
from utils.inception import table_structure
from rest_framework.exceptions import ParseError
import re

class DbViewSet(BaseView):
    queryset = Dbconf.objects.all()
    serializer_class = DbSerializer
    search_fields = ['name', 'user', 'password','host', 'port', 'env']

class InceptionCheckView(BaseView):
    forbidden_word_list = ['drop ','use ']
    # 重写create--因为发任务是post发过来的,(注意view中的create就是对应前端请求的post方法(等价于post)。而序列化中的create方法是view中的create去写数据时调用它)。
    def create(self, request, *args, **kwargs):
        # 接收前端数据
        request_data = request.data
        print(request_data)
        sql_content = request_data.get('sql_content')
        #匹配关键字在不在sql中，不在就抛异常:
        forbidden_words = [fword for fword in self.forbidden_word_list if
                           re.search(re.compile(fword, re.I), sql_content)]
        if forbidden_words:
            raise ParseError({'禁用': forbidden_words})

        db_addr = '--user=root; --password=123456; --host=127.0.0.1; --port=3306; --enable-check;'
        db_name = 'pro1'
        sql_review = table_structure(db_addr, db_name, sql_content)
        print(sql_review)
        result, status = sql_review.get('result'), sql_review.get('status')
        # 判断检测语法错误，有则返回
        if status == -1 or len(result) == 1:  # 兼容2种版本的抛错
            raise ParseError({self.connect_error: result})
        success_sqls = []
        exception_sqls = []
        for sql_result in result:
            error_message = sql_result[4]
            if error_message == 'None' or re.findall('Warning', error_message):
                success_sqls.append(sql_result)
            else:
                exception_sqls.append(error_message)
        if exception_sqls:
            raise ParseError({self.exception_sqls: exception_sqls})
        return Response(self.ret)

点确定后效果如图:

b.

11

22