Azure vm 扩展脚本自动部署Elasticsearch集群

一、完整过程比较长，我仅给出Azure vm extension script 一键部署Elasticsearch集群的安装脚本，有需要的同学，可以邮件我，我给你完整的ARM Template

如果你不是用Azure VM ，也没关系，我的bash脚本都是一个个的函数，你可以自己改造成自己的Linux安装脚本。

二、此脚本的诞生是为了解决两个问题：

1. ELK在线安装有时候会异常缓慢，导致整个在线安装脚本奔溃
2. 一个一个vm节点部署Elasticsearch集群比较繁琐，此处实现一键自动生成集群

大致的思路是先离线下载好安装包，存放到自己的容易下载的地方，我此处用的是Azure blob storage，只要达到一个目的：你能通过一个链接，能快速，方便地访问下载到Elasticsearch的安装包即可。利用Azure VM extension，调用我给出的脚本，自行下载和安装。

三、达到的目的：

1. 运行成功后，自动创建多台虚拟机，并且虚拟机ES之间组成集群
2. ES集群得到初步优化，vm system在适配ES上也得到初步优化

四、安装脚本：

#!/bin/bash
# Copyright (c) Microsoft.  All rights reserved.
# Licensed under ELASTIC LICENSE
##
# Author : Wenbo Yang  July/16/2018
##
# Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ELK-Semantic-Logging/ELK/AzureRM/logstash-on-ubuntu/logstash-install-ubuntu.sh
# Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch/scripts/elasticsearch-ubuntu-install.sh
# Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch-vmss/install-elasticsearch.sh
# Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ES-MultiNode/es-ubuntu-install.sh
##

# Log method to control/redirect log output
log()
{
    echo "$1"
    logger "$1"
}

if [ "${UID}" -ne 0 ];
then
    log "Script executed without root permissions"
    echo "You must be root to run this program." >&2
    exit 3
fi

# TEMP FIX - Re-evaluate and remove when possible
# This is an interim fix for hostname resolution in current VM
grep -q "${HOSTNAME}" /etc/hosts
if [ $? == 0 ]
then
  echo "${HOSTNAME} found in /etc/hosts"
else
  echo "${HOSTNAME} not found in /etc/hosts"
  # Append it to the hosts file if not there
  echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts
  log "hostname ${HOSTNAME} added to /etc/hosts"
fi

#Loop through options passed
while getopts :p:c:m:e:k:h optname; do
  log "Option $optname set with value ${OPTARG}"
  case $optname in
    p) #set cluster name
      FIRSTPRIVATEIP=${OPTARG}
      ;;
    c) #set cluster name
      NODECOUNT=${OPTARG}
      ;;
    e) #set master mode
      ES_DOWNLOAD_URL=${OPTARG}
      ;;
    k) #set master mode
      KIBANA_DOWNLOAD_URL=${OPTARG}
      ;;
    h) #show help
      help
      exit 2
      ;;
    \?) #unrecognized option - show help
      echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed."
      help
      exit 2
      ;;
  esac
done

# Usage: get_discovery_endpoints start_address node_count
# Example: get_discovery_endpoints 10.0.1.4 3
# (returns ["10.0.1.4", "10.0.1.5", "10.0.1.6"]
get_discovery_endpoints()
{
    declare start_address=$1
    declare address_prefix=${start_address%.*}     # Everything up to last dot (not including)
    declare -i address_suffix_start=${start_address##*.}  # Last part of the address, interpreted as a number
    declare retval='['
    declare -i i
    declare -i suffix

    for (( i=0; i<$2; ++i )); do suffix=$(( address_suffix_start + i )) retval+="\"${address_prefix}.${suffix}\", " done retval=${retval:0:-2} # Remove last comma and space retval+=']' echo $retval } # Install Oracle Java install_java() { log "Installing Java" add-apt-repository -y ppa:webupd8team/java apt-get -y update > /dev/null
    echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
    echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
    apt-get -y install oracle-java8-installer  > /dev/null
    java -version
    if [ $? -ne 0 ]; then
        log "Java installation failed"
        exit 1
    fi
}

# Install ES with Debian Package manually
install_es()
{
    log "Installing Elaticsearch"
    sudo wget -q "$ES_DOWNLOAD_URL" -O elasticsearch.deb
    sudo dpkg -i elasticsearch.deb
}

# Install Kibana with Debina Package manually
install_kibana()
{
    log "Installing Kibana"
    sudo wget -q "$KIBANA_DOWNLOAD_URL" -O kibana.deb
    sudo dpkg -i kibana.deb
}

# Configure elasticsearch
configure_es()
{
    log "Update configuration"
    mv /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.bak
    echo "cluster.name: elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    echo "node.name: ${HOSTNAME}" >> /etc/elasticsearch/elasticsearch.yml
    declare -i minimum_master_nodes=$((($NODECOUNT / 2) + 1))
    echo "discovery.zen.minimum_master_nodes: 2" >> /etc/elasticsearch/elasticsearch.yml
    discovery_endpoints=$(get_discovery_endpoints $FIRSTPRIVATEIP $NODECOUNT)
    echo $discovery_endpoints
    echo "discovery.zen.ping.unicast.hosts: $discovery_endpoints" >> /etc/elasticsearch/elasticsearch.yml
    IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
    echo "network.host: ${IPADDRESS}" >> /etc/elasticsearch/elasticsearch.yml
    echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml
    echo "bootstrap.memory_lock: true" >> /etc/elasticsearch/elasticsearch.yml

    echo "node.master: true" >> /etc/elasticsearch/elasticsearch.yml
    echo "node.data: true" >> /etc/elasticsearch/elasticsearch.yml
    echo "path.data: /var/lib/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    echo "path.logs: /var/log/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    sudo update-rc.d elasticsearch defaults 95 10
    service elasticsearch start
    #sudo systemctl stop elasticsearch.service
    sleep 30

    if [ <code>systemctl is-failed elasticsearch.service</code> == 'failed' ];
    then
        log &quot;Elasticsearch unit failed to start&quot;
        exit 1
    fi
}

# Configure kibana
configure_kibana()
{
    IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
    echo &quot;server.host: \&quot;${IPADDRESS}\&quot;&quot; &gt;&gt; /etc/kibana/kibana.yml
    echo &quot;elasticsearch.url: \&quot;http://${IPADDRESS}:9200\&quot;&quot; &gt;&gt; /etc/kibana/kibana.yml
    sudo update-rc.d kibana defaults 95 10
    service kibana start
    #sudo systemctl stop kibana.service
    sleep 10

    if [ <code>systemctl is-failed kibana.service</code> == 'failed' ];
    then
        log &quot;Kibana unit failed to start&quot;
        exit 1
    fi
}

configure_system()
{
    echo &quot;options timeout:1 attempts:5&quot; &gt;&gt; /etc/resolvconf/resolv.conf.d/head
    resolvconf -u
    ES_HEAP=<code>free -m |grep Mem | awk '{if ($2/2 &gt;31744)  print 31744;else printf &quot;%.0f&quot;, $2/2;}'</code>
    echo &quot;ES_JAVA_OPTS=\&quot;-Xms${ES_HEAP}m -Xmx${ES_HEAP}m\&quot;&quot; &gt;&gt; /etc/default/elasticsearch
    echo &quot;JAVA_HOME=$JAVA_HOME&quot; &gt;&gt; /etc/default/elasticsearch
    echo 'MAX_OPEN_FILES=65536' &gt;&gt; /etc/default/elasticsearch
    echo 'MAX_LOCKED_MEMORY=unlimited' &gt;&gt; /etc/default/elasticsearch

    #https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#systemd
    mkdir -p /etc/systemd/system/elasticsearch.service.d
    touch /etc/systemd/system/elasticsearch.service.d/override.conf
    echo '[Service]' &gt;&gt; /etc/systemd/system/elasticsearch.service.d/override.conf
    echo 'LimitMEMLOCK=infinity' &gt;&gt; /etc/systemd/system/elasticsearch.service.d/override.conf
}

log &quot; ---------------begin------------------- &quot;
install_java
log &quot;Master Node install elasticsearch and kibana&quot;
log &quot;Install configure and start elasticsearch&quot;
install_es
configure_system
configure_es
log &quot;Install configure and start kibana&quot;
install_kibana
configure_kibana