[k8s]openshiftv1.5.1安装笔记

centos7安装

net.ifnames=0 biosdevname=0

初始化系统

yum install wget -y

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all
yum makecache
yum install lrzsz ntpdate sysstat dos2unix wget telnet  tree  bind-utils  net-tools  vim -y

ulimit -SHn 65535
echo '* - nofile 65535' >>/etc/security/limits.conf

\cp /etc/ssh/sshd_config /etc/ssh/sshd_config.ori
sed -i 's#\#UseDNS yes#UseDNS no#g' /etc/ssh/sshd_config
sed -i 's#GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config
systemctl restart sshd

echo '*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com >/dev/null 2 >&1' >>/var/spool/cron/root
ntpdate ntp1.aliyun.com &&hwclock -w

echo 1 > /proc/sys/net/ipv4/ip_forward

sysctl -w net.ipv4.ip_forward=1

sed -i "s#keepcache=0#keepcache=1#g" /etc/yum.conf
systemctl stop postfix
systemctl disable postfix

设置vim

set nu
set cursorline
set nobackup
set ruler
set autoindent
set vb t_vb=
set ts=4
set expandtab
source /etc/vimrc

centos7改ip和主机名脚本

[root@node3 ~]# cat shell/init.sh 

#!/bin/sh
################################################
# this script is created by chocolee.
# e_mail:781647046@qq.com
# qqinfo:781647046
# blog:http://www.cnblogs.com/iiiiher/
# version:1.1
# update_date:2016-10-8 09:48:04
################################################

#Source function library.
source /etc/init.d/functions

initHostnameIPADDRS(){
echo ""
echo "================配置主机名和ip地址====================="
    sed -i "6c IPADDR=$IPADDRS_eth0" /etc/sysconfig/network-scripts/ifcfg-eth0
    #sed -i "6c IPADDR=$IPADDRS_eth1" /etc/sysconfig/network-scripts/ifcfg-eth1

   #sed -i "140c ListenAddress=$IPADDRS_eth0:52000" /etc/ssh/sshd_config
   # sed -i "141c ListenAddress=$IPADDRS_eth1:22" /etc/ssh/sshd_config

    echo "$HOSTNAME" > /etc/hostname
    /bin/hostname $HOSTNAME

    echo ""
    echo "===================Debuging=============================="
    echo '#grep "IPADDRS" /etc/sysconfig/network-scripts/ifcfg-eth0'
    grep "IPADDR" /etc/sysconfig/network-scripts/ifcfg-eth0
    echo ""
    echo '#grep "HOSTNAME" /etc/sysconfig/network'
    grep "HOSTNAME" /etc/sysconfig/network
echo ""
action "配置hostname和ip地址显示格式完成" /bin/true
echo "=======================notice========================="
echo " "
#echo "下次请用$IPADDRS_eth0:52000登录$HOSTNAME"
echo ""
  sleep 2
}

#判断IP是否符合标准规则
function judge_ip(){
        #这里local $1出错，用2>/dev/null屏蔽掉错误，暂未发现影响输出结果
        local $1 2>/dev/null
        TMP_TXT=/tmp/iptmp.txt
        echo $1 > ${TMP_TXT}
        IPADDRS=`grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' ${TMP_TXT}`
        #判断有没有符合***.***.***.***规则的IP
        if [ ! -z "${IPADDRS}" ];then
                local j=0;
                #通过循环来检测每个点之前的数值是否符合要求
                for ((i=1;i<=4;i++))
                do
                    local IP_NUM=`echo "${IPADDRS}" |awk -F. "{print $"$i"}"`
                    #判断IP_NUM是否在0与255之间
                    if [ "${IP_NUM}" -ge 0 -a "${IP_NUM}" -le 255 ];then
                            ((j++));
                    else
                            return 1
                    fi
                done
                #通过j的值来确定是否继续匹配规则，循环四次，若都正确j=4.
                if [ "$j" -eq 4 ];then
            #确认是否为自己想要输入的IP地址
            read -n 1 -p "你输入的IP是${IPADDRS},确认输入：Y|y；重新输入：R|r：" OK
            echo
            case ${OK} in
                        Y|y) return 0;;
                R|r) return 1;;
                *) return 1;;
            esac
                else
                        return 1
                fi
        else
                return 1
        fi
}

echo "========================================"
echo '          Linux Optimization            '
echo "========================================"

#hostname
read -p "Please enter HOSTNAME: " HOSTNAME

#ip
read -p "Please enter eth0:IPADDR 192.168.8.x: " IPADDRS_eth0
#read -p "Please enter eth1:IPADDR 10.1.1.x: " IPADDRS_eth1

judge_ip "${IPADDRS_eth0}";
#judge_ip "${IPADDRS_eth1}";
i=`echo $?`
#循环直到输入正确的IP为止
until [ "$i" -eq 0 ];do
    echo -e "\033[31m你输入了错误的IP：${IPADDRS} ====>>>>\033[0m"
    read -p "重新输入IP，示例“192.168.8.233”，请输入：" IPADDRS
    judge_ip "${IPADDRS}";
    i=`echo $?`
done

initHostnameIPADDRS
systemctl restart network
systemctl restart sshd

初始化主机名和ip

修改hosts

192.168.8.140 lb.pp100.net
192.168.8.141 master1.pp100.net
192.168.8.142 master2.pp100.net
192.168.8.143 master3.pp100.net
192.168.8.144 node1.pp100.net
192.168.8.145 node2.pp100.net
192.168.8.146 node3.pp100.net
192.168.8.147 etcd1.pp100.net
192.168.8.148 etcd2.pp100.net
192.168.8.149 etcd3.pp100.net
192.168.8.140 openshift-cluster.pp100.net

master1与其他机器做互信

下载1.5.1的镜像

docker pull openshift/origin-pod:v1.5.1
docker pull openshift/origin:v1.5.1
docker pull openshift/origin-deployer:v1.5.1
docker pull openshift/origin-docker-registry:v1.5.1
docker pull openshift/origin-haproxy-router:v1.5.1
docker pull openshift/origin-logging-deployer:v1.5.1
docker pull openshift/origin-metrics-cassandra:v1.5.1
docker pull openshift/origin-metrics-deployer:v1.5.1
docker pull openshift/origin-metrics-hawkular-metrics:v1.5.1
docker pull openshift/origin-metrics-heapster:v1.5.1
docker pull openshift/origin-sti-builder:v1.5.1
docker pull openshift/origin-logging-deployer:v1.5.1
docker pull openshift/origin-logging-elasticsearch:v1.5.1
docker pull openshift/origin-logging-curator:v1.5.1
docker pull openshift/origin-logging-fluentd:v1.5.1
docker pull openshift/origin-logging-kibana:v1.5.1
docker pull openshift/origin-logging-deployment:v1.5.1

docker save -o openshift_origin-pod_v1.5.1.tar                      openshift/origin-pod:v1.5.1
docker save -o openshift_origin_v1.5.1.tar                          openshift/origin:v1.5.1
docker save -o openshift_origin-deployer_v1.5.1.tar                 openshift/origin-deployer:v1.5.1
docker save -o openshift_origin-docker-registry_v1.5.1.tar          openshift/origin-docker-registry:v1.5.1
docker save -o openshift_origin-haproxy-router_v1.5.1.tar           openshift/origin-haproxy-router:v1.5.1
docker save -o openshift_origin-logging-deployer_v1.5.1.tar         openshift/origin-logging-deployer:v1.5.1
docker save -o openshift_origin-metrics-cassandra_v1.5.1.tar        openshift/origin-metrics-cassandra:v1.5.1
docker save -o openshift_origin-metrics-deployer_v1.5.1.tar         openshift/origin-metrics-deployer:v1.5.1
docker save -o openshift_origin-metrics-hawkular-metrics_v1.5.1.tar openshift/origin-metrics-hawkular-metrics:v1.5.1
docker save -o openshift_origin-metrics-heapster_v1.5.1.tar         openshift/origin-metrics-heapster:v1.5.1
docker save -o openshift_origin-sti-builder_v1.5.1.tar              openshift/origin-sti-builder:v1.5.1
docker save -o openshift_origin-logging-deployer_v1.5.1.tar         openshift_origin-logging-deployer:v1.5.1
docker save -o openshift_origin-logging-elasticsearch_v1.5.1.tar    openshift_origin-logging-elasticsearch:v1.5.1
docker save -o openshift_origin-logging-curator_v1.5.1.tar          openshift_origin-logging-curator:v1.5.1
docker save -o openshift_origin-logging-fluentd_v1.5.1.tar          openshift_origin-logging-fluentd:v1.5.1
docker save -o openshift_origin-logging-kibana_v1.5.1.tar           openshift_origin-logging-kibana:v1.5.1
docker save -o openshift_origin-logging-deployment_v1.5.1.tar       openshift_origin-logging-deployment:v1.5.1

master1上pip安装ansible

下载ansible openshift安装脚本

git clone https://github.com/openshift/openshift-ansible.git

ansible安装hosts

vim /etc/ansible/hosts

[OSEv3:children]
masters
nodes
etcd
lb
[OSEv3:vars]
ansible_ssh_user=root
deployment_type=origin
openshift_version=1.5.1
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
openshift_master_cluster_method=native
openshift_master_cluster_hostname=openshift-cluster.pp100.net
openshift_master_cluster_public_hostname=openshift-cluster.pp100.net
openshift_master_default_subdomain=pp100.net
[masters]
master1.pp100.net
master2.pp100.net
master3.pp100.net
[etcd]
etcd1.pp100.net
etcd2.pp100.net
etcd3.pp100.net
[lb]
lb.pp100.net
[nodes]
master[1:3].pp100.net openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
node[1:3].pp100.net openshift_node_labels="{'region': 'primary', 'zone': 'shenzhen'}"

启动ansible安装

//先清理环境,然后安装

ansible-playbook ~/openshift-ansible/playbooks/adhoc/uninstall.yml
ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/byo/config.yml -b -v --private-key=~/.ssh/id_rsa

使master可调度

oc adm manage-node 192.168.8.141 --schedulable=true
oc adm manage-node 192.168.8.142 --schedulable=true
oc adm manage-node 192.168.8.143 --schedulable=true

检查master

oc get nodes
netstat -ltnp #8443
ps -aux|grep openshift
systemctl status origin-master-api origin-master-controllers origin-node dnsmasq | grep Active

检查etcd

etcdctl -C https://192.168.8.141:2379,https://192.168.8.142:2379,https://192.168.8.142:2379 --ca-file=/etc/etcd/ca.crt --cert-file=/etc/etcd/peer.crt --key-file=/etc/etcd/peer.key cluster-health

# 端口 进程
ps -aux|grep -E "openshift|etcd"
 systemctl status etcd | grep Active -B3

检查node

netstat -ltnp
ps -aux|grep openshift
systemctl status origin-node dnsmasq | grep Active -B3

配置dnsmasq

rpm -qc dnsmasq
/etc/dbus-1/system.d/dnsmasq.conf
/etc/dnsmasq.conf

[root@master1 dnsmasq.d]# pwd
/etc/dnsmasq.d
[root@master1 dnsmasq.d]# cat origin-dns.conf
no-resolv
domain-needed
server=/cluster.local/172.30.0.1
address=/.pp100.net/192.168.6.141
address=/gitlab.pp100.net/192.168.6.73
address=/gogs.pp100.net/192.168.6.85
addn-hosts=/etc/dnsmasq.d/names/name.list
[root@master1 dnsmasq.d]# cat origin-upstream-dns.conf
server=192.168.6.6
server=114.114.114.114
[root@master1 dnsmasq.d]# cat names/name.list
192.168.8.141 openshift-cluster.pp100.net
192.168.8.142 openshift-cluster.pp100.net
192.168.8.143 openshift-cluster.pp100.net

将dnsmasq配置提交到所有的master和node节点（也可以每个master和node节点都上面三个操作步骤执行一遍）

# ansible nodes -m copy -a 'src=/etc/dnsmasq.d/ dest=/etc/dnsmasq.d/'
# ansible nodes -m copy -a 'src=/etc/resolv.conf dest=/etc/resolv.conf'

重启NetworkManager进程

# systemctl restart NetworkManager dnsmasq
# systemctl status NetworkManager dnsmasq | grep Active -B3

配置iptables，开放dns端口

# iptables-save > /etc/sysconfig/iptables
# vim /etc/sysconfig/iptables
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
...
#在OS_FIREWALL_ALLOW相关配置下面添加
-A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT
-A OS_FIREWALL_ALLOW -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT
...
#使其生效
# iptables-restore /etc/sysconfig/iptables

验证dns

# nslookup openshift-cluster.pp100.net 192.168.8.141
# nslookup qq.com 192.168.8.141

用户管理

//创建用户
# htpasswd -c /etc/origin/master/htpasswd lanny
New password:
Re-type new password:
Adding password for user lanny

//如果要删除用户，执行以下命令：
# htpasswd -D /etc/origin/master/htpasswd lanny
Deleting password for user lanny

//添加授权
# oc adm policy add-cluster-role-to-user cluster-admin lanny

//登录
# oc login -u lanny -n default
Authentication required for https://openshift-cluster.pp100.net:8443 (openshift)
Username: lanny
Password:
Login successful.
You have access to the following projects and can switch between them with 'oc project <projectname>':
  * default
    kube-system
    logging
    management-infra
    openshift
    openshift-infra
Using project "default".

浏览器访问

https://openshift-cluster.pp100.net:8443

导入镜像服务端脚本

\\脚本
/data/images/openshift_v1.5.1
at lo[root@test52 openshift_v1.5.1]# cat load_images.sh
HTTP_SERVER=192.168.6.52:8000
load_images()
{
    images=(
openshift_origin-deployer_v1.5.1.tar
openshift_origin-docker-registry_v1.5.1.tar
openshift_origin-haproxy-router_v1.5.1.tar
openshift_origin-logging-curator_v1.5.1.tar
openshift_origin-logging-deployer_v1.5.1.tar
openshift_origin-logging-elasticsearch_v1.5.1.tar
openshift_origin-logging-fluentd_v1.5.1.tar
openshift_origin-metrics-cassandra_v1.5.1.tar
openshift_origin-metrics-deployer_v1.5.1.tar
openshift_origin-metrics-hawkular-metrics_v1.5.1.tar
openshift_origin-metrics-heapster_v1.5.1.tar
openshift_origin-pod_v1.5.1.tar
openshift_origin-sti-builder_v1.5.1.tar
openshift_origin_v1.5.1.tar
        )
    for i in "${!images[@]}"; do
        curl -L http://$HTTP_SERVER/${images[$i]} > /root/images/${images[$i]}
        docker load < /root/images/${images[$i]}
    done
}
load_images

\\服务端开启py http服务

\\客户端curl导入

阿里docker加速器

阿里加速器地址

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://2sm5kxd3.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

配置docker

# cat /etc/sysconfig/docker
OPTIONS=' --selinux-enabled --insecure-registry=172.30.0.0/16 --log-driver=json-file --log-opt max-size=50m --signature-verification=false'

OPTIONS=' --selinux-enabled --selinux-enabled --log-driver=journald --insecure-registry=172.30.0.0/16 --log-driver=json-file --log-opt max-size=50m --signature-verification=false'