OpenShift实战(三)：OpenShift持久化存储Registry

1、查看Registry组件的DC关于volume的定义　

　　可以看到registry-storage这个挂载点被指向了一个/registry目录，使用的是empty directory，即数据保存在计算节点上，当registry容器重启，所在node节点产生变化，数据即丢失。

[root@master1 ~]# oc volumes dc/docker-registry --all
deploymentconfigs/docker-registry
  empty directory as registry-storage
    mounted at /registry
  secret/registry-certificates as registry-certificates
mounted at /etc/secrets

2、备份Registry

　　查看当前使用的空间

[root@master1 ~]# oc get pods
NAME                       READY     STATUS    RESTARTS   AGE
docker-registry--91s28    /       Running             3d
registry-console--jkcg9   /       Running             3d
router--90m4s             /       Running             3d
router--gqpmw             /       Running             3d
router--wdkkc             /       Running             3d
[root@master1 ~]# oc rsh docker-registry--91s28 'du' '-sh' '/registry'
.7G    /registry

　　将/registry数据导出

[root@master1 ~]# mkdir /export/registry_bak
[root@master1 ~]# cd !$
cd /export/registry_bak
[root@master1 registry_bak]# oc rsync docker-registry--91s28:/registry .
receiving incremental file list
registry/
registry/docker/
registry/docker/registry/
registry/docker/registry/v2/
registry/docker/registry/v2/blobs/
registry/docker/registry/v2/blobs/sha256/
registry/docker/registry/v2/blobs/sha256//
registry/docker/registry/v2/blobs/sha256//043c1366c361738b7518f87d87071910347763605e0866b6ebba2032415cbbc8/
registry/docker/registry/v2/blobs/sha256//043c1366c361738b7518f87d87071910347763605e0866b6ebba2032415cbbc8/data
…
…
…
sent  bytes  received  bytes  10586095.68 bytes/sec
total size is   speedup is 1.00
[root@master1 registry_bak]# du -sh .
.7G    .

3、创建Registry持久化PV

　　创建分支，此处使用NFS

[root@master1 ~]# mkdir -p /export/pv/docker_registry
[root@master1 ~]# yum install nfs-utils rpcbind -y
[root@master1 ~]# chown -R nfsnobody:nfsnobody /export/pv/
[root@master1 ~]# systemctl start rpcbind
[root@master1 ~]# systemctl enable rpcbind
[root@master1 ~]# exportfs -r
[root@master1 ~]# systemctl start nfs-server
[root@master1 ~]# systemctl enable nfs-server
[root@master1 export]# echo "/export/pv/docker_registry/  *(rw,sync,all_squash)" >> /etc/exports
[root@master1 export]# exportfs -r
[root@master1 export]# systemctl reload nfs-server
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
[root@master1 ~]# setenforce
[root@master1 opt]# mkdir /opt/pv/
[root@master1 opt]# cd !$

　　创建PV

[root@master1 pv]# cat pv.json
{
  "apiVersion": "v1",
  "kind": "PersistentVolume",
  "metadata": {
    "name": "registrypv"
  },
  "spec": {
    "capacity": {
      "storage": "100Gi"
    },
    "accessModes": [ "ReadWriteOnce" ],
    "nfs": {
      "path": "/export/pv/docker_registry",
      "server": "192.168.10.110"
    },
    "persistentVolumeReclaimPolicy": "Retain"
  }
}

[root@master1 pv]# oc create -f pv.json
persistentvolume "registrypv" created

　　添加PV标签

[root@master1 pv]# oc label pv registrypv disktype=registry
persistentvolume "registrypv" labeled
[root@master1 pv]# oc get pv --show-labels
NAME         CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS      CLAIM     STORAGECLASS   REASON    AGE       LABELS
registrypv   100Gi      RWO           Retain          Available                                      2m        disktype=registry

　　创建PVC

[root@master1 pv]# cat registry_pvc.json
{
  "apiVersion": "v1",
  "kind": "PersistentVolumeClaim",
  "metadata": {
    "name": "docker-registry-claim"
  },
  "spec": {
    "accessModes": [
      "ReadWriteOnce"
    ],
    "selector": {
      "matchLabels": {
        "disktype": "registry"
      }
    },
    "resources": {
      "requests": {
        "storage": "100Gi"
      }
    }
  }
}

[root@master1 pv]# oc create -f registry_pvc.json
persistentvolumeclaim "docker-registry-claim" created

　　查看PV及PVC状态

[root@master1 pv]# oc get pv
NAME         CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS    CLAIM                           STORAGECLASS   REASON    AGE
registrypv   100Gi      RWO           Retain          Bound     default/docker-registry-claim                            8m
[root@master1 pv]# oc get pvc
NAME                    STATUS    VOLUME       CAPACITY   ACCESSMODES   STORAGECLASS   AGE
docker-registry-claim   Bound     registrypv   100Gi      RWO                          54s

4、恢复数据

　　将备份的数据恢复到创建的NFS目录

[root@master1 ~]# cd /export/
[root@master1 export]# mv registry_bak/registry/* pv/docker_registry/
[root@master1 export]# ls pv/docker_registry/
docker
[root@master1 export]# chown -R nfsnobody:nfsnobody /export/

　　为registry添加持久化卷请求，并与挂载点关联

[root@master1 export]# oc volumes dc/docker-registry --add --name=registry-storage -t pvc --claim-name=docker-registry-claim --overwrite
deploymentconfig "docker-registry" updated
[root@master1 export]# oc get pods
NAME                       READY     STATUS    RESTARTS   AGE
docker-registry--91s28    /       Running             3d
docker-registry--deploy   /       Running             6s

　　DC被重新定义后，openshift会创建新的容器实例。

　　查看容器状态

[root@master1 export]# oc get pods -o wide
NAME                       READY     STATUS              RESTARTS   AGE       IP               NODE
docker-registry--91s28    /       Running                       3d        10.130.0.4       node1.xxx.net
docker-registry--deploy   /       Running                       2m        10.130.0.27      node1.xxx.net
docker-registry--t1snn    /       ContainerCreating             2m        <none>           node2.xxx.net
registry-console--jkcg9   /       Running                       3d        10.128.0.5       node4.xxx.net
router--90m4s             /       Running                       3d        192.168.10.113   node2.xxx.net
router--gqpmw             /       Running                       3d        192.168.10.116   node3.xxx.net
router--wdkkc             /       Running                       3d        192.168.10.112   node1.xxx.net

[root@master1 export]# oc get pods -o wide
NAME                       READY     STATUS    RESTARTS   AGE       IP               NODE
docker-registry--t1snn    /       Running             5m        10.131.0.35      node2.xxx.net
registry-console--jkcg9   /       Running             3d        10.128.0.5       node4.xxx.net
router--90m4s             /       Running             3d        192.168.10.113   node2.xxx.net
router--gqpmw             /       Running             3d        192.168.10.116   node3.xxx.net
router--wdkkc             /       Running             3d        192.168.10.112   node1.xxx.net

　　创建成功后检查Registry大小

[root@master1 export]# oc rsh docker-registry--t1snn "du" "-sh"  "/registry"
.7G    /registry
[root@master1 export]# oc volumes dc/docker-registry
deploymentconfigs/docker-registry
  pvc/docker-registry-claim (allocated 100GiB) as registry-storage
    mounted at /registry
  secret/registry-certificates as registry-certificates
mounted at /etc/secrets