elasticsearch 数据导入(九)

注:重复造轮子了 以下可以改为spring batch 来导入。

说明

maven依赖

官方客户端 https://www.elastic.co/guide/en/elasticsearch/client/java-rest/6.4/index.html

<dependency>
                <groupId>org.elasticsearch</groupId>
                <artifactId>elasticsearch</artifactId>
                <version>6.5.0</version>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.elasticsearch.client</groupId>
                <artifactId>elasticsearch-rest-high-level-client</artifactId>
                <version>6.5.0</version>
                <exclusions>
                    <exclusion>
                        <artifactId>commons-codec</artifactId>
                        <groupId>commons-codec</groupId>
                    </exclusion>
                    <!--此处要排除掉自带的，这个自带的版本低，会报错-->
                    <exclusion>
                        <artifactId>elasticsearch</artifactId>
                        <groupId>org.elasticsearch</groupId>
                    </exclusion>
                </exclusions>
            </dependency>

Util类

配置类

package com.crb.ocms.product.domain.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;

/**
 * @Project crb-product-domain
 * @PackageName com.crb.ocms.product.admin.config
 * @ClassName ESConfiguration
 * @Author liqiang
 * @Date 2019/3/6 5:52 PM
 * @Description 用于加载es的相关配置
 */
@ConfigurationProperties(prefix = "esconfig")
@Data
public class ESConfiguration {
    /**
     * index别名
     */
    private String aliasName;

    /**
     * 索引名字
     */
    private String indexName;

    /**
     * 全量索引的最大处理线程大小
     */
    private int threadSize;

    /**
     * 全量索引并行执行每个线程每次执行数据大小
     */
    private int treadDataSize;

    /**
     * typename
     */
    private String typeName;

    /**
     * ESURL
     */
    private String esUrl;

    /**
     * es ip
      */
    private String host;

    /**
     * es端口
     */
    private Integer port;

    /***
     *   用于标识是否正在处理的rediskey
     */
    private String redissKey;

    /**
     *mappingJson
     */
    private String mappingJson;
}

util工具类

package com.crb.ocms.product.service.impl;

import com.crb.ocms.product.domain.config.ESConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;

/**
 * @Project crb-product-service
 * @PackageName com.crb.ocms.demo.service.test.impl
 * @ClassName ESHLRestUtil
 * @Author liqiang
 * @Date 2019/2/25 13:45
 * @Description es工具类
 */
@Slf4j
@Component
@EnableConfigurationProperties(ESConfiguration.class)
public class ESHLRestUtil {
    private RestHighLevelClient client = null;

    public RestHighLevelClient getClient() {
        return client;
    }

    public void setClient(RestHighLevelClient client) {
        this.client = client;
    }
    ESConfiguration esConfiguration;

    @Autowired
    public ESHLRestUtil(ESConfiguration configuration) {
        client = new SimpleRestHighLevelClient(RestClient.builder(new HttpHost(configuration.getHost(), configuration.getPort(), "http")));
        this.esConfiguration=configuration;
    }
    public SimpleRestHighLevelClient getSimpleClient(){
        return (SimpleRestHighLevelClient)client;
    }

    /**
     *根据indexname获得index名字 支持通配符匹配
     * @param indexName
     * @return
     */
    public   String [] getIndexNames(String indexName)  {

        GetIndexResponse getIndexResponse= null;
        try {
            getIndexResponse = getIndexInfo(indexName);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return getIndexResponse!=null?getIndexResponse.getIndices():null;
    }

    /**
     * 根据index名字获得index信息
     * @param indexName
     * @return
     */
    public GetIndexResponse  getIndexInfo(String indexName) throws IOException {
      //GetIndexRequest request = new GetIndexRequest().indices(indexName);
//        try {
//            GetIndexResponse getIndexResponse =client.indices().get(request, RequestOptions.DEFAULT);
//            return  getIndexResponse;
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
        //master_timeout
        /**
         * 因为现在是api使用6.5 线上是6.24 master_timeout   使用以下方式 替换掉
         */
        GetIndexRequest request = new GetIndexRequest().indices(indexName);
        String[] indices = request.indices() == null ? Strings.EMPTY_ARRAY : request.indices();
        String endpoint =indexName;
        Request httpequest = new Request("GET", endpoint);
        httpequest.addParameter("ignore_unavailable", "false");
        httpequest.addParameter("expand_wildcards", "open");
        httpequest.addParameter("allow_no_indices", "true");
        org.apache.http.HttpEntity entity = new NStringEntity("", ContentType.APPLICATION_JSON);
        httpequest.setEntity(entity);
        Response httpResponse= getClient().getLowLevelClient().performRequest(httpequest);
        GetIndexResponse getIndexResponse=getSimpleClient().parseGetIndexRespons(httpResponse);
        return  getIndexResponse;
    }

    /**
     * 获得最大index名字
     * @param names indexName_number 格式
     * @return
     */
    public  Integer getMaxIndexNumber(String []names) {
        if(names==null||names.length<=0){
            return null;
        }
        List<Integer> indexNumber=new ArrayList<Integer>();
        for (String name:
                names) {
            String[] arrs=name.split("_");
            if(arrs.length<=1){
                continue;
            }
            indexNumber.add(Integer.valueOf(arrs[1]));
        }
        indexNumber.sort(new Comparator<Integer>() {
            @Override
            public int compare(Integer o1, Integer o2) {
                 return o2 - o1;
            }
        });
        return CollectionUtils.isEmpty(indexNumber)?null:indexNumber.get(0);
    }
    /**
     * 验证索引是否存在
     *
     * @param index 索引名称
     * @return
     * @throws Exception
     */
    public boolean indexExists(String index) throws Exception {
        GetIndexRequest request = new GetIndexRequest();
        request.indices(index);
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        return exists;
    }

    /**
     * 创建index
     *
     * @param index
     * @param indexType
     * @param properties 结构: {name:{type:text}} {age:{type:integer}}
     * @return
     * @throws Exception
     */
    public boolean indexCreate(String index, String indexType,
                               Map properties) throws Exception {

        if (indexExists(index)) {
            return true;
        }
        CreateIndexRequest request = new CreateIndexRequest(index);
        request.settings(Settings.builder().put("index.number_of_shards", 3)
                .put("index.number_of_replicas", 2));
        Map jsonMap = new HashMap<>();
        Map mapping = new HashMap<>();
        mapping.put("properties", properties);
        jsonMap.put(indexType, mapping);
        request.mapping(indexType, jsonMap);

        CreateIndexResponse createIndexResponse = client.indices().create(
                request,RequestOptions.DEFAULT);
        boolean acknowledged = createIndexResponse.isAcknowledged();
        return acknowledged;
    }

    /**
     * 删除指定索引
     * @param indexName
     * @return
     * @throws IOException
     */
    public boolean deleteIndex(String indexName) throws IOException {
        DeleteIndexRequest request = new DeleteIndexRequest(indexName);
        AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
        return deleteIndexResponse.isAcknowledged();
    }
    /**
     * 创建索引
     * @param index       索引名字
     * @param settiongs settiongs
     * @return
     * @throws Exception
     * @author lqiang
     */
    public boolean indexCreate(String index, String settiongs
    ) throws Exception {

        if (indexExists(index)) {
            return true;
        }
        CreateIndexRequest request = new CreateIndexRequest(index);
//        request.settings(Settings.builder().put("index.number_of_shards", 3)
//                .put("index.number_of_replicas", 2));
        request.source(settiongs, XContentType.JSON);

        CreateIndexResponse createIndexResponse = client.indices().create(
                request,RequestOptions.DEFAULT);
        boolean acknowledged = createIndexResponse.isAcknowledged();
        return acknowledged;
    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param josonStr
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, String josonStr) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.source(josonStr, XContentType.JSON);
        IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param map
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 批量创建更新文档
     *
     * @param index
     * @param indexType
     * @param list      建索引时传入_id作为docuemntId
     * @return
     * @throws Exception
     */
    public boolean documentCreateBulk(String index, String indexType,
                                      List<Map<String, Object>> list) throws Exception {

        if (list.size() > 0) {

            BulkRequest bulkRequest = new BulkRequest();

            for (Map map : list) {
                IndexRequest indexRequest;

                Object idObj = map.get("_id");
                if (idObj != null && StringUtils.isNotBlank(idObj.toString())) {
                    map.remove("_id");
                    String documentId = idObj.toString();
                    indexRequest = new IndexRequest(index, indexType, documentId);
                } else {
                    indexRequest = new IndexRequest(index, indexType);
                }
                indexRequest.source(map);
                bulkRequest.add(indexRequest);

            }

            BulkResponse bulkResponse = client.bulk(bulkRequest,RequestOptions.DEFAULT);

            if (bulkResponse.hasFailures()) {
                System.out.println("索引异常信息：" + bulkResponse.buildFailureMessage());
                return false;
            }
        }

        return true;

    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param routing
     * @param map
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, String routing, Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.routing(routing);
        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 创建索引
     *
     * @param index
     * @param indexType
     * @param josonStr
     * @return
     * @throws Exception
     */
    public String documentCreate(String index, String indexType, String josonStr)
            throws Exception {
        IndexRequest request = new IndexRequest(index, indexType);

        request.source(josonStr, XContentType.JSON);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        String id = indexResponse.getId();
        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return id;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return id;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return null;
    }

    /**
     * 创建索引
     *
     * @param index
     * @param indexType
     * @param map
     * @return
     * @throws Exception
     */
    public String documentCreate(String index, String indexType,
                                 Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType);

        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        String id = indexResponse.getId();
        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return id;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return id;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return null;
    }

    public boolean documentDelete(String index, String indexType,
                                  String documentId) throws Exception {
        DeleteRequest request = new DeleteRequest(index, indexType, documentId);
        DeleteResponse deleteResponse = client.delete(request,RequestOptions.DEFAULT);
        if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 为指定index指定别名
     * @param indexName index名字
     * @param aliasesName 别名名字
     * @return
     */
    public boolean aliases(String indexName,String aliasesName){
            IndicesAliasesRequest request=new IndicesAliasesRequest();
            IndicesAliasesRequest.AliasActions aliasAction =
                    new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
                            .index(indexName)
                            .alias(aliasesName);
            request.addAliasAction(aliasAction);
            try {
                AcknowledgedResponse indicesAliasesResponse =
                        getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
                return indicesAliasesResponse.isAcknowledged();
            } catch (IOException e) {
                log.info("指定别名失败");
                return  false;
            }
    }

    /**
     * 指定别名 并删除oldIndex的别名
     * @param oldindex
     * @param newIndex
     * @param aliasesName
     * @return
     */
    public boolean aliases(String oldindex, String newIndex,String aliasesName) {
        IndicesAliasesRequest request=new IndicesAliasesRequest();
        //旧的删除
        IndicesAliasesRequest.AliasActions aliasAction =
                new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.REMOVE)
                        .index(oldindex)
                        .alias(aliasesName);
        request.addAliasAction(aliasAction);
        //新的绑定
        aliasAction =
                new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
                        .index(newIndex)
                        .alias(aliasesName);
        request.addAliasAction(aliasAction);
        try {
            AcknowledgedResponse indicesAliasesResponse =
                    getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
            return indicesAliasesResponse.isAcknowledged();
        } catch (IOException e) {
            return  false;
        }
    }
    /**
     * 根据id获得文档信息
     * @param index indexName
     * @param type typeName
     * @param id id
     * @return
     * @throws IOException
     */
    public GetResponse getDoucmentById(String index, String type, String id) throws IOException {
        GetRequest getRequest = new GetRequest(
                index,//索引
                type,//类型
                id);//文档ID
        GetResponse getResponse = client.get(getRequest,RequestOptions.DEFAULT);
        return getResponse;
    }
    /**
     * 简单的单表根据条件进行查询
     *
     * @param index      index
     * @param type       type
     * @param parameters 参数
     * @return
     */
    public List<String> queryByMatch(String index, String type, Map<String, String> parameters) {
        List<String> jsons=new ArrayList<String>();
        try {
            SearchRequest searchRequest=new SearchRequest();
            BoolQueryBuilder booleanQueryBuilder= QueryBuilders.boolQuery();
            SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
            for (String key :
                        parameters.keySet()) {
                    booleanQueryBuilder.must().add(QueryBuilders.termQuery(key,parameters.get(key)));
            }
            searchSourceBuilder.query(booleanQueryBuilder);
            searchRequest.source(searchSourceBuilder);

            SearchResponse response = client.search(searchRequest,RequestOptions.DEFAULT);
            if(response.getHits().getHits()!=null){
                for (SearchHit searchHit:
                response.getHits().getHits() ) {
                    jsons.add(searchHit.getSourceAsString());
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return jsons;
    }
    /**
     * 只包含字母
     *
     * @return 验证成功返回true，验证失败返回false
     */
    public static boolean checkLetter(String cardNum) {
        String regex = "^[A-Za-z]+$";
        return Pattern.matches(regex, cardNum);
    }

    /**
     * 验证中文
     *
     * @param chinese 中文字符
     * @return 验证成功返回true，验证失败返回false
     */
    public static boolean checkChinese(String chinese) {
        String regex = "^[\u4E00-\u9FA5]+$";
        return Pattern.matches(regex, chinese);
    }

    /**
     * Description:提示词，支持中文、拼音、首字母等
     * <p>
     * 1、检测搜索词是中文还是拼音
     * 2、若是中文，直接按照name字段提示
     * 3、若是拼音（拼音+汉字），先按照name.keyword_pinyin获取，若是无结果按照首字母name.keyword_first_py获取
     * @param index
     * @param type
     * @param field 提示字段名字
     * @param text 文本
     * @return
     * @author liqiang
     */
    public Set<String> getSuggestWord(String index, String type, String field, String text) throws IOException {
        String postField=field;
        if (checkLetter(text)) {
            postField = field + ".keyword_pinyin";
        } else if (checkChinese(text)) {
            postField = field;
        } else {
            postField = field + ".keyword_pinyin";
        }
        Set<String> suggestTexts= postSuggestWord(index,type,postField,text);
        if(org.springframework.util.CollectionUtils.isEmpty(suggestTexts)){
            return postSuggestWord(index,type,field+".keyword_first_py",text);
        }
        return suggestTexts;
    }
    /**
     * Description:提示词，支持中文、拼音、首字母等
     * @param index
     * @param type
     * @param field 提示字段名字
     * @param text 文本
     * @return
     * @author liqiang
     */
    private Set<String> postSuggestWord(String index, String type, String field, String text) throws IOException {

        // 1、创建search请求
        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.types(type);
        // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.fetchSource(false);//不返回_source数据
        sourceBuilder.size(0);//忽略hits
        //做查询建议
        //词项建议
        SuggestionBuilder termSuggestionBuilder =
                SuggestBuilders.completionSuggestion(field).text(text);
        SuggestBuilder suggestBuilder = new SuggestBuilder();
        suggestBuilder.addSuggestion("suggest_productName", termSuggestionBuilder);
        sourceBuilder.suggest(suggestBuilder);
        searchRequest.source(sourceBuilder);
        Set<String> suggestTextList = new HashSet<String>();
        //3、发送请求
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        //4、处理响应
        //搜索结果状态信息
        if (RestStatus.OK.equals(searchResponse.status())) {
            // 获取建议结果
            Suggest suggest = searchResponse.getSuggest();
            CompletionSuggestion termSuggestion = suggest.getSuggestion("suggest_productName");
            for (CompletionSuggestion.Entry entry : termSuggestion.getEntries()) {
                for (CompletionSuggestion.Entry.Option option : entry) {
                    suggestTextList.add(option.getText().string());
                }
            }
        }
        return suggestTextList;
    }
    /**
     * @Project demo
     * @PackageName com.crb.ocms.product.service.impl
     * @ClassName SimpleRestHighLevelClient
     * @Author qiang.li
     * @Date 2019/4/2 10:45 AM
     * @Description es版本降级导致的高级api会生成部分属性 低版本不能识别  将解析结果的方法暴露在外面
     */
    public class SimpleRestHighLevelClient extends RestHighLevelClient {
        public SimpleRestHighLevelClient(RestClientBuilder restClientBuilder) {
            super(restClientBuilder);
        }

        /**
         * 父类解析的解析响应内容是受保护的 所以定义一个继承
         * 解决版本不一致的 自定义请求 解析
         * @param response
         * @param entityParser
         * @param <Req>
         * @param <Resp>
         * @return
         * @throws IOException
         */
        public <Req extends Validatable, Resp> Resp parseEntity(Response response,CheckedFunction<XContentParser, Resp, IOException> entityParser)throws IOException {

            return this.parseEntity(response.getEntity(),entityParser);
        }

        /**
         * 解析获得idnex的响应
         * @param response
         * @return
         * @throws IOException
         */
        public GetIndexResponse parseGetIndexRespons(Response response) throws IOException {
            return  parseEntity(response,GetIndexResponse::fromXContent);
        }

        /**
         * 解析查询响应
         * @param response
         * @return
         * @throws IOException
         */
        public SearchResponse parseSearchResponse(Response response) throws IOException {
            return parseEntity(response,SearchResponse::fromXContent);
        }

        /**
         * 解析索引迁移响应
         * @param response
         * @return
         * @throws IOException
         */
        public BulkByScrollResponse parseBulkByScrollResponse(Response response) throws IOException {
            return  parseEntity(response,BulkByScrollResponse::fromXContent);
        }

    }

}

封装的处理器

工作原理

将指定条件的数据迁移到新索引,然后再新索引上面进行导入（多线程并行导入）,导入完毕删除老索引和别名 然后为新索引绑定别名 实现不停机更新

抽象接口

package com.crb.ocms.product.serviceTool;

import org.elasticsearch.index.reindex.ReindexRequest;

import java.io.IOException;
import java.util.concurrent.Future;

/**
 * @Project crb-product-service
 * @PackageName com.crb.ocms.product.serviceTool
 * @ClassName EsImportService
 * @Author liqiang
 * @Date 2019/3/29 1:28 PM
 * @Description es导入抽象接口
 */
public interface ESImportService {

    /**
     * 获得索引迁移条件
     * @param reindexRequest
     * @return
     * @throws IOException
     */
    public boolean reindex(ReindexRequest reindexRequest) throws IOException;

    /**
     * 异步导入
     * @return
     */
    public Future<Boolean> importAllAsyn();
}

抽象的处理器

使用模板模式将通用代码抽出来

package com.crb.ocms.product.serviceTool;

import com.crb.ocms.product.domain.config.ESConfiguration;
import com.crb.ocms.product.domain.entity.MdProduct;
import com.crb.ocms.product.domain.redisskey.MdEsProductRedisKeyEnum;
import com.crb.ocms.product.domain.util.exceptions.OCmsExceptions;
import com.crb.ocms.product.service.impl.ESHLRestUtil;
import com.hazelcast.util.StringUtil;
import lombok.extern.log4j.Log4j2;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.redisson.api.RedissonClient;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.client.RestTemplate;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

/**
 * @Project crb-product-service
 * @PackageName com.crb.ocms.product.serviceTool
 * @ClassName ESAbstrctImpor
 * @Author liqiang
 * @Date 2019/3/29 1:35 PM
 * @Description 抽象的es导入处理器
 */
@Log4j2
public abstract class ESAbstractImport implements  ESImportService {
   private RestTemplate restTemplate;
    private ESConfiguration esConfiguration;
    private ESHLRestUtil eshlRestUtil;
    private RedissonClient redissonClient;
    /**
     * 当前index名字
     */
    private String templateIndexName;
    /**
     * 之前的index名字
     */
    private String beforeIndexName;
    public ESAbstractImport(RestTemplate restTemplate, ESConfiguration esConfiguration,RedissonClient redissonClient){
        esConfiguration.setIndexName("cmsproudct");
        esConfiguration.setAliasName("cmsproductAliasName");
        this.restTemplate=restTemplate;
        this.esConfiguration=esConfiguration;
        this.eshlRestUtil=new ESHLRestUtil(esConfiguration);
        this.redissonClient=redissonClient;
    }

    /**
     * 获得创建索引的mapping抽象方法
     * @return
     */
    public abstract String  getMapping();

    /**
     * 获得所有迁移requestBody抽象方法
     * @return
     */
    public abstract  ReindexRequest getReindex();

    /**
     * es索引秦阿姨

     * @return
     */
    @Override
    public boolean reindex(ReindexRequest reindexRequest) throws IOException {

      //  BulkByScrollResponse bulkByScrollResponse= eshlRestUtil.getClient().reindex(reindexRequest,RequestOptions.DEFAULT);
        /**
         * 因为现在是api使用6.5 线上是6.24 查询语句多生成zero_terms_query   使用以下方式 替换掉
         */
        String endpoint ="/_reindex";
        Request httpequest = new Request("POST", endpoint);
        BytesRef source = XContentHelper.toXContent(reindexRequest, XContentType.JSON, false).toBytesRef();
        org.apache.http.HttpEntity entity = new NStringEntity(new String(source.bytes).replaceAll("\"zero_terms_query\":\"NONE\","," "), ContentType.APPLICATION_JSON);
        httpequest.setEntity(entity);
        Response httpResponse=eshlRestUtil.getClient().getLowLevelClient().performRequest(httpequest);
        BulkByScrollResponse getIndexResponse=eshlRestUtil.getSimpleClient().parseBulkByScrollResponse(httpResponse);
        return true;
    }

    /**
     * 导入数据 根据设置的线程大小根据数据大小算出线程数量 通过多线程并行处理
     * @return
     */
    public boolean importAll() throws IOException {
        log.info("正在产品全量导入2:"+esConfiguration.getEsUrl());
        if (redissonClient.getMap(esConfiguration.getRedissKey()).isExists()) {
            throw new OCmsExceptions("全量索引正在导入中....");
        }
        String []indexNames=eshlRestUtil.getIndexNames(esConfiguration.getIndexName()+"*");

        if (indexNames!=null&&indexNames.length>0) {
            Integer number= eshlRestUtil.getMaxIndexNumber(indexNames);
            beforeIndexName=number==null?esConfiguration.getIndexName():esConfiguration.getIndexName()+"_"+number;
            templateIndexName=esConfiguration.getIndexName()+"_"+(number==null?1:(number+1));
        } else {
            templateIndexName = esConfiguration.getIndexName();
        }
        boolean isSuccess = true;
        long count = getCount();
        //根据线程处理大小获得页码
        int index = (new Double(Math.ceil(count*1.0 / esConfiguration.getTreadDataSize()))).intValue();
        if (index <= 0 && count > 0) {
            index = 1;
        }
        //获得最大处理线程大小
        int threadSize = index > esConfiguration.getThreadSize() ? esConfiguration.getThreadSize() : index;
        //导入处理器
        ESAbstractImport.ProcessImportIndex processImportIndex = new ESAbstractImport.ProcessImportIndex(esConfiguration.getTreadDataSize(), index, templateIndexName);
        ThreadPoolExecutor executorService=null;
        try {
            //第一次导入直接创建索引
            if (beforeIndexName == null) {
                log.info("开始创建索引");
                eshlRestUtil.indexCreate(templateIndexName,getMapping());
                log.info("创建索引成功");
            } else {
                log.info("开始创建索引1");
                eshlRestUtil.indexCreate(templateIndexName,getMapping());
                log.info("创建索引成功1");
                log.info("开始创建索引2");
                //非第一次导入将子集数据迁移进来
                reindex(getReindex());
                log.info("开始创建索引2");
            }

            /** 线程池的自定义配置，IO密集型任务. */
            executorService = new ThreadPoolExecutor(
                    // 核心线程数
                    threadSize,
                    // 最大线程数
                    100,
                    // 存活时间，30s
                    30,
                    TimeUnit.SECONDS,
                    new ArrayBlockingQueue<Runnable>(150),
                    Executors.defaultThreadFactory(),
                    new ThreadPoolExecutor.CallerRunsPolicy());

            redissonClient.getMap(esConfiguration.getRedissKey()).put("templateIndexName", templateIndexName);

            //统一保存异步处理结果
            List<Future<Boolean>> processResult = new ArrayList<Future<Boolean>>();
            for (int i = 0; i < threadSize; i++) {
                processResult.add(executorService.submit(processImportIndex));
            }

            for (Future<Boolean> future :
                    processResult) {
                if (!future.get()) {
                    isSuccess = future.get();
                    break;
                }
            }
            //表示重建成功
            if (isSuccess) {
                //表示第一次初始化 指定别名
                if (StringUtil.isNullOrEmpty(beforeIndexName)) {
                    boolean isFail = false;
                    int i = 0;
                    do {
                        isFail = !eshlRestUtil.aliases(templateIndexName,esConfiguration.getAliasName());
                        if(i>0) {
                            //休眠
                            Thread.sleep(i * 1000);
                        }
                        i++;
                    } while (isFail && i < 10);
                    if (isFail) {
                        log.info("开始删除索引2");
                        eshlRestUtil.deleteIndex(templateIndexName);
                        log.info("开始删除成功");
                    }
                } else {
                    boolean isFail = false;
                    int i = 0;
                    do {
                        //重新绑定别名 失败重试十次
                        isFail = !eshlRestUtil.aliases(beforeIndexName,templateIndexName,esConfiguration.getAliasName());
                        if(i>0) {
                            //休眠
                            Thread.sleep(i * 1000);
                        }
                        i++;
                    } while (isFail && i < 10);
                    if (isFail) {
                        //删除备份索引
                        eshlRestUtil.deleteIndex(templateIndexName);
                        throw new OCmsExceptions("索引重做失败,请重试");
                    } else {
                        //删除备份索引
                        eshlRestUtil.deleteIndex(beforeIndexName);
                    }
                }
            } else {
                for (Future<Boolean> future :
                        processResult) {
                    if (!future.get()) {
                        //没有重做成功删除创建的新索引
                        eshlRestUtil.deleteIndex(templateIndexName);
                        future.isCancelled();//取消其他正在执行的线程
                    }
                }
            }
        } catch (Exception e) {
            String uuid = UUID.randomUUID().toString();
            log.error(uuid + "\n" + e.getMessage());
            eshlRestUtil.deleteIndex(templateIndexName);
            throw new OCmsExceptions("<ES文档导入失败>异常编码" + uuid + "\n" + e.getMessage());
        } finally {
            if (executorService != null) {
                executorService.shutdown();
            }
            redissonClient.getBucket(esConfiguration.getRedissKey()).delete();

        }
        return isSuccess;
    }

    /**
     * 异步的导入方法
     * @return
     */
    @Override
    public Future<Boolean> importAllAsyn() {
        ExecutorService executorService=new ThreadPoolExecutor(1, 1,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>());
        return executorService.submit(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                return importAll();
            }
        });
    }

    /**
     * 抽象的处理导入方法
     * @param pageRequest
     * @return
     * @throws IOException
     */
    public abstract   boolean processImport(PageRequest pageRequest) throws IOException;

    public abstract Long getCount();

    /**
     * 用于处理批量索引导入
     */
    class ProcessImportIndex implements Callable<Boolean> {

        //数据总页数
        private int pageCount;
        //总条数
        private int pageSize;

        private String indexName;
        private AtomicLong currentIndex = new AtomicLong();
        //当前处理页数
        AtomicInteger currentPage = new AtomicInteger();

        public ProcessImportIndex(int pageSize, int pageCount, String indexName) {
            this.pageSize = pageSize;
            this.pageCount = pageCount;
            this.indexName = indexName;
        }

        /**
         * Computes a result, or throws an exception if unable to do so.
         *
         * @return computed result
         * @throws Exception if unable to compute a result
         */
        @Override
        public Boolean call() throws Exception {
            try {
                return process();
            } catch (Exception e) {
                e.printStackTrace();
                return false;
            }
        }

        public boolean process() throws IOException {
            int page = currentPage.getAndIncrement();
            redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("pageCount", pageCount);
            while (page < pageCount) {
                redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("currentPage", page+1);
                processImport(new PageRequest(page, pageSize));
                page = currentPage.getAndIncrement();
            }
            return true;
        }

        public AtomicInteger getCurrentPage() {
            return currentPage;
        }

        public void setCurrentPage(AtomicInteger currentPage) {
            this.currentPage = currentPage;
        }

    }

    public String getTemplateIndexName() {
        return templateIndexName;
    }

    public void setTemplateIndexName(String templateIndexName) {
        this.templateIndexName = templateIndexName;
    }

    public interface BulkCallback {
        public void process(MdProduct mdProduct);
    }

    public RestTemplate getRestTemplate() {
        return restTemplate;
    }

    public void setRestTemplate(RestTemplate restTemplate) {
        this.restTemplate = restTemplate;
    }

    public ESConfiguration getEsConfiguration() {
        return esConfiguration;
    }

    public void setEsConfiguration(ESConfiguration esConfiguration) {
        this.esConfiguration = esConfiguration;
    }

    public ESHLRestUtil getEshlRestUtil() {
        return eshlRestUtil;
    }

    public void setEshlRestUtil(ESHLRestUtil eshlRestUtil) {
        this.eshlRestUtil = eshlRestUtil;
    }

    public RedissonClient getRedissonClient() {
        return redissonClient;
    }

    public void setRedissonClient(RedissonClient redissonClient) {
        this.redissonClient = redissonClient;
    }

    public String getBeforeIndexName() {
        return beforeIndexName;
    }

    public void setBeforeIndexName(String beforeIndexName) {
        this.beforeIndexName = beforeIndexName;
    }
}

使用例子

package com.crb.ocms.product.serviceTool;

import com.crb.ocms.product.domain.config.ESConfiguration;
import com.crb.ocms.product.domain.entity.MdProduct;
import com.crb.ocms.product.domain.repository.MdProductRepository;
import com.crb.ocms.product.domain.vo.req.FindProductCharacterStocksVo;
import com.crb.ocms.product.domain.vo.resp.ProductCharacterStocksVo;
import com.crb.ocms.product.service.MdProductService;
import com.crb.ocms.product.service.feign.IcProductStoreAccountService;
import com.crb.ocms.product.service.impl.MdESProductServiceImpl;
import lombok.extern.log4j.Log4j2;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.redisson.api.RedissonClient;
import org.springframework.data.domain.Example;
import org.springframework.data.domain.ExampleMatcher;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.client.RestTemplate;

import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;

/**
 * @Project crb-product-service
 * @PackageName com.crb.ocms.product.serviceTool
 * @ClassName ProductStockImport
 * @Author liqiang
 * @Date 2019/3/29 2:15 PM
 * @Description 处理库存全量索引导入
 */
@Log4j2
@Deprecated
public class ProductStockImport extends  ProductIndexImport {
    MdProductRepository mdProductRepository;
    IcProductStoreAccountService icProductStoreAccountService;
    String regionCode;
    public ProductStockImport(RestTemplate restTemplate, ESConfiguration esConfiguration, RedissonClient redissonClient, MdProductRepository mdProductRepository, IcProductStoreAccountService icProductStoreAccountService, String regionCode) {
        super(restTemplate, esConfiguration, redissonClient);
        this.mdProductRepository=mdProductRepository;
        this.regionCode=regionCode;
        this.icProductStoreAccountService=icProductStoreAccountService;
    }

    /**
     * 索引迁移条件。将指定条件数据迁移到新的索引
     * @return
     */
    @Override
    public ReindexRequest getReindex() {
        ReindexRequest reindexRequest=new ReindexRequest();
        reindexRequest.setSourceIndices(getBeforeIndexName());
        reindexRequest.setSourceQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.matchPhraseQuery("info","ic_product_store_account")));
        reindexRequest.setDestIndex(getTemplateIndexName());
        return reindexRequest;

    }

    /**
     * 已经迁移到经销权 先禁用
     * @param pageRequest
     * @return
     * @throws IOException
     */
    @Deprecated
    @Override
    public boolean processImport(PageRequest pageRequest) throws IOException {
        MdProduct mdProduct = new MdProduct();
        mdProduct.setRegionCode(regionCode);
        ExampleMatcher exampleMatcher = ExampleMatcher.matching().withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id");;
        Example example=Example.of(mdProduct,exampleMatcher);
        Page<MdProduct> result = mdProductRepository.findAll(example,pageRequest);
        for (MdProduct md:result){
            if (md.getFullPalletSaleFlag()==null){
                md.setFullPalletSaleFlag(0);
            }
        }
        List<Long>  productIds=result.getContent().stream().map(MdProduct::getMdProductId).collect(Collectors.toList());
        FindProductCharacterStocksVo findProductCharacterStocksVo=new FindProductCharacterStocksVo();
        findProductCharacterStocksVo.setProductIds(productIds);
        findProductCharacterStocksVo.setRegionCode(regionCode);
        List<ProductCharacterStocksVo> productCharacterStocksVoList= icProductStoreAccountService.findProductCharacterStocks(findProductCharacterStocksVo);
        BulkRequest bulkRequest=new BulkRequest();
        for (ProductCharacterStocksVo productCharacterStocksVo:
                productCharacterStocksVoList) {
            if(productCharacterStocksVo.getCharacterId().longValue()==-1L){
                continue;
            }
            XContentBuilder builder = XContentFactory.jsonBuilder();
            builder.startObject();
            builder.field("productId",productCharacterStocksVo.getProductId());
            builder.field("factoryId",productCharacterStocksVo.getFactoryId());
            builder.field("characterId",productCharacterStocksVo.getCharacterId());
            builder.field("sumCount",productCharacterStocksVo.getSumCount());
            builder.field("regionCode",regionCode);
            builder.field("info").startObject()
                    .field("name","ic_product_store_account")
                    .field("parent",productCharacterStocksVo.getProductId()+"_"+regionCode)
                    .endObject();
            builder.endObject();
            String indexId="ic_product_store_account_"+productCharacterStocksVo.getProductId()+"_"+productCharacterStocksVo.getCharacterId()+"_"+productCharacterStocksVo.getFactoryId();
            IndexRequest indexRequest=new IndexRequest(getTemplateIndexName(),getEsConfiguration().getTypeName(), indexId);
            indexRequest.source(builder);
            indexRequest.routing(productCharacterStocksVo.getProductId()+"_"+regionCode);
            bulkRequest.add(indexRequest);
        }
        log.info(bulkRequest.getDescription());
        getEshlRestUtil().getClient().bulk(bulkRequest, RequestOptions.DEFAULT);
        return true;

    }

    /**
     * 数据总条数
     * @return
     */
    @Override
    public Long getCount() {
        MdProduct mdProduct = new MdProduct();
        mdProduct.setRegionCode(regionCode);
        ExampleMatcher exampleMatcher = ExampleMatcher.matching().
                withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id");
        Example example=Example.of(mdProduct,exampleMatcher);
       return mdProductRepository.count(example);
    }
}

processImport就只用关系查询出指定分页的数据 然后导入到es