使用java操作elasticsearch(1)

1.安装elasticsearch

　　这儿用的是5.6.9的版本，下载安装过程较为简单,在官网上下载好后解压到文件夹。需要注意的是在elasticsearch-5.6.9\config下的elasticsearch.yml文件夹中

　　配置data与log的地址，自己创建文件夹后如下配置

　　

 # ----------------------------------- Paths ------------------------------------
 #
 # Path to directory where to store the data (separate multiple locations by comma):
 #
 path.data: D:/java/soft/elasticsearch/dev/data
 #
 # Path to log files:
 #
 path.logs: D:/java/soft/elasticsearch/dev/log
 #

　启动elasticsearch   点击elasticksearch.bat即可启动  注意电脑上要安装1.8以上的jdk

出现以下消息则说明启动成功

上面红圈的9300端口则是与es进行数据交互时的端口，下面的9200则是查看数据的端口，ip为安装es的ip

启动项目后，访问http://ip:9200   ip是安装elasticsearch服务的机器ip，如果出现以下信息即代表安装成功

　　

 {
     "name": "kSs9lbx",
     "cluster_name": "elasticsearch",
     "cluster_uuid": "Wu6n-WzvQuecDEMWspm_aA",
     "version": {
         "number": "5.6.9",
         "build_hash": "877a590",
         "build_date": "2018-04-12T16:25:14.838Z",
         "build_snapshot": false,
         "lucene_version": "6.6.1"
     },
     "tagline": "You Know, for Search"
 }

2.创建java的springboot工程

　　2.1pom.xml文件依赖

　　

<parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>1.5.2.RELEASE</version>
    <relativePath/> <!-- lookup parent from repository -->
</parent>

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.version>1.8</java.version>
    <fastjson.version>1.2.31</fastjson.version>
    <jackson.version>2.8.7</jackson.version>
</properties>

<dependencies>
    <!-- set jetty server -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        <exclusions>
            <exclusion>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-tomcat</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-configuration-processor</artifactId>
        <optional>true</optional>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-jetty</artifactId>
    </dependency>
    <!-- aop -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-aop</artifactId>
    </dependency>
    <!-- json处理 -->
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>${fastjson.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/net.sourceforge.javacsv/javacsv -->
    <dependency>
        <groupId>net.sourceforge.javacsv</groupId>
        <artifactId>javacsv</artifactId>
        <version>2.0</version>
    </dependency>
    <!-- jackson json begin -->
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-core</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-annotations</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <dependency>
        <groupId>com.fasterxml.jackson.module</groupId>
        <artifactId>jackson-module-jaxb-annotations</artifactId>
        <version>${jackson.version}</version>
    </dependency>
    <!-- jackson json end -->
    <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-lang3</artifactId>
        <version>3.3.2</version>
    </dependency>
    <!-- druid datasource -->
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>druid</artifactId>
        <version>1.0.27</version>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-validation</artifactId>
    </dependency>
    <dependency>
        <groupId>commons-io</groupId>
        <artifactId>commons-io</artifactId>
        <version>2.4</version>
    </dependency>
    <dependency>
        <groupId>com.github.theborakompanioni</groupId>
        <artifactId>thymeleaf-extras-shiro</artifactId>
        <version>${thymeleaf-shiro.version}</version>
    </dependency>
    <!-- mysql驱动 -->
    <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <scope>runtime</scope>
    </dependency>
    <!-- spring boot 开发工具 -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-devtools</artifactId>
        <optional>true</optional>
    </dependency>
    <!-- 测试 -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
        <scope>test</scope>
    </dependency>
    <!--elasticsearrch-->
    <dependency>
        <groupId>org.elasticsearch</groupId>
        <artifactId>elasticsearch</artifactId>
        <version>5.6.9</version>
    </dependency>

    <dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>transport</artifactId>
        <version>5.6.9</version>
    </dependency>

    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-data-jpa</artifactId>
    </dependency>

</dependencies>

<repositories>
    <repository>
        <id>aliyun</id>
        <name>aliyun</name>
        <url>http://maven.aliyun.com/nexus/content/groups/public</url>
    </repository>
</repositories>

<build>
    <plugins>
        <plugin>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-maven-plugin</artifactId>
            <configuration>
                <includeSystemScope>true</includeSystemScope>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>2.18.1</version>
            <configuration>
                <skipTests>true</skipTests>
            </configuration>
        </plugin>
    </plugins>
</build>

　　2.2application.yml文件配置

　　

server:
  port: 9999
es:
  host: 127.0.0.1
  port: 9300
spring:
  datasource:
    initialize: false #false时不进行数据库表结构和数据初始化
    username: root
    #password: root
    password: root
    url: jdbc:mysql://localhost:3306/estest?createDatabaseIfNotExist=true&autoReconnect=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
    type: com.alibaba.druid.pool.DruidDataSource
    #连接池配置
    driverClassName: com.mysql.jdbc.Driver
    # 初始化大小，最小，最大
    initialSize: 5
    minIdle: 5
    maxActive: 50
    # 配置获取连接等待超时的时间
    maxWait: 60000
    # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
    timeBetweenEvictionRunsMillis: 60000
    # 配置一个连接在池中最小生存的时间，单位是毫秒
    minEvictableIdleTimeMillis: 300000
    validationQuery: SELECT 1 FROM DUAL
    testWhileIdle: true
    testOnBorrow: false
    testOnReturn: false
    # 打开PSCache，并且指定每个连接上PSCache的大小
    poolPreparedStatements: true
    maxPoolPreparedStatementPerConnectionSize: 20
    # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙
    filters: stat,log4j
    # 通过connectProperties属性来打开mergeSql功能；慢SQL记录
    connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000

　　2.3 创建elasticsearch的配置类

　　

 package spakt.demo.config;

 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

 import java.net.InetAddress;
 import java.net.UnknownHostException;

 @Configuration
 public class EsConfiguration {

     private String clusterName = "elasticsearch";
     @Value("${es.host}")
     private String esHost;

     @Value("${es.port}")
     private String esPort;

     @Bean
     public TransportClient getTransportClient() throws UnknownHostException {

         Settings sets = Settings.builder().put("cluster.name", clusterName) //集群名字
                 .put("client.transport.sniff", true)
                 .build();
         InetSocketTransportAddress inet = new InetSocketTransportAddress(InetAddress.getByName(esHost), Integer.parseInt(esPort));
         TransportClient transportClient = new PreBuiltTransportClient(sets).addTransportAddress(inet);
         return transportClient;
     }
 }

　　这儿注意 如果是es6.x以后的版本 ，在引入es包之后，还要引入netty插件包，这儿以6.7.2为例

　　

         <!-- https://mvnrepository.com/artifact/org.elasticsearch.plugin/transport-netty4-client -->
         <dependency>
             <groupId>org.elasticsearch.plugin</groupId>
             <artifactId>transport-netty4-client</artifactId>
             <version>6.7.2</version>
         </dependency>

　　并且上面的配置中InetSocketTransportAddress 修改为TransportAddress即可 具体如下

　　

 　　 @Bean
     public TransportClient initTransportClient() throws UnknownHostException {
         Settings build = Settings.builder().put("cluster.name", clusterName).put("client.transport.sniff", true).build();
         TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(address),port);
         TransportClient transportClient = new PreBuiltTransportClient(build).addTransportAddress(transportAddress);
         return transportClient;
     }

　　

　　2.4 创建测试的controller类  包含添加，查询，删除方法

　　

 package spakt.demo.elasticsearch;

 import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
 import spakt.demo.bean.Prisoner;

 import javax.annotation.Resource;
 import java.text.DecimalFormat;
 import java.util.*;

 /**
  *
  */
 @Controller
 public class EsController {

 //    @Resource(name = "jdbc2")
     @Autowired
     private JdbcTemplate jdbcTemplate;

     @Autowired
     private TransportClient transportClient;

     /**
      * 插入数据方法
      *
      * @return
      */
     @RequestMapping("/es")
     @ResponseBody
     public Object addEs() {

         //要插入的数据
         List<Map<String, Object>> prisonerListList = jdbcTemplate.
                 queryForList("select id,name,fact,family_address as address from da_prisoner ");
         //批处理类
         BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
         int count = 0;
         for (Map<String, Object> prisoner : prisonerListList) {

             //插入方法
             //jyplatform 代表index   理解为数据库中的库
             //prisoner   代表type   理解为数据库总的表
             //UUID  个人理解是为数据库中的主键
             String uuid = UUID.randomUUID().toString().replace("-", "");
             IndexRequestBuilder indexRequestBuilder = transportClient.
                     prepareIndex("jyplatform", "prisoner",uuid)
                     .setSource(prisoner);
             //单个执行
             //IndexResponse indexResponse = indexRequestBuilder.get();

             //批量执行 将添加器加入批处理中
             bulkRequestBuilder.add(indexRequestBuilder);
             if (count % 5000 == 0) {
                 //每隔5000条执行一次批量插入的操作
                 BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();
                 System.err.println(bulkItemResponses.getTook());
                 float num = (float) count / prisonerListList.size();
                 DecimalFormat df = new DecimalFormat("0.00");
                 String s = df.format(num * 100);
                 System.err.println("================================" + s + "%");
             }
             count++;
         }
         return prisonerListList;
     }

     /**
      * 查询数据
      *
      * @return
      */
     @RequestMapping("/get")
     @ResponseBody
     public Object getMap(String content) {
         List<Prisoner> prisonerList = new ArrayList<>();
         // 构造搜索条件
         BoolQueryBuilder query = QueryBuilders.boolQuery();
         //fact 即是field  理解为表中的字段
         query.filter(QueryBuilders.matchPhraseQuery("fact", content));
         //开始查询
         SearchRequestBuilder searchRequestBuilder = transportClient
                 .prepareSearch("jyplatform")   //确定index
                 .setTypes("prisoner")   //确定type
                 .setFrom(0)  //从多少条开始  分页使用
                 .setSize(10000) //查多少条  分页使用 size-from>10000会报异常
                 .setQuery(query); //加入查询条件
         //结果高亮显示
         HighlightBuilder highlightBuilder = new HighlightBuilder();
         highlightBuilder.field("fact");
         highlightBuilder.preTags("<span style=\"color:red\">");
         highlightBuilder.postTags("</span>");
         searchRequestBuilder.highlighter(highlightBuilder);
         //获取查询结果
         SearchResponse searchResponse = searchRequestBuilder.get();
         if (searchResponse.status() != RestStatus.OK) {
             System.err.println("查询结果错误");
             return "error";
         }
         //得到查询的结果
         SearchHits hits = searchResponse.getHits();
         for (SearchHit hit : hits) {
             //进行高亮处理
             String fact = "";
             Map<String, HighlightField> highlightFields = hit.getHighlightFields();
             HighlightField field = highlightFields.get("fact");
             if(field!=null){
                 Text[] fragments = field.fragments();
                 for (Text fragment : fragments) {
                     fact+=fragment.toString();
                 }
             }

             prisonerList.add(new Prisoner(hit.getSource().get("id").toString(),
                                           hit.getSource().get("name").toString(),
                                           StringUtils.isBlank(fact)?hit.getSource().get("fact").toString():fact,
                                           hit.getSource().get("address").toString()));
         }
         return prisonerList;
     }

     /**
      * 删除数据
      */
     @RequestMapping("/delete")
     @ResponseBody
     public String delete() {
         //开始查询
         SearchRequestBuilder searchRequestBuilder = transportClient
                 .prepareSearch("jyplatform") //确定index
                 .setTypes("prisoner") //确定type
                 .setFrom(0)  // begin
                 .setSize(10000); //size  size-from>10000会报异常
         //获取查询结果
         SearchHits hits = searchRequestBuilder.get().getHits();
         //批量操作
         BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
         for (SearchHit hit : hits) {
             //删除方法
             //jyplatform 代表index   理解为数据库中的库
             //prisoner   代表type   理解为数据库总的表
             // hit.getId()  这条数据的id
             DeleteRequestBuilder deleteRequestBuilder = transportClient.
                     prepareDelete("jyplatform", "prisoner", hit.getId());
             //批量删除
 //                bulkRequestBuilder.add(deleteRequestBuilder);
 //                if(i%5000 ==0){
 //                    bulkRequestBuilder.execute();
 //                }
             //单个操作
             deleteRequestBuilder.get();
         }
         return "OK";
     }

 }

　　2.5 创建启动类启动测试

　　

 package spakt.demo;

 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.ImportResource;

 @SpringBootApplication
 @ImportResource(value = "classpath:spring-*.xml")
 public class DemoApplication {

     public static void main(String[] args) {
         SpringApplication.run(DemoApplication.class, args);
     }
 }

总结

　　目前只学习了下初步的使用，复杂的应用待后面继续挖掘。集群也，还有分词器一大堆东西待待填坑