关于BufferedWriter.write超过30W条数据写入过慢问题。

原创文章，转载请注明出处！

------------------------------------------------------------

今天接到一个项目需求变更，是关于从数据库查询到30W条数据放到一个List中，然后使用StringBuffer把List取出放入到StringBuffer中。最后使用 BufferedWriter.write(buffer.toString());添加到文件中，此时会消耗几个小时才能把数据存放到文件中。

项目源码：

 public void generateCommonSMSFile(String localFilePath,String date) throws Exception{
         try {
             List smsList = getSingleDataSMS("03",date);
             List listid = new ArrayList();
             List subList = new ArrayList();
             StringBuffer buffer = new StringBuffer("S|2|S0355||||");
             buffer.append(smsList.size() + "|");
             buffer.append("\r\n");
             if(smsList != null && smsList.size() != 0) {
                 for(int i = 0;i < smsList.size();i++) {
                     OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i);
                     String transDate = msdd.getTransDate();
                     String leastDate = msdd.getLeastDate();
                     buffer.append(String.valueOf(i+1));
                     buffer.append("|");
                     buffer.append(msdd.getMobile());
                     buffer.append("|");
                     buffer.append(msdd.getCustNo());
                     buffer.append("|");
                     buffer.append(msdd.getAdvMemo1());
                     buffer.append("|");
                     buffer.append(CommonUtil.deleteZero(transDate.substring(5, 7))+"月"+CommonUtil.deleteZero(transDate.substring(8))+"日");
                     buffer.append("|");
                     buffer.append(msdd.getTransAmount());
                     buffer.append("|");
                     buffer.append(CommonUtil.deleteZero(leastDate.substring(4, 6))+"月"+CommonUtil.deleteZero(leastDate.substring(6))+"日");
                     buffer.append("|");
                     buffer.append("FQ"+msdd.getConsumeCode());
                     buffer.append("|");
                     buffer.append(msdd.getInstNum());
                     buffer.append("|");
                     buffer.append(msdd.getFirstAmount());
                     buffer.append("|");
                     buffer.append(msdd.getEachAmount());
                     buffer.append("|");
                     buffer.append(msdd.getTotalPoundage());
                     buffer.append("|");
                     buffer.append(msdd.getAdvMemo2());
                     buffer.append("|");
                     buffer.append("\r\n");
                     subList.add(msdd.getSeqId());
                     if((i+1)%5000==0){
                         listid.add(subList);
                         subList=new ArrayList();
                     }
                 }
                 listid.add(subList);
             }
             File file1 = FileUtils.createFile(localFilePath);
             BufferedWriter bw = FileUtils.getWrite(file1);
             bw.write(buffer.toString());
             bw.close();
             if(listid != null && listid.size() >= 1) {
                 for(int i=0;i<listid.size();i++) {
                     OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行开始");
                     long msBefore = System.currentTimeMillis();

                     updateMosSingleData((List)listid.get(i));

                     long msAfter = System.currentTimeMillis();
                     OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒");
                 }
             }
         } catch (Exception e) {
             OMSLogger.error("生成单笔分期批量短信文件异常");
             throw new Exception("生成单笔分期批量短信文件异常",e);
         }

     }

查询数据库，返回list供上面的数据写入文件的方法使用

 private List getSingleDataSMS(String instFlag,String vdate) throws PafaDAOException {

         List SmsList = new ArrayList();
         List list = new ArrayList();
         OmsSingleDataDTO msdd = null;
 //        list = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",null);
         HashMap map =new HashMap();
         map.put("instFlag", instFlag);
         map.put("vdate", vdate);
         int total = 0;
         total = ((Integer)executeSqlDao.queryForObject("SELECT-SMS-SingleData-COUNT", map)).intValue();
 //            防止大于1万条时出错,循环处理
         for(int i=0; i<total; i+=5000){
             HashMap paramMap =new HashMap();
             List rs = new ArrayList();
             paramMap.put("startNum", String.valueOf(i+1));
             paramMap.put("endNum", String.valueOf(i+5000));
             paramMap.put("instFlag", instFlag);
             paramMap.put("vDate", vdate);
             rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap);
             if (rs!=null){
                 for(int j=0; j<rs.size(); j++){
                     list.add(rs.get(j));
                 }
             }
         }

         if(list != null && list.size() >= 1) {
             Set set = commonService.getLogoValue(null);
             for(int m = 0;m < list.size();m++) {
                 msdd = (OmsSingleDataDTO)list.get(m);
                 if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) {
                     // 更新处理状态
                     this.updateMosSingleData2(msdd.getSeqId());
                     continue;
                 }
                 String custNo = msdd.getCustNo();
                 custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo;
                 msdd.setCustNo(custNo);
                 SmsList.add(msdd);
             }
         }

         return SmsList;
     }

以上是使用的一次全部查询出数据，然后一次写入到文件中，但是这种方法会用几个小时才能完全写入。下面是自己优化后的方案：

 public void generateSMSFile(String localFilePath,String date) throws Exception{
         try {
             File file1 = FileUtils.createFile(localFilePath);
             BufferedWriter bw = FileUtils.getWrite(file1);
             StringBuffer buffer = new StringBuffer("S|1|S0180||||");

             List smslistCount = getSingleDataSMS("03", date);
             buffer.append(smslistCount.size() + "|");

             buffer.append("\r\n");

             List subList = new ArrayList();
             List listid = new ArrayList();
             int count = 1;
             int total = 0;
             total = generateSMSFilestpeSize(date);

             //防止大于1万条时出错,循环处理
             for(int k=0; k<total; k+=5000){
                 List smsList = getSingleDataSMSstep("03", date, k, k+5000);
                 if(smsList != null && smsList.size() != 0) {
                     for(int i = 0;i < smsList.size();i++) {
                         OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i);
                         String transDate = msdd.getTransDate();
                         String leastDate = msdd.getLeastDate();
                         count = count +1;
                         buffer.append(String.valueOf(count));
                         buffer.append("|");
                         buffer.append(msdd.getMobile());
                         buffer.append("|");
                         buffer.append(msdd.getCustNo());
                         buffer.append("|");
                         buffer.append(transDate.substring(5,7));
                         buffer.append("|");
                         buffer.append(transDate.substring(8,10));
                         buffer.append("|");
                         buffer.append(msdd.getTransAmount());
                         buffer.append("|");
                         buffer.append(msdd.getFirstAmount());
                         buffer.append("|");
                         buffer.append(msdd.getEachAmount());
                         buffer.append("|");
                         buffer.append(leastDate.substring(4,6));
                         buffer.append("|");
                         buffer.append(leastDate.substring(6,8));
                         buffer.append("|");
                         buffer.append(msdd.getConsumeCode());
                         buffer.append("|");
                         buffer.append("\r\n");
                         subList.add(msdd.getSeqId());
                         if((i+1)%5000==0){
                             listid.add(subList);
                             subList=new ArrayList();
                         }
                     }
                     listid.add(subList);
                 }
                 bw.write(buffer.toString());
                 buffer = new StringBuffer();
                 if(listid != null && listid.size() >= 1) {
                     for(int i=0;i<listid.size();i++) {
                         OMSLogger.info("保费类 - 第"+ (count) +"次运行开始");
                         long msBefore = System.currentTimeMillis();

                         updateMosSingleData((List)listid.get(i));

                         long msAfter = System.currentTimeMillis();
                         OMSLogger.info("保费类 - 第"+ (count) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒");
                     }
                 }
             }
             bw.close();

         } catch (Exception e) {
             OMSLogger.error("生成单笔分期批量短信文件异常");
             throw new Exception("生成单笔分期批量短信文件异常",e);
         }
     }

每次获取5000条数据返回list，写入文件之后再次执行该方法在获取5000条数据

     private List getSingleDataSMSstep(String instFlag,String vdate,int startNum,int endNum) throws PafaDAOException {

         List SmsList = new ArrayList();
         List list = new ArrayList();
         OmsSingleDataDTO msdd = null;
         HashMap paramMap =new HashMap();
         List rs = new ArrayList();
         paramMap.put("startNum", startNum);
         paramMap.put("endNum", endNum);
         paramMap.put("instFlag", instFlag);
         paramMap.put("vDate", vdate);
         rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap);
         if (rs!=null){
             for(int j=0; j<rs.size(); j++){
                 list.add(rs.get(j));
             }
         }

         if(list != null && list.size() >= 1) {
             Set set = commonService.getLogoValue(null);
             for(int m = 0;m < list.size();m++) {
                 msdd = (OmsSingleDataDTO)list.get(m);
                 if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) {
                     // 更新处理状态
                     this.updateMosSingleData2(msdd.getSeqId());
                     continue;
                 }
                 String custNo = msdd.getCustNo();
                 custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo;
                 msdd.setCustNo(custNo);
                 SmsList.add(msdd);
             }
         }

         return SmsList;
     }

补充一个自己写的小例子：

　　

 package myTempTest;

 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;

 public class ioTestBuffer {
     public static void main(String[] args) throws IOException {
         File file1 = new File("d:\\io\\out.txt");//最终写入文件地址
         BufferedWriter bw = new BufferedWriter(new FileWriter(file1));
         StringBuffer buffer = new StringBuffer("");
         for (int j = ; j < ; j++) {
             //之所以I循环1W次就写出一次，是因为buffer超过1W次append之后极容易在次append的时候报错。
             for (int i = ; i < ; i++) {
                 buffer.append(i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+"");
                 buffer.append("\r\n");//换行
             }
             bw.write(buffer.toString());//1W次I循环结束 写入到文件中
             buffer = new StringBuffer("");//格式化buffer。
             buffer.append(j+"-------------------------------------");//第 j 此循环加上个记号
             buffer.append("\r\n");//一次J循环结束 换行
         }
         bw.close();
     }
 }