新增数据导出接口，可传入时间过滤字段和时间范围

soc-modules/soc-modules-system/pom.xml

@@ -113,7 +113,11 @@
             <groupId>com.xunmei</groupId>
             <artifactId>soc-common-websocket</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>org.apache.ant</groupId>
+            <artifactId>ant</artifactId>
+            <version>1.8.0</version>
+        </dependency>
     </dependencies>
 
     <build>

soc-modules/soc-modules-system/src/main/java/com/xunmei/system/controller/ExportSqlController.java

@@ -1,5 +1,7 @@
 package com.xunmei.system.controller;
 
+import cn.hutool.core.date.DateUtil;
+import com.xunmei.common.core.constant.Constants;
 import com.xunmei.common.log.annotation.Log;
 import com.xunmei.common.log.enums.BusinessType;
 import com.xunmei.system.service.ExportSqlService;
@@ -12,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletResponse;
+import java.util.Date;
 import java.util.List;
 
 @Slf4j
@@ -41,6 +44,18 @@ public class ExportSqlController {
         }
     }
 
+    @GetMapping("/table/filter/date/range")
+    public void exportDataBaseWithFilter(HttpServletResponse response,String tableName,String filterColumnName,String startDate,String endDate){
+        try {
+            //验证日期参数是否传入正常
+            Date start = DateUtil.parse(startDate, Constants.DAILY_FORMAT);
+            Date end = DateUtil.parse(endDate, Constants.DAILY_FORMAT);
+            exportSqlService.exportTableOneWithFilter(response,tableName,filterColumnName,DateUtil.format(start,Constants.DAILY_FORMAT),DateUtil.format(end,Constants.DAILY_FORMAT));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     @GetMapping("/tables")
     public void exportDataBase(HttpServletResponse response,@RequestParam List<String> tableNames){
         try {

soc-modules/soc-modules-system/src/main/java/com/xunmei/system/service/ExportSqlService.java

@@ -1,6 +1,7 @@
 package com.xunmei.system.service;
 
 import javax.servlet.http.HttpServletResponse;
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -22,6 +23,13 @@ public interface ExportSqlService {
     void exportTableOne(HttpServletResponse response,String tableName)throws Exception;
 
     /**
+     * 导出单表数据
+     * @param tableName
+     * @throws Exception
+     */
+    void exportTableOneWithFilter(HttpServletResponse response, String tableName, String filterColumnName, String startDate, String endDate)throws Exception;
+
+    /**
      * 导出多个表sql文件
      * @param tableNames
      * @throws Exception

soc-modules/soc-modules-system/src/main/java/com/xunmei/system/service/impl/ExportSqlServiceImpl.java

@@ -1,21 +1,33 @@
 package com.xunmei.system.service.impl;
 
+import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
+import com.xunmei.common.core.constant.Constants;
+import com.xunmei.common.core.domain.registerbook.vo.PdfToZipTempVo;
+import com.xunmei.common.core.utils.DateHelper;
 import com.xunmei.system.mapper.ObjectMapper;
 import com.xunmei.system.service.ExportSqlService;
 import com.xunmei.system.util.ExportDatabaseUtil;
 import com.xunmei.system.util.Sm4Util;
+import org.apache.tools.zip.ZipEntry;
+import org.apache.tools.zip.ZipOutputStream;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.system.ApplicationHome;
 import org.springframework.stereotype.Service;
 
+import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletResponse;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.OutputStream;
 import java.net.URLEncoder;
+import java.util.Date;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
 
 @Service
 public class ExportSqlServiceImpl implements ExportSqlService {
@@ -58,6 +70,22 @@ public class ExportSqlServiceImpl implements ExportSqlService {
     }
 
     @Override
+    public void exportTableOneWithFilter(HttpServletResponse response, String tableName, String filterColumnName, String startDate, String endDate) throws Exception {
+        ApplicationHome home = new ApplicationHome(getClass());
+        String path = home.getSource().getParent();
+        File file = new File(path,tableName+"_"+startDate +"_"+endDate+ ".sql");
+        if(file.exists()) {
+            file.delete();
+        }
+        //为方便处理日期边界问题 将结束日期加1天
+        Date end= DateUtil.parse(endDate);
+        String newEndDateStr =DateUtil.format(DateUtil.offsetDay(end, 1), Constants.DAILY_FORMAT);
+        ExportDatabaseUtil.exportTableSqlWithDateFilter(url,username,password,tableName,filterColumnName,startDate,newEndDateStr,file);
+        downloadCompressFile(response,file);
+    }
+
+
+    @Override
     public void exportTables(HttpServletResponse response, List<String> tableNames) throws Exception {
         ApplicationHome home = new ApplicationHome(getClass());
         String path = home.getSource().getParent();
@@ -119,4 +147,84 @@ public class ExportSqlServiceImpl implements ExportSqlService {
         }
     }
 
+        /**
+         * 导出数据
+         * @param response
+         * @param file
+         */
+        private void downloadCompressFile(HttpServletResponse response,File file){
+//        OutputStream os;//新建一个输出流对象
+            try {
+                response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(file.getName()+".zip", "UTF-8"));
+                response.setContentType("application/octet-stream");
+                ServletOutputStream outputStream = response.getOutputStream();
+                ZipOutputStream zos = new ZipOutputStream(outputStream);
+                zos.setEncoding("GBK");
+                FileInputStream fileInputStream = new FileInputStream(file);
+                zos.putNextEntry(new ZipEntry(file.getName()));
+                byte[] buffer = new byte[4096];
+                int len;
+                while ((len = fileInputStream.read(buffer)) > 0) {
+                    zos.write(buffer, 0, len);
+                }
+                zos.closeEntry();
+                fileInputStream.close();
+                zos.flush();
+                zos.close();
+                outputStream.close();
+            }catch (Exception e){
+                System.out.println("文件下载失败");
+            }finally {
+                if(file.exists()){
+                    file.delete();
+                }
+            }
+
+//        try {
+//            String zipName = URLEncoder.encode(pdfDto.getFileName() + DateHelper.getDateString(new Date()) + ".zip", "UTF-8");
+//            final CountDownLatch count = new CountDownLatch(registerBookPdfList.size());
+//            ServletOutputStream outputStream = response.getOutputStream();
+//            ZipOutputStream zos = new ZipOutputStream(outputStream);
+//            response.setContentType("application/octet-stream");
+//            response.setHeader("Content-Disposition", "attachment; filename=" + zipName);
+//            List<PdfToZipTempVo> pdfToZipTempVoList = registerBookPdfList.parallelStream().map(pdf -> {
+//                        return resolve(pdf, count);
+//                    }).filter(Objects::nonNull)
+//                    .collect(Collectors.toList());
+//            count.await();
+//            pdfToZipTempVoList.removeIf(pdfToZipTempVo -> !FileUtil.exist(pdfToZipTempVo.getFile()));
+//            log.info("登记簿全部下载完成,开始压缩文件,数量:{}", pdfToZipTempVoList.size());
+//            for (PdfToZipTempVo tempVo : pdfToZipTempVoList) {
+//                log.info("当前开始处理第{}个文件 ", pdfToZipTempVoList.indexOf(tempVo) + 1);
+//                deal(zos, tempVo);
+//            }
+//            zos.flush();
+//            zos.close();
+//            outputStream.close();
+//            log.info("登记簿批量导出压缩文件完成,文件数量:{}", pdfToZipTempVoList.size());
+//        } catch (Throwable e) {
+//            String errMsg = String.format("登记簿导出失败:%s", e);
+//            log.error(errMsg);
+//        } finally {
+//            final File temp = new File(TEMP_DIR_NAME);
+//            if (temp.exists()) {
+//                FileUtil.del(temp);
+//                log.info("临时目录已删除");
+//            }
+//        }
+    }
+
+    public void deal(ZipOutputStream zos, PdfToZipTempVo zipTempVo) throws Throwable {
+        zos.setEncoding("GBK");
+        FileInputStream fileInputStream = new FileInputStream(zipTempVo.getFile());
+        zos.putNextEntry(new ZipEntry(zipTempVo.getEntryName()));
+        byte[] buffer = new byte[4096];
+        int len;
+        while ((len = fileInputStream.read(buffer)) > 0) {
+            zos.write(buffer, 0, len);
+        }
+        zos.closeEntry();
+        fileInputStream.close();
+    }
+
 }

soc-modules/soc-modules-system/src/main/java/com/xunmei/system/util/ExportDatabaseUtil.java

@@ -1,5 +1,7 @@
 package com.xunmei.system.util;
 
+import cn.hutool.core.date.DateUtil;
+import com.xunmei.common.core.constant.Constants;
 import com.xunmei.common.security.utils.SecurityUtils;
 
 import java.io.BufferedWriter;
@@ -8,6 +10,7 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.sql.*;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 public class ExportDatabaseUtil {
@@ -118,6 +121,61 @@ public class ExportDatabaseUtil {
     }
 
     /**
+     * 根据表名和过滤字段导出指定间范围内的数据 sql语句
+     * @param url
+     * @param username
+     * @param password
+     * @param tableName 表名
+     * @param filterColumnName 过滤字段名
+     * @param startDate 开始日期
+     * @param endDate 结束日期
+     * @param file
+     */
+    public static void exportTableSqlWithDateFilter(String url,String username,String password,String tableName,String filterColumnName,String startDate,String endDate, File file){
+        Connection conn = null;
+        Statement stmt = null;
+        ResultSet rs = null;
+
+        try {
+            // 建立与数据库的连接
+            conn = DriverManager.getConnection(url, username, password);
+
+            // 创建Statement对象
+            stmt = conn.createStatement();
+
+            FileWriter fw = new FileWriter(file, false);
+            BufferedWriter bw = new BufferedWriter(fw);
+
+            //写入table
+            writeTableWithDateFilter(stmt,rs,bw,tableName,filterColumnName,startDate,endDate);
+            bw.flush();
+            bw.close();
+            // 关闭资源
+            fw.close();
+
+            System.out.println("数据库已成功导出！");
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            // 关闭资源
+            try {
+                if (rs != null) {
+                    rs.close();
+                }
+                if (stmt != null) {
+                    stmt.close();
+                }
+                if (conn != null) {
+                    conn.close();
+                }
+            } catch (SQLException ex) {
+                ex.printStackTrace();
+            }
+        }
+    }
+
+
+    /**
      * 根据表名导出sql语句
      * @param url
      * @param username
@@ -224,6 +282,44 @@ public class ExportDatabaseUtil {
     }
 
     /**
+     * 导出指定表名sql
+     * @param stmt
+     * @param rs
+     * @param bw
+     * @param tableName
+     * @throws Exception
+     */
+    private static void writeTableWithDateFilter(Statement stmt,ResultSet rs,BufferedWriter bw,String tableName,String filterColumnName,String startDate,String endDate) throws Exception {
+
+        //写入文件头部
+        wirteHeader(bw);
+
+        //写入表格导出头部
+        writeTableHeader(bw,tableName);
+
+        //写入按时间范围删除数据语句
+        writeDeleteTableDataWithFilterSql(bw,tableName,filterColumnName,startDate,endDate);
+
+        Date start = DateUtil.parse(startDate);
+        Date end = DateUtil.parse(endDate);
+
+        // 数据量较大时一次查询全部数据很慢且内存也会爆炸，改为按天查
+        for (Date current = start; !current.after(end); current = DateUtil.offsetDay(current, 1)) {
+            Date currentEnd=DateUtil.offsetDay(current, 1);
+            if(currentEnd.after(end))
+            {
+                return;
+            }
+            writeTableDateFilterInsertSql(stmt, rs, bw, tableName, filterColumnName,
+                    DateUtil.format(current,  Constants.DAILY_FORMAT),
+                    DateUtil.format(currentEnd,  Constants.DAILY_FORMAT));
+        }
+
+        //写入插入语句
+//        writeTableDateFilterInsertSql(stmt,rs,bw,tableName,filterColumnName,startDate,endDate);
+    }
+
+    /**
      * 写入sql语句
      * @param tables 数据库所有表名
      * @param stmt Statement对象
@@ -314,6 +410,23 @@ public class ExportDatabaseUtil {
     }
 
     /**
+     * 写入按时间范围删除表数据SQL
+     * @param bw
+     * @param tableName
+     * @param filterColumnName
+     * @param startDate
+     * @param endDate
+     * @throws SQLException
+     * @throws IOException
+     */
+    private static void writeDeleteTableDataWithFilterSql(BufferedWriter bw,String tableName,String filterColumnName,String startDate,String endDate) throws SQLException, IOException {
+        //写入删除表数据语句
+        String dropTableDataSql =String.format("DELETE FROM %s WHERE `%s` >= '%s' AND `%s`< '%s' ;",tableName,filterColumnName,startDate,filterColumnName,endDate);
+        bw.write(dropTableDataSql);
+        bw.newLine();
+    }
+
+    /**
      * 写入insert语句
      * @param stmt Statement对象
      * @param rs ResultSet结果对象
@@ -358,6 +471,57 @@ public class ExportDatabaseUtil {
     }
 
     /**
+     * 按过滤条件写入insert语句
+     * @param stmt
+     * @param rs
+     * @param bw
+     * @param tableName
+     * @param filterColumnName
+     * @param startDate
+     * @param endDate
+     * @throws SQLException
+     * @throws IOException
+     */
+    private static void writeTableDateFilterInsertSql(Statement stmt,ResultSet rs,BufferedWriter bw,String tableName,String filterColumnName,String startDate,String endDate) throws SQLException, IOException {
+        String columnStr = getColumnStr(stmt, tableName);
+
+//        String selectSql = "SELECT "+ columnStr +" FROM `" + tableName + "`";
+        String selectSql =String.format("SELECT %s FROM `%s` WHERE `%s` >= '%s' AND `%s`< '%s' ;",columnStr,tableName,filterColumnName,startDate,filterColumnName,endDate);
+        // 生成INSERT INTO语句
+        rs = stmt.executeQuery(selectSql);
+
+
+        int columnCount = rs.getMetaData().getColumnCount();
+        while (rs.next()) {
+            StringBuilder insertIntoBuilder = new StringBuilder();
+            insertIntoBuilder.append("INSERT INTO " ).append(tableName).append(" (").append(columnStr).append(") ")
+                    .append(" VALUES (");
+
+            for (int i = 1; i <= columnCount; i++) {
+                int columnType = rs.getMetaData().getColumnType(i);
+                String value = rs.getString(i);
+                if (value == null) {
+                    insertIntoBuilder.append("NULL");
+                } else {
+                    String val = dealColumnType(columnType,value);
+                    insertIntoBuilder.append(val);
+                }
+                if (i < columnCount) {
+                    insertIntoBuilder.append(", ");
+                }
+            }
+
+            insertIntoBuilder.append(");");
+
+            System.out.println(insertIntoBuilder.toString());
+            // 写入输出流
+            bw.write(insertIntoBuilder.toString());
+            bw.newLine();
+        }
+    }
+
+
+    /**
      * 获取数据库的所有表名
      * @param stmt Statement对象
      * @param rs 查询结果对象