某一日，
他
谁？
客户！
突然说系统上传附件失败了;
查了一下日志发现两个节点都已经挂了
日志如下
节点01
image.png
节点02
image.png
存在进程但是无响应能力
开始简单的以为只是应用太繁忙导致消耗了太多内存导致不够用
企业微信截图_17358685051781.png
所以说还是太年轻了！！！！！！！！没有看到问题的本质，直到领导要求要排查根本原因在重视起来看这块逻辑到底做了啥。
开始是通过日志，以及gc 日志排查，完全看不出来原因，再尔看了代码如下文件服务Pdf2ImageServiceImpl，咋一看好像也没啥毛病，该关闭的流也关闭了，不至于出现内存泄漏回收不掉，到底是为啥？？？？百思不得琪姐，why ?
@Override
public PdfToImageResp doTransition(PdfToImageReq req) throws Exception {
long startTime = System.currentTimeMillis();
bizLog.info("Pdf2Image,fileId:{}", req.getFileId());
BaseFile baseFile = baseFileService.getById(req.getFileId());
ExceptionUtils.businessException(Objects.isNull(baseFile), "文件不存在");
ExceptionUtils.businessException(!FileContentTypeEnums.PDF.getValue().equalsIgnoreCase(baseFile.getFileContentType()), "文件必须为PDF文件");
File localPdf = new File(FileUtil.genTempName(fileConfig.getFileTempPath(), baseFile.getId() + "." + FileContentTypeEnums.PDF.getValue()));
try {
fileClient.download(baseFile.getId(), localPdf);
ExceptionUtils.businessException(!localPdf.exists(), "文件下载失败");
ExceptionUtils.businessException(localPdf.length() > FILE_MAX_SIZE, String.format("材料文件大小 %s ,最大可支持 %s", localPdf.length(), FILE_MAX_SIZE));
int totalWidth = 0; // 总宽度
int totalHeight = 0; //总高度
List<BufferedImage> itemImages = new ArrayList<>();
try (PDDocument document = PDDocument.load(localPdf)) {
int pageCount = document.getNumberOfPages();
PDFRenderer renderer = new PDFRenderer(document);
for (int i = 0; i < pageCount; i++) {
BufferedImage image = renderer.renderImageWithDPI(i, DEFAULT_DPI);
int imageHeight = image.getHeight();
int imageWidth = image.getWidth();
totalHeight += imageHeight;
totalWidth = Math.max(totalWidth, imageWidth);
itemImages.add(image);
}
}
int shiftHeight = 0; // 当前偏移高度
BufferedImage imageResult = new BufferedImage(totalWidth, totalHeight, BufferedImage.TYPE_INT_RGB);// 保存每张图片的像素值
for (int i = 0; i < itemImages.size(); i++) {
BufferedImage image = itemImages.get(i);
int imageHeight = image.getHeight();
int imageWidth = image.getWidth();
int[] singleImgRGB = image.getRGB(0, 0, imageWidth, imageHeight, null, 0, imageWidth);// 一张图片中的RGB数据
if (i > 0) {
shiftHeight += imageHeight;// 计算高度偏移量,第一页从顶部开始写入流
}
imageResult.setRGB(0, shiftHeight, totalWidth, imageHeight, singleImgRGB, 0, totalWidth); // 写入流中
}
try (ByteArrayOutputStream ops = new ByteArrayOutputStream()) {
ImageIO.write(imageResult, FileContentTypeEnums.PNG.getValue(), ops);// 写图片到ops
try (ByteArrayInputStream ips = new ByteArrayInputStream(ops.toByteArray())) {
FileUploadReq fileUploadReq = new FileUploadReq();
fileUploadReq.setFileName(baseFile.getFileName() + "." + FileContentTypeEnums.PNG.getValue());
fileUploadReq.setGroup(FileGroupEnums.getEnumsByValue(baseFile.getGroupId()));
fileUploadReq.setContentType(FileContentTypeEnums.PNG);
fileUploadReq.setBizId(baseFile.getId());
fileUploadReq.setRemark("doTransitionFrom:" + baseFile.getId());
FileUploadResp imageFileInfo = fileClient.upload(fileUploadReq, ips, ips.available());
return PdfToImageResp.builder().pdfFileId(baseFile.getId()).imageFileId(imageFileInfo.getFileId()).build();
}
}
} finally {
FileUtils.forceDeleteOnExit(localPdf);
bizLog.info("Pdf2Image,fileId:{} 结束,用时:{} ms", req.getFileId(), System.currentTimeMillis() - startTime);
}
}
直到，看到它，原附件
这附件居然有54 页！！！结合代码大概猜到端倪了，每页都要单独读取转成img 放在内存里面，不大才怪。
企业微信截图_17364133392282.png
验证：
public static void main(String[] args)throws Exception {
Runtime runtime = Runtime.getRuntime();
long totalMemory = runtime.totalMemory();
long freeMemory = runtime.freeMemory();
long usedMemory = totalMemory - freeMemory;
System.out.println("usedMemory1:"+usedMemory/(1024*1024));
long startTime = System.currentTimeMillis();
File localPdf = new File("C:\\Users\\bb\\Documents\\1874788094250815505.pdf");
try {
int totalWidth = 0; // 总宽度
int totalHeight = 0; //总高度
List<BufferedImage> itemImages = new ArrayList<>();
try (PDDocument document = PDDocument.load(localPdf)) {
int pageCount = document.getNumberOfPages();
PDFRenderer renderer = new PDFRenderer(document);
for (int i = 0; i < pageCount; i++) {
BufferedImage image = renderer.renderImage(i,4F);
int imageHeight = image.getHeight();
int imageWidth = image.getWidth();
totalHeight += imageHeight;
totalWidth = Math.max(totalWidth, imageWidth);
itemImages.add(image);
}
}
long freeMemory2 = runtime.freeMemory();
long usedMemory2 = usedMemory - freeMemory2;
System.out.println("usedMemory2:"+usedMemory2/(1024*1024));
int shiftHeight = 0; // 当前偏移高度
BufferedImage imageResult = new BufferedImage(totalWidth, totalHeight, BufferedImage.TYPE_INT_RGB);// 保存每张图片的像素值
for (int i = 0; i < itemImages.size(); i++) {
BufferedImage image = itemImages.get(i);
int imageHeight = image.getHeight();
int imageWidth = image.getWidth();
int[] singleImgRGB = image.getRGB(0, 0, imageWidth, imageHeight, null, 0, imageWidth);// 一张图片中的RGB数据
if (i > 0) {
shiftHeight += imageHeight;// 计算高度偏移量,第一页从顶部开始写入流
}
try {
imageResult.setRGB(0, shiftHeight, totalWidth, imageHeight, singleImgRGB, 0, totalWidth); // 写入流中
}catch (Exception e){
}
}
long freeMemory3 = runtime.freeMemory();
long usedMemory3 = usedMemory2 - freeMemory3;
System.out.println("usedMemory3:"+usedMemory3/(1024*1024));
File file = new File("C:\\Users\\bb\\Documents\\whh.png");
try ( FileOutputStream ops = new FileOutputStream(file)) {
ImageIO.write(imageResult, FileContentTypeEnums.PNG.getValue(), ops);// 写图片到ops
}
long freeMemory4 = runtime.freeMemory();
long usedMemory4 = usedMemory3 - freeMemory4;
System.out.println("usedMemory4:"+usedMemory4/(1024*1024));
} finally {
bizLog.info("Pdf2Image,fileId:{} 结束,用时:{} ms", "", System.currentTimeMillis() - startTime);
}
}
验证结果：6.8m 的pdf 居然要这么多内存，变天了，怪不得生产给了一个G 都要挂
image.png
那怎么办？
能不能压缩？试了不行
试了回收对象出发gc 还是不行
要不换个框架？找了半天没有发现合适的
或者搞个大文件，小页码的附件？有效果，但是不明显；算了放弃了吧能加内存改什么代码！！！
喵的！那就先分页，在合并，分步做，看能不能回收对象。以时间换空间
public static void main(String[] args) {
// PDF文件路径
String pdfFilePath ="C:\\Users\\bb\\Documents\\1874788094250815505.pdf";
// 输出图片文件夹路径
String outputDir = "C:\\Users\\bb\\Documents\\";
String imageMerge = "C:\\Users\\bb\\Documents\\imageMergewhh.png";
ArrayList<String> arrayList = new ArrayList<>();
// 设置DPI（越高图片越清晰，但文件也会更大）
int dpi = 100;
extracted(pdfFilePath, outputDir, arrayList, dpi);
imageMerge(arrayList.get(0),arrayList.get(1),imageMerge);
for (int i=2;i<arrayList.size();i++){
imageMerge(imageMerge,arrayList.get(i),imageMerge);
System.gc();
}
}
private static void extracted(String pdfFilePath, String outputDir, ArrayList<String> arrayList, int dpi) {
try (PDDocument document = PDDocument.load(new File(pdfFilePath))) {
PDFRenderer pdfRenderer = new PDFRenderer(document);
BufferedImage bim = null;
// 遍历PDF每一页并转换为图片
for (int page = 0; page < document.getNumberOfPages(); ++page) {
// 使用BufferedImage来表示图像
bim = pdfRenderer.renderImageWithDPI(page, dpi);
// 生成文件名
String fileName = outputDir + "pdf_page_" + (page + 1) + ".png";
arrayList.add(fileName);
// 将图片保存为PNG格式
ImageIO.write(bim, "png", new File(fileName));
System.out.println("Saved page " + (page + 1) + " as image.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
private static void imageMerge(String imgpath1,String imgpath2,String imageMerge){
try {
// 加载第一张图片
BufferedImage image1 = ImageIO.read(new File(imgpath1));
// 加载第二张图片
BufferedImage image2 = ImageIO.read(new File(imgpath2));
int width = image1.getWidth();
int height = image1.getHeight()+image2.getHeight();
// 创建一个新的buffered image，用于合并两张图片
BufferedImage combinedImage = new BufferedImage(
width, // 宽度
height, // 高度取决于哪张图片更高
BufferedImage.TYPE_INT_RGB // 图片类型
);
// 绘制第一张图片到合并图片
Graphics2D g2d1 = combinedImage.createGraphics();
g2d1.drawImage(image1, 0, 0, image1.getWidth(),image1.getHeight(),null);
g2d1.drawImage(image2, 0,image1.getHeight(),image2.getWidth(), image2.getHeight(), null);
g2d1.dispose();
// 保存合并后的图片
ImageIO.write(combinedImage, "png", new File(imageMerge));
image1=null;image2=null;combinedImage=null;g2d1=null;
} catch (IOException e) {
e.printStackTrace();
}
}
和大佬验证了一下
主动触发gc
企业微信截图_17364983571973.png
不主动触发gc
企业微信截图_17364987083534.png
差异：
主动触发gc 的内存曲线要陡一点，说明pdf 每页读成img 时有效的回收掉无用对象，后面合并img 时所需的内存都一样作业紧张无法回收；
合并后的图片居然有十多M 比原始pdf 还大，很大。
image.png
上面算法执行结果是：耗时：16:27:54 ----- 16:31:40 3分45秒，内存使用如图
image.png
上面有个弊端不断地合并图片导致图片不断增大还每次加载到内存，结果如图不断地增加内存使用过量；
优化：
使用像二分法类似 先将页码两两合成小文件，不断地两两合并，直至最后一个
public static void main(String[] args) {
// PDF文件路径
String pdfFilePath ="C:\\Users\\bb\\Documents\\1874788094250815505.pdf";
// 输出图片文件夹路径
String outputDir = "C:\\Users\\bb\\Documents\\";
String imageMerge = "C:\\Users\\bb\\Documents\\imageMergewhh.png";
ArrayList<String> arrayList = new ArrayList<>();
ArrayList<String> imageMergeList = new ArrayList<>();
// 设置DPI（越高图片越清晰，但文件也会更大）
int dpi = 100;
extracted(pdfFilePath, outputDir, arrayList, dpi);
String lastpath = bisectionMerge(outputDir, arrayList, imageMergeList, 0);
}
private static String bisectionMerge (String imageMerge, ArrayList<String> arrayList, ArrayList<String> imageMergeList ,int count) {
String lastpath = null;
if(arrayList.size()%2 == 1){
lastpath = arrayList.remove(arrayList.size()-1);
}
for (int i = 0; i< arrayList.size(); i = i+2){
imageMergeList.add(imageMerge+"merge"+count+i+".png");
imageMerge(arrayList.get(i), arrayList.get(1+i), imageMerge+"merge"+count+i+".png");
System.gc();
}
if(lastpath !=null){
imageMergeList.add(lastpath);
}
arrayList.clear(); arrayList.addAll(imageMergeList);imageMergeList.clear();
if (arrayList.size()==1){
return arrayList.get(0);
}
count++;
return bisectionMerge(imageMerge,arrayList,imageMergeList,count);
}
private static void imageMerge(String imgpath1,String imgpath2,String imageMerge){
try {
// 加载第一张图片
BufferedImage image1 = ImageIO.read(new File(imgpath1));
// 加载第二张图片
BufferedImage image2 = ImageIO.read(new File(imgpath2));
int width = image1.getWidth();
int height = image1.getHeight()+image2.getHeight();
// 创建一个新的buffered image，用于合并两张图片
BufferedImage combinedImage = new BufferedImage(
width, // 宽度
height, // 高度取决于哪张图片更高
BufferedImage.TYPE_INT_RGB // 图片类型
);
// 绘制第一张图片到合并图片
Graphics2D g2d1 = combinedImage.createGraphics();
g2d1.drawImage(image1, 0, 0, image1.getWidth(),image1.getHeight(),null);
g2d1.drawImage(image2, 0,image1.getHeight(),image2.getWidth(), image2.getHeight(), null);
g2d1.dispose();
// 保存合并后的图片
ImageIO.write(combinedImage, "png", new File(imageMerge));
image1=null;image2=null;combinedImage=null;g2d1=null;
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
FileUtils.forceDelete(new File(imgpath1));
FileUtils.forceDelete(new File(imgpath2));
} catch (IOException e) {
e.printStackTrace();
}
}
}
执行结果耗时：16:34:57 ---- 16:36:20 一分25秒 内存占用如图，只有到后面的图片变大的时候才会占用一些内存，内存上限也比优化前少了20%以上，高内存的时间大幅缩短。垃圾回收次数也减少了很多，算是一个明显的提升
image.png
