记录下springboot集成aws云S3对象存储的大体步骤:
1.maven引入aws s3 sdk
2.编写配置类注入spring
3.编写业务类封装对象存储的基本调用方法
4.编写测试controller,调用测试
maven引入aws s3 sdk
挑选适配自身springboot版本的即可
<!-- S3 -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<!-- <version>2.16.60</version> -->
<!-- <scope>compile</scope> -->
</dependency>
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/aws-core -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>aws-core</artifactId>
<!-- <version>2.32.6</version> -->
</dependency>
编写配置类注入spring
配置yml里增加S3配置:
#-------------------S3----------------------------
aws:
accessKeyId: "xxx"
secretAccessKey: "xxx"
endpointUrl: "https://s3.us-east-2.amazonaws.com"
region: "us-east-2"
public-bucket: xxx
outerDomain: https://xxx
innerDomain: https://xxx
如上填写清楚S3的基础信息,以及需要操作的桶名称、需要返回的域名信息等
新增AwsS3Config:
import java.net.URI;
import java.net.URISyntaxException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
@Configuration
public class AwsS3Config {
@Value("${aws.accessKeyId}")
private String accessKeyId;
@Value("${aws.secretAccessKey}")
private String secretAccessKey;
@Value("${aws.endpointUrl}")
private String endpointUrl;
@Value("${aws.outerDomain}")
private String outerDomain;
@Value("${aws.innerDomain}")
private String innerDomain;
@Value("${aws.region}")
private String regionStr;
@Bean
public S3Client s3Client(){
AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey);
S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(true).build();
try{
S3Client s3 = S3Client.builder()
.endpointOverride(new URI(endpointUrl))
.credentialsProvider(StaticCredentialsProvider.create(awsBasicCredentials))
.region(Region.of(regionStr))
.serviceConfiguration(s3Config)
.build();
return s3;
} catch (URISyntaxException e){
throw new RuntimeException(e.getMessage());
}
}
@Bean
public S3Presigner s3Presigner(){
try{
S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(true).build();
S3Presigner presigner = S3Presigner.builder()
.endpointOverride(new URI(endpointUrl))
.region(Region.of(regionStr))
.build();
return presigner;
} catch (URISyntaxException e){
throw new RuntimeException(e.getMessage());
}
}
public String getAccessKeyId() {
return accessKeyId;
}
public void setAccessKeyId(String accessKeyId) {
this.accessKeyId = accessKeyId;
}
public String getSecretAccessKey() {
return secretAccessKey;
}
public void setSecretAccessKey(String secretAccessKey) {
this.secretAccessKey = secretAccessKey;
}
public String getEndpointUrl() {
return endpointUrl;
}
public void setEndpointUrl(String endpointUrl) {
this.endpointUrl = endpointUrl;
}
public String getOuterDomain() {
return outerDomain;
}
public void setOuterDomain(String outerDomain) {
this.outerDomain = outerDomain;
}
public String getInnerDomain() {
return innerDomain;
}
public void setInnerDomain(String innerDomain) {
this.innerDomain = innerDomain;
}
public String getRegionStr() {
return regionStr;
}
public void setRegionStr(String regionStr) {
this.regionStr = regionStr;
}
}
编写业务类封装对象存储的基本调用方法
新增AwsS3Handler:
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.google.common.collect.Lists;
import com.szlanyou.cloud.projectname.common.minio.MinioFile;
import com.szlanyou.cloud.projectname.common.minio.MinioPathUtils;
import com.szlanyou.cloud.projectname.common.otautils.CommonUtils;
import com.szlanyou.cloud.projectname.common.util.LogUtils;
import jakarta.annotation.Resource;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.http.SdkHttpResponse;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
@Service
public class AwsS3Handler {
@Resource
private S3Client s3Client;
@Resource
private S3Presigner s3Presigner;
@Resource
private AwsS3Config awsS3Config;
/**
* 对象列表
* @param bucket bucket
* @param key 对象路径
* @param pageToken 下一页token
* @param pageSize 分页大小
*/
public List<MinioFile> list(String bucket, String key, String pageToken, Integer pageSize) throws UnsupportedEncodingException {
if(StringUtils.isBlank(bucket)) throw new RuntimeException("bucket 不能为空");
ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder();
// 设置bucket
builder.bucket(bucket);
// 设置一次请求返回多少数据
builder.maxKeys(pageSize);
if(!StringUtils.isBlank(key)) {
// 设置文件路径分隔符,用于查找
// builder.prefix(key).delimiter("/");
builder.prefix(key);
}
ListObjectsV2Request listObjReq = builder.build();
ListObjectsV2Response listObjRes = s3Client.listObjectsV2(listObjReq);
// 获取下一页数据
if(listObjRes.isTruncated() && !StringUtils.isBlank(pageToken)){
listObjReq = listObjReq.toBuilder().continuationToken(pageToken).build();
listObjRes = s3Client.listObjectsV2(listObjReq);
}
List<S3Object> s3ObjectList = listObjRes.contents();
// 获取下一页token
String pageNextToken = listObjRes.nextContinuationToken();
String finalPageNextToken = StringUtils.isBlank(pageNextToken) ? "" : URLEncoder.encode(pageNextToken, "utf-8");
// 重新组装为自己需要的数据格式
List<MinioFile> s3ObjList = Lists.transform(s3ObjectList, (s3Object) -> {
MinioFile vo = new MinioFile();
vo.setObjectName(s3Object.key());
vo.setSize(s3Object.size());
vo.setLastModifiedTimeStamp(s3Object.lastModified().toEpochMilli());
vo.setPageToken(finalPageNextToken);
vo.setFileUrl(MinioPathUtils.getBaseUrl() + s3Object.key());
return vo;
});
return s3ObjList;
}
/**
* 异步完整上传不分片
* @param bucket bucket
* @param key 对象路径
* @param file 文件对象
*/
//@Async("awsThreadPoolExecutor")
public MinioFile singleUpload(String bucket, String key, String localFilePath) throws IOException {
Long startTime = System.currentTimeMillis() / 1000;
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
RequestBody requestBody = null;
PutObjectResponse putObjectResponse = null;
File file = new File(localFilePath);
try (InputStream in = new FileInputStream(localFilePath)){
requestBody = RequestBody.fromInputStream(in, file.length());
putObjectResponse = s3Client.putObject(putObjectRequest, requestBody);
SdkHttpResponse sdkHttpResponse = putObjectResponse.sdkHttpResponse();
if(!sdkHttpResponse.isSuccessful()){
throw new RuntimeException("上传对象存储失败, statusCode:" + sdkHttpResponse.statusCode() + "statusText:" + sdkHttpResponse.statusText());
}
} catch (FileNotFoundException e) {
throw new RuntimeException("minio文件不存在:" + localFilePath);
} catch (IOException e) {
throw new RuntimeException("minio文件关闭失败");
} catch (RuntimeException e) {
throw e;
}
long endTime = System.currentTimeMillis() / 1000;
LogUtils.info("上传文件(" + key + ")总计耗费时间为:" + (endTime - startTime) + " 秒");
HeadObjectResponse headResponse = s3Client.headObject(b -> b
.bucket(bucket)
.key(key)
);
MinioFile result = new MinioFile();
result.setFileUrl(MinioPathUtils.getBaseUrl() + key);
result.setObjectName(key);
result.setSize(headResponse.contentLength());
result.setLastModifiedTimeStamp(headResponse.lastModified().toEpochMilli());
return result;
}
public MinioFile singleUpload(String bucket, String customDir, InputStream in, String fileName) throws Exception{
String localPath = CommonUtils.getLocalRandomPath();
CommonUtils.createDirs(localPath);
String localFilePath = localPath + fileName;
File file = new File(localFilePath);
try {
FileUtils.copyToFile(in, file);
return this.singleUpload(bucket, customDir + fileName, localFilePath);
} catch (Exception e) {
throw e;
} finally {
CommonUtils.deleteFileAll(localPath);
if(null != in) in.close();
}
}
/**
* 对象下载,返回url下载地址
* @param bucket bucket
* @param key 对象路径
*/
//public String downloadTmpUrl(String bucket, String key){
// if(!StringUtils.isBlank(bucket) || !StringUtils.isBlank(key)) throw new RuntimeException("参数错误");
//
// GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(bucket).key(key).build();
// GetObjectPresignRequest objectPresignRequest = GetObjectPresignRequest.builder()
// .signatureDuration(Duration.ofMinutes(10))
// .getObjectRequest(objectRequest)
// .build();
// PresignedGetObjectRequest presignedGetObjectRequest = s3Presigner.presignGetObject(objectPresignRequest);
// String url = presignedGetObjectRequest.url().toString();
//
// return url;
//}
public void downloadToFile(String bucket, String key, String localFilePath) {
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
s3Client.getObject(getObjectRequest, ResponseTransformer.toFile(Paths.get(localFilePath)));
}
/**
* 对象删除,支持批量删除
* @param bucket bucket
* @param keyList 多个key组成的json数组转化成list对象
*/
public void delete(String bucket, String key){
if(StringUtils.isBlank(bucket) || StringUtils.isBlank(key)) throw new RuntimeException("参数错误");
List<ObjectIdentifier> identifierList = new ArrayList<>();
identifierList.add(ObjectIdentifier.builder().key(key).build());
try{
Delete delete = Delete.builder().objects(identifierList).build();
DeleteObjectsRequest deleteObjectRequest = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
s3Client.deleteObjects(deleteObjectRequest);
} catch (Exception ex){
LogUtils.info("delete S3 failure:key:" + key);
}
}
public void deleteSingleFile(String bucket, String key) {
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
try {
s3Client.deleteObject(deleteObjectRequest);
LogUtils.info("delete S3 success:key: " + key);
} catch (Exception e) {
LogUtils.info("delete S3 failure:key:" + key);
}
}
public boolean doesObjectExist(String bucket, String key) {
try {
HeadObjectRequest headObjectRequest = HeadObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
HeadObjectResponse headObjectResponse = s3Client.headObject(headObjectRequest);
return true;
} catch (S3Exception e) {
if (e.statusCode() == 404 || e.statusCode() == 403) {
return false;
}
throw e; // 其他异常重新抛出
}
}
public MinioFile getFileInfo(String bucket, String key) {
MinioFile file = null;
try {
HeadObjectRequest headObjectRequest = HeadObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
HeadObjectResponse headObjectResponse = s3Client.headObject(headObjectRequest);
file = new MinioFile();
file.setFileUrl(MinioPathUtils.getBaseUrl() + key);
file.setLastModifiedTimeStamp(headObjectResponse.lastModified().toEpochMilli());
file.setObjectName(key);
file.setSize(headObjectResponse.contentLength());
} catch (S3Exception e) {
LogUtils.info("读取文件报错:{}", e.toString());
}
return file;
}
}
依赖的上传文件后返回的实体类MinioFile(从minio改过来懒得换名了):
public class MinioFile {
private String fileUrl;
private String objectName;
private long size;
private long lastModifiedTimeStamp;//13位时间戳
private String lastModifiedStr;//yyyy-MM-dd HH:mm:ss
private String pageToken;
public String getFileUrl() {
return fileUrl;
}
public void setFileUrl(String fileUrl) {
this.fileUrl = fileUrl;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public long getLastModifiedTimeStamp() {
return lastModifiedTimeStamp;
}
public void setLastModifiedTimeStamp(long lastModifiedTimeStamp) {
this.lastModifiedTimeStamp = lastModifiedTimeStamp;
}
public String getLastModifiedStr() {
return lastModifiedStr;
}
public void setLastModifiedStr(String lastModifiedStr) {
this.lastModifiedStr = lastModifiedStr;
}
public String getPageToken() {
return pageToken;
}
public void setPageToken(String pageToken) {
this.pageToken = pageToken;
}
}
依赖的MinioPathUtils:
import com.szlanyou.cloud.common.utils.helper.StringHelper;
import com.szlanyou.cloud.projectname.common.otautils.CommonUtils;
import com.szlanyou.cloud.projectname.common.otautils.DateUtil;
public class MinioPathUtils {
public static final String minioPath = "xxx";//自定义业务前缀
public static String getMinioTmpDir() {
return minioPath + "/tmp/" + DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
public static String getMinioDefaultDir() {
return minioPath + "/" + DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
public static String getMinioCustomDir(String path) {
return minioPath + path + DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
public static String getBaseUrl() {
return CommonUtils.getPropFromSpring("aws.outerDomain") + "/";
}
public static String getS3Bucket() {
return CommonUtils.getPropFromSpring("aws.public-bucket");
}
}
编写测试controller,调用测试
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.szlanyou.cloud.ota.diagnosis.upgrade.test.param.S3TesDownloadToFileParam;
import com.szlanyou.cloud.ota.diagnosis.upgrade.test.param.S3TestListParam;
import com.szlanyou.cloud.ota.diagnosis.upgrade.util.aws3.AwsS3Handler;
import com.szlanyou.cloud.projectname.common.minio.MinioFile;
import com.szlanyou.cloud.projectname.common.minio.MinioPathUtils;
import com.szlanyou.cloud.projectname.common.otautils.CommonUtils;
import com.szlanyou.cloud.projectname.common.util.LogUtils;
import com.szlanyou.cloud.projectname.common.vo.ResponseVO;
import jakarta.annotation.Resource;
@RestController
@RequestMapping(value = "s3/s3test/", produces = {MediaType.APPLICATION_JSON_VALUE})
public class S3TestController {
@Resource
AwsS3Handler awsS3Handler;
@RequestMapping(value = "upload", method = { RequestMethod.POST })
public ResponseVO<?> upload(@RequestBody S3TesDownloadToFileParam params) throws IOException {
String filePath = params.getLocalPath();
String key = params.getDirPath();
System.out.println("key===" + params.getDirPath());
MinioFile file = awsS3Handler.singleUpload(MinioPathUtils.getS3Bucket(), key, filePath);
return new ResponseVO<>(file);
}
@RequestMapping(value = "list", method = { RequestMethod.POST })
public ResponseVO<?> list(@RequestBody S3TestListParam params) throws IOException {
String key = params.getDirPath();
List<MinioFile> list = awsS3Handler.list(MinioPathUtils.getS3Bucket(), key, "", 10);
return new ResponseVO<>(list);
}
@RequestMapping(value = "downloadToFile", method = { RequestMethod.POST })
public ResponseVO<?> downloadToFile(@RequestBody S3TesDownloadToFileParam params) throws IOException {
awsS3Handler.downloadToFile(MinioPathUtils.getS3Bucket(), params.getDirPath(), params.getLocalPath());
return new ResponseVO<>();
}
@RequestMapping(value = "doesObjectExist", method = { RequestMethod.POST })
public ResponseVO<?> doesObjectExist(@RequestBody S3TestListParam params) throws IOException {
boolean flag = awsS3Handler.doesObjectExist(MinioPathUtils.getS3Bucket(), params.getDirPath());
return new ResponseVO<>(flag);
}
@RequestMapping(value = "delete", method = { RequestMethod.POST })
public ResponseVO<?> delete(@RequestBody S3TestListParam params) throws IOException {
awsS3Handler.deleteSingleFile(MinioPathUtils.getS3Bucket(), params.getDirPath());
return new ResponseVO<>();
}
@RequestMapping(value = "sumSizeByDir", method = { RequestMethod.POST })
public ResponseVO<?> sumSizeByDir(@RequestBody S3TestListParam params) throws IOException {
String key = params.getDirPath();
List<MinioFile> list = awsS3Handler.list(MinioPathUtils.getS3Bucket(), key, "", 1000);
//sumSize sumSizeStr fileNum
Map<String, String> resultMap = null;
resultMap = this.execS3SumSize(list, resultMap);
while(!CollectionUtils.isEmpty(list) && !StringUtils.isBlank(list.get(0).getPageToken())) {
String token = list.get(0).getPageToken();
list = awsS3Handler.list(MinioPathUtils.getS3Bucket(), key, token, 1000);
resultMap = this.execS3SumSize(list, resultMap);
}
return new ResponseVO<>(resultMap);
}
@RequestMapping(value = "removeByDir", method = { RequestMethod.POST })
public ResponseVO<?> removeByDir(@RequestBody S3TestListParam params) throws IOException {
String key = params.getDirPath();
List<MinioFile> list = awsS3Handler.list(MinioPathUtils.getS3Bucket(), key, "", 1000);
//totalNum successNum failNum
Map<String, String> resultMap = null;
resultMap = this.execRemoveByDirMap(list, resultMap);
while(!CollectionUtils.isEmpty(list) && !StringUtils.isBlank(list.get(0).getPageToken())) {
String token = list.get(0).getPageToken();
list = awsS3Handler.list(MinioPathUtils.getS3Bucket(), key, token, 1000);
resultMap = this.execRemoveByDirMap(list, resultMap);
}
return new ResponseVO<>(resultMap);
}
private Map<String, String> execRemoveByDirMap(List<MinioFile> list, Map<String, String> resultMap) {
if(Objects.isNull(resultMap) || resultMap.isEmpty()) {
resultMap = new HashMap<>();
}
long successNum = null == resultMap.get("successNum") ? 0L : Long.parseLong(resultMap.get("successNum").toString());
long failNum = null == resultMap.get("failNum") ? 0L : Long.parseLong(resultMap.get("failNum").toString());
long totalNum = null == resultMap.get("totalNum") ? 0L : Long.parseLong(resultMap.get("totalNum").toString());
if(!CollectionUtils.isEmpty(list)) {
totalNum += list.size();
for(MinioFile file : list) {
try {
awsS3Handler.deleteSingleFile(MinioPathUtils.getS3Bucket(), file.getObjectName());
successNum++;
} catch (Exception e) {
failNum++;
LogUtils.info("文件删除失败:{},失败原因是:{}", file.getObjectName(), e.getMessage());
}
}
}
resultMap.put("totalNum", totalNum+"");
resultMap.put("successNum", successNum+"");
resultMap.put("failNum", failNum+"");
return resultMap;
}
private Map<String, String> execS3SumSize(List<MinioFile> list, Map<String, String> resultMap) {
if(Objects.isNull(resultMap) || resultMap.isEmpty()) {
resultMap = new HashMap<>();
}
long sumSize = null == resultMap.get("sumSize") ? 0L : Long.parseLong(resultMap.get("sumSize").toString());
long fileNum = null == resultMap.get("fileNum") ? 0L : Long.parseLong(resultMap.get("fileNum").toString());
if(!CollectionUtils.isEmpty(list)) {
fileNum += list.size();
for(MinioFile file : list) {
sumSize += file.getSize();
}
}
resultMap.put("sumSize", sumSize + "");
resultMap.put("sumSizeStr", 0 == sumSize? "0B" : CommonUtils.formatFileSize(sumSize));
resultMap.put("fileNum", fileNum + "");
return resultMap;
}
}
注意事项
如果对象存储的桶权限没有开启公有访问,则本地不能操作文件,需要部署到aws环境下内网调用访问;
因为是自定义的文件下载域名(可以配置私有证书双向认证),需要使用aws云的 route53+cloudfront来实现域名指定到私有桶路径(cloudfront 控制台设置Route domains to CloudFront),控制台操作步骤此处省略。