package com.dashboard.aws.lambda; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.S3Object; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * 把历史数据从旧格式迁移到我们现在使用的新格式 table/date/001.csv ,这样新旧数据保持一致 */ public class S3CsvReorganizer { private static final String BUCKET = "dashboard-for-backup"; /** * 匹配: * dashboard-for-backup/dashboard_record_measure/2025-12-25.csv */ private static final Pattern FILE_PATTERN = Pattern.compile("^(.*/)(\\d{4}-\\d{2}-\\d{2})\\.csv$"); /** * 重组指定目录下的 CSV 文件 * * 例如: * dashboard-for-backup/dashboard_record_measure/ */ public void reorganize(String prefix) { S3Client s3 = S3Client.builder() .region(Region.AP_NORTHEAST_1) .httpClient(UrlConnectionHttpClient.builder().build()) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( "AKA", "IFwUPLd") ) ) .build(); String continuationToken = null; do { // 构建列表请求 ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() .bucket(BUCKET) .prefix(prefix); // 如果有续延token,设置它以获取下一页 if (continuationToken != null) { builder.continuationToken(continuationToken); } ListObjectsV2Response response = s3.listObjectsV2(builder.build()); for (S3Object obj : response.contents()) { String oldKey = obj.key(); // 跳过目录对象 if (oldKey.endsWith("/")) { continue; } Matcher matcher = FILE_PATTERN.matcher(oldKey); if (!matcher.matches()) { continue; } String basePath = matcher.group(1); // dashboard-for-backup/dashboard_record_measure/ String date = matcher.group(2); // 2025-12-25 String newKey = basePath + date + "/00000001.csv"; // 如果目标文件和源文件相同则跳过 if (oldKey.equals(newKey)) { continue; } System.out.println("Moving:"); System.out.println(" From: " + oldKey); System.out.println(" To : " + newKey); // 1. 服务端复制 s3.copyObject( CopyObjectRequest.builder() .sourceBucket(BUCKET) .sourceKey(oldKey) .destinationBucket(BUCKET) .destinationKey(newKey) .build() ); // 2. 删除旧文件 // s3.deleteObject( // DeleteObjectRequest.builder() // .bucket(BUCKET) // .key(oldKey) // .build() // ); System.out.println("Moved successfully."); } continuationToken = response.isTruncated() ? response.nextContinuationToken() : null; } while (continuationToken != null); } public static void main(String[] args) { S3CsvReorganizer reorganizer = new S3CsvReorganizer(); reorganizer.reorganize( "dashboard_record_accumulate/" ); } }