dashboard-statistics-lambda/src/test/java/com/dashboard/aws/lambda/S3CsvReorganizer.java

package com.dashboard.aws.lambda;


import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 把历史数据从旧格式迁移到我们现在使用的新格式 table/date/001.csv ，这样新旧数据保持一致
 */
public class S3CsvReorganizer {

    private static final String BUCKET = "dashboard-for-backup";

    /**
     * 匹配：
     * dashboard-for-backup/dashboard_record_measure/2025-12-25.csv
     */
    private static final Pattern FILE_PATTERN =
            Pattern.compile("^(.*/)(\\d{4}-\\d{2}-\\d{2})\\.csv$");

    /**
     * 重组指定目录下的 CSV 文件
     *
     * 例如：
     * dashboard-for-backup/dashboard_record_measure/
     */
    public void reorganize(String prefix) {
        S3Client s3 = S3Client.builder()
                .region(Region.AP_NORTHEAST_1)
                .httpClient(UrlConnectionHttpClient.builder().build())
                .credentialsProvider(
                        StaticCredentialsProvider.create(
                                AwsBasicCredentials.create(
                                        "AKA",
                                        "IFwUPLd")
                        )
                )
                .build();

        String continuationToken = null;

        do {
            // 构建列表请求
            ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder()
                    .bucket(BUCKET)
                    .prefix(prefix);

            // 如果有续延token，设置它以获取下一页
            if (continuationToken != null) {
                builder.continuationToken(continuationToken);
            }

            ListObjectsV2Response response = s3.listObjectsV2(builder.build());

            for (S3Object obj : response.contents()) {
                String oldKey = obj.key();

                // 跳过目录对象
                if (oldKey.endsWith("/")) {
                    continue;
                }

                Matcher matcher = FILE_PATTERN.matcher(oldKey);
                if (!matcher.matches()) {
                    continue;
                }

                String basePath = matcher.group(1);   // dashboard-for-backup/dashboard_record_measure/
                String date = matcher.group(2);       // 2025-12-25

                String newKey = basePath + date + "/00000001.csv";

                // 如果目标文件和源文件相同则跳过
                if (oldKey.equals(newKey)) {
                    continue;
                }

                System.out.println("Moving:");
                System.out.println("  From: " + oldKey);
                System.out.println("  To  : " + newKey);

                // 1. 服务端复制
                s3.copyObject(
                        CopyObjectRequest.builder()
                                .sourceBucket(BUCKET)
                                .sourceKey(oldKey)
                                .destinationBucket(BUCKET)
                                .destinationKey(newKey)
                                .build()
                );

                // 2. 删除旧文件
//                s3.deleteObject(
//                        DeleteObjectRequest.builder()
//                                .bucket(BUCKET)
//                                .key(oldKey)
//                                .build()
//                );

                System.out.println("Moved successfully.");
            }

            continuationToken = response.isTruncated()
                    ? response.nextContinuationToken()
                    : null;

        } while (continuationToken != null);
    }

    public static void main(String[] args) {
        S3CsvReorganizer reorganizer = new S3CsvReorganizer();

        reorganizer.reorganize(
                "dashboard_record_accumulate/"
        );
    }
}