From a89ff65254986fb5487c49075c9f0b8a7dc422bb Mon Sep 17 00:00:00 2001 From: "review512jwy@163.com" <“review512jwy@163.com”> Date: Sun, 12 Oct 2025 13:33:29 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=87=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 39 +++++ pom.xml | 103 +++++++++++ .../aws/lambda/handler/BackupHandler.java | 89 ++++++++++ .../aws/lambda/service/MySQLService.java | 77 ++++++++ .../aws/lambda/service/S3Service.java | 83 +++++++++ .../com/dashboard/aws/lambda/util/Arith.java | 165 ++++++++++++++++++ .../dashboard/aws/lambda/util/CsvUtil.java | 29 +++ src/main/resources/logback.xml | 13 ++ 8 files changed, 598 insertions(+) create mode 100644 .gitignore create mode 100644 pom.xml create mode 100644 src/main/java/com/dashboard/aws/lambda/handler/BackupHandler.java create mode 100644 src/main/java/com/dashboard/aws/lambda/service/MySQLService.java create mode 100644 src/main/java/com/dashboard/aws/lambda/service/S3Service.java create mode 100644 src/main/java/com/dashboard/aws/lambda/util/Arith.java create mode 100644 src/main/java/com/dashboard/aws/lambda/util/CsvUtil.java create mode 100644 src/main/resources/logback.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc3f89c --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..4abeda9 --- /dev/null +++ b/pom.xml @@ -0,0 +1,103 @@ + + 4.0.0 + com.zhukovasky + dashboard-statistics-lambda + jar + 1.0-SNAPSHOT + dashboard-statistics-lambda Maven Webapp + http://maven.apache.org + + + junit + junit + 3.8.1 + test + + + + + com.amazonaws + aws-lambda-java-core + 1.4.0 + + + + + software.amazon.awssdk + s3 + 2.35.4 + compile + + + + + mysql + mysql-connector-java + 8.0.33 + + + + + org.apache.commons + commons-csv + 1.14.1 + + + + + software.amazon.awssdk + url-connection-client + 2.34.9 + compile + + + + + org.slf4j + slf4j-api + 1.7.36 + + + ch.qos.logback + logback-classic + 1.2.12 + + + + + + dashboard-statistics-lambda + + + + org.apache.maven.plugins + maven-dependency-plugin + + + package + + copy-dependencies + + + ${project.build.directory}/lib + false + true + false + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + 17 + + + + + \ No newline at end of file diff --git a/src/main/java/com/dashboard/aws/lambda/handler/BackupHandler.java b/src/main/java/com/dashboard/aws/lambda/handler/BackupHandler.java new file mode 100644 index 0000000..bf93674 --- /dev/null +++ b/src/main/java/com/dashboard/aws/lambda/handler/BackupHandler.java @@ -0,0 +1,89 @@ +package com.dashboard.aws.lambda.handler; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.dashboard.aws.lambda.service.MySQLService; +import com.dashboard.aws.lambda.service.S3Service; +import com.dashboard.aws.lambda.util.CsvUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.*; + +public class BackupHandler implements RequestHandler, String> { + + private static final Logger logger = LoggerFactory.getLogger(BackupHandler.class); + + private final MySQLService mysqlService = new MySQLService(); + private final S3Service s3Service = new S3Service(); + + @Override + public String handleRequest(Map event, Context context) { + try { + LocalDate targetDate = LocalDate.now(ZoneId.of("Asia/Tokyo")).minusDays(3); + long cutoff = targetDate.atStartOfDay(ZoneId.of("Asia/Tokyo")).toInstant().toEpochMilli(); + + // 查询符合条件的企业ID + List companyIds = mysqlService.getActiveCompanyIds(); + logger.info("company id list: {}", companyIds); + + for (Long companyId : companyIds) { + String schema = "data_center_dongjian_" + companyId; + + for (String table : List.of("dashboard_record_accumulate", "dashboard_record_measure")) { + // 查询旧数据 + List> rows = mysqlService.queryOldData(schema, table, cutoff); + if (rows.isEmpty()) { + logger.info("[{}][{}] no data....", schema, table); + continue; + } + + // 按年月日分组 + Map>> grouped = new HashMap<>(); + for (Map row : rows) { + int y = (int) row.get("date_year"); + int m = (int) row.get("date_month"); + int d = (int) row.get("date_day"); + // 格式化年月日，不足两位补0 + String monthStr = String.format("%02d", m); + String dayStr = String.format("%02d", d); + String key = String.format("%d-%s-%s", y, monthStr, dayStr); + + grouped.computeIfAbsent(key, k -> new ArrayList<>()).add(row); + } + + // 上传每个分组 + boolean allSuccess = true; + for (Map.Entry>> entry : grouped.entrySet()) { + String dateKey = entry.getKey(); + byte[] csvBytes = CsvUtil.toCsvBytes(entry.getValue()); + String s3Key = String.format("%s/%s/%s.csv", table, companyId, dateKey); + try { + s3Service.upload(s3Key, csvBytes); + logger.info("[{}][{}] backup success -> {}", schema, table, s3Key); + } catch (Exception e) { + allSuccess = false; + logger.error("[{}][{}] backup failed -> {}", schema, table, s3Key, e); + } + } + + // 全部上传成功才删除数据 + if (allSuccess) { + mysqlService.deleteOldData(schema, table, cutoff); + logger.info("[{}][{}] delete old data", schema, table); + } else { + logger.error("[{}][{}] not all files uploaded successfully, skip delete", schema, table); + } + } + } + + return "backup process finished"; + + } catch (Exception e) { + logger.error("backup error", e); + return "backup error：" + e.getMessage(); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/dashboard/aws/lambda/service/MySQLService.java b/src/main/java/com/dashboard/aws/lambda/service/MySQLService.java new file mode 100644 index 0000000..0761d9f --- /dev/null +++ b/src/main/java/com/dashboard/aws/lambda/service/MySQLService.java @@ -0,0 +1,77 @@ +package com.dashboard.aws.lambda.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.*; +import java.util.*; + +public class MySQLService { + + private static final Logger logger = LoggerFactory.getLogger(MySQLService.class); + + private static final String MYSQL_URL = System.getenv("DB_URL"); // 统一实例URL + private static final String DB_USER = System.getenv("DB_USER"); + private static final String DB_PASSWORD = System.getenv("DB_PASSWORD"); + + + /** + * 获取符合条件的企业ID + */ + public List getActiveCompanyIds() throws SQLException { + String sql = """ + SELECT id FROM data_center_new.basic_company + WHERE flag != 1 + AND (parent_id = -1 OR parent_id = 1) + """; + + try (Connection conn = DriverManager.getConnection(MYSQL_URL, DB_USER, DB_PASSWORD); + PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + List ids = new ArrayList<>(); + while (rs.next()) { + ids.add(rs.getLong("id")); + } + return ids; + } + } + + public List> queryOldData(String schema, String table, long cutoff) throws SQLException { + String sql = String.format("SELECT * FROM %s.%s WHERE upload_at < ?", schema, table); + logger.info("query sql: {}, cutoff: {}", sql, cutoff); + + try (Connection conn = DriverManager.getConnection(MYSQL_URL, DB_USER, DB_PASSWORD); + PreparedStatement ps = conn.prepareStatement(sql)) { + + ps.setLong(1, cutoff); + + ResultSet rs = ps.executeQuery(); + ResultSetMetaData meta = rs.getMetaData(); + List> list = new ArrayList<>(); + + while (rs.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= meta.getColumnCount(); i++) { + row.put(meta.getColumnName(i), rs.getObject(i)); + } + list.add(row); + } + return list; + } + } + + public void deleteOldData(String schema, String table, long cutoff) throws SQLException { + String sql = String.format("DELETE FROM %s.%s WHERE upload_at < ?", schema, table); + logger.info("deleteOldData sql: {}, cutoff: {}", sql, cutoff); + + try (Connection conn = DriverManager.getConnection(MYSQL_URL, DB_USER, DB_PASSWORD); + PreparedStatement ps = conn.prepareStatement(sql)) { + + ps.setLong(1, cutoff); + int deleted = ps.executeUpdate(); + logger.info("[{}.{}] 删除 {} 条记录", schema, table, deleted); + } + } + +} diff --git a/src/main/java/com/dashboard/aws/lambda/service/S3Service.java b/src/main/java/com/dashboard/aws/lambda/service/S3Service.java new file mode 100644 index 0000000..98918da --- /dev/null +++ b/src/main/java/com/dashboard/aws/lambda/service/S3Service.java @@ -0,0 +1,83 @@ +package com.dashboard.aws.lambda.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +import java.util.Objects; + +public class S3Service { + + private static final Logger logger = LoggerFactory.getLogger(S3Service.class); + + String accessKey = System.getenv("ACCESS_KEY_ID"); + String secretKey = System.getenv("SECRET_ACCESS_KEY"); + String bucketName = System.getenv("S3_BUCKET"); + + private final S3Client s3; + private final String bucket; + + public S3Service() { + + Objects.requireNonNull(accessKey, "环境变量 AWS_ACCESS_KEY_ID 不能为空"); + Objects.requireNonNull(secretKey, "环境变量 AWS_SECRET_ACCESS_KEY 不能为空"); + Objects.requireNonNull(bucketName, "环境变量 S3_BUCKET 不能为空"); + + this.bucket = bucketName; + + this.s3 = S3Client.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKey, secretKey) + )) + .region(Region.AP_NORTHEAST_1) // 东京区域 + .build(); + } + + public S3Service(String accessKey, String secretKey, String bucketName) { + Objects.requireNonNull(accessKey, "AccessKey不能为空"); + Objects.requireNonNull(secretKey, "SecretKey不能为空"); + Objects.requireNonNull(bucketName, "S3桶名不能为空"); + + this.bucket = bucketName; + + this.s3 = S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKey, secretKey) + )) + .region(Region.AP_NORTHEAST_1) // 东京区域 + .build(); + } + + /** + * 上传字节数组到指定 key + */ + public void upload(String key, byte[] content) { + Objects.requireNonNull(key, "S3 key不能为空"); + Objects.requireNonNull(content, "上传内容不能为空"); + + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .build(); + + PutObjectResponse response = s3.putObject(request, RequestBody.fromBytes(content)); + logger.info("上传成功: {}, ETag={}", key, response.eTag()); + } + + /** + * 关闭 S3Client + */ + public void close() { + if (s3 != null) { + s3.close(); + } + } +} diff --git a/src/main/java/com/dashboard/aws/lambda/util/Arith.java b/src/main/java/com/dashboard/aws/lambda/util/Arith.java new file mode 100644 index 0000000..8c084f4 --- /dev/null +++ b/src/main/java/com/dashboard/aws/lambda/util/Arith.java @@ -0,0 +1,165 @@ +package com.dashboard.aws.lambda.util; +import java.math.BigDecimal; + +/** + * 进行BigDecimal对象的加减乘除，四舍五入等运算的工具类 + * + * 由于Java的简单类型不能够精确的对浮点数进行运算，这个工具类提供精 + * 确的浮点数运算，包括加减乘除和四舍五入。 + */ +public class Arith { + //默认除法运算精度 + private static final int DEF_DIV_SCALE = 10; + + //这个类不能实例化 + private Arith(){ + } + + /** + * 提供精确的加法运算。 + * @param v1 被加数 + * @param v2 加数 + * @return 两个参数的和 + */ + public static double add(double v1,double v2){ + BigDecimal b1 = new BigDecimal(Double.toString(v1)); + BigDecimal b2 = new BigDecimal(Double.toString(v2)); + return b1.add(b2).doubleValue(); + } + + /** + * 提供精确的减法运算。 + * @param v1 被减数 + * @param v2 减数 + * @return 两个参数的差 + */ + public static double sub(double v1,double v2){ + BigDecimal b1 = new BigDecimal(Double.toString(v1)); + BigDecimal b2 = new BigDecimal(Double.toString(v2)); + return b1.subtract(b2).doubleValue(); + } + + /** + * 提供精确的乘法运算。 + * @param v1 被乘数 + * @param v2 乘数 + * @return 两个参数的积 + */ + public static double mul(double v1,double v2){ + BigDecimal b1 = new BigDecimal(Double.toString(v1)); + BigDecimal b2 = new BigDecimal(Double.toString(v2)); + return b1.multiply(b2).doubleValue(); + } + + /** + * 提供（相对）精确的除法运算，当发生除不尽的情况时，精确到 + * 小数点以后10位，以后的数字四舍五入。 + * @param v1 被除数 + * @param v2 除数 + * @return 两个参数的商 + */ + public static double div(double v1,double v2){ + return div(v1,v2,DEF_DIV_SCALE); + } + + /** + * 提供（相对）精确的除法运算。当发生除不尽的情况时，由scale参数指 + * 定精度，以后的数字四舍五入。 + * @param v1 被除数 + * @param v2 除数 + * @param scale 表示表示需要精确到小数点以后几位。 + * @return 两个参数的商 + */ + public static double div(double v1,double v2,int scale){ + if(scale<0){ + throw new IllegalArgumentException( + "The scale must be a positive integer or zero"); + } + BigDecimal b1 = new BigDecimal(Double.toString(v1)); + BigDecimal b2 = new BigDecimal(Double.toString(v2)); + return b1.divide(b2,scale,BigDecimal.ROUND_HALF_UP).doubleValue(); + } + + /** + * 提供精确的小数位四舍五入处理。 + * @param v 需要四舍五入的数字 + * @param scale 小数点后保留几位 + * @return 四舍五入后的结果 + */ + public static double round(double v,int scale){ + if(scale<0){ + throw new IllegalArgumentException( + "The scale must be a positive integer or zero"); + } + BigDecimal b = new BigDecimal(Double.toString(v)); + BigDecimal one = new BigDecimal("1"); + return b.divide(one,scale,BigDecimal.ROUND_HALF_UP).doubleValue(); + } + + /** + * 提供精确的类型转换(Float) + * @param v 需要被转换的数字 + * @return 返回转换结果 + */ + public static float convertsToFloat(double v){ + BigDecimal b = BigDecimal.valueOf(v); + return b.floatValue(); + } + + /** + * 提供精确的类型转换(Int)不进行四舍五入 + * @param v 需要被转换的数字 + * @return 返回转换结果 + */ + public static int convertsToInt(double v){ + BigDecimal b = BigDecimal.valueOf(v); + return b.intValue(); + } + + /** + * 提供精确的类型转换(Long) + * @param v 需要被转换的数字 + * @return 返回转换结果 + */ + public static long convertsToLong(double v){ + BigDecimal b = BigDecimal.valueOf(v); + return b.longValue(); + } + + /** + * 返回两个数中大的一个值 + * @param v1 需要被对比的第一个数 + * @param v2 需要被对比的第二个数 + * @return 返回两个数中大的一个值 + */ + public static double returnMax(double v1,double v2){ + BigDecimal b1 = BigDecimal.valueOf(v1); + BigDecimal b2 = BigDecimal.valueOf(v2); + return b1.max(b2).doubleValue(); + } + + /** + * 返回两个数中小的一个值 + * @param v1 需要被对比的第一个数 + * @param v2 需要被对比的第二个数 + * @return 返回两个数中小的一个值 + */ + public static double returnMin(double v1,double v2){ + BigDecimal b1 = BigDecimal.valueOf(v1); + BigDecimal b2 = BigDecimal.valueOf(v2); + return b1.min(b2).doubleValue(); + } + + /** + * 精确对比两个数字 + * @param v1 需要被对比的第一个数 + * @param v2 需要被对比的第二个数 + * @return 如果两个数一样则返回0，如果第一个数比第二个数大则返回1，反之返回-1 + */ + public static int compareTo(double v1,double v2){ + BigDecimal b1 = BigDecimal.valueOf(v1); + BigDecimal b2 = BigDecimal.valueOf(v2); + return b1.compareTo(b2); + } + +} \ No newline at end of file diff --git a/src/main/java/com/dashboard/aws/lambda/util/CsvUtil.java b/src/main/java/com/dashboard/aws/lambda/util/CsvUtil.java new file mode 100644 index 0000000..2c4cbc4 --- /dev/null +++ b/src/main/java/com/dashboard/aws/lambda/util/CsvUtil.java @@ -0,0 +1,29 @@ +package com.dashboard.aws.lambda.util; + +import java.io.ByteArrayOutputStream; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +public class CsvUtil { + public static byte[] toCsvBytes(List> rows) { + if (rows == null || rows.isEmpty()) return new byte[0]; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(out); + + // 表头 + List headers = rows.get(0).keySet().stream().toList(); + writer.println(String.join(",", headers)); + + // 数据 + for (Map row : rows) { + writer.println(String.join(",", headers.stream() + .map(h -> String.valueOf(row.getOrDefault(h, ""))) + .toList())); + } + + writer.flush(); + return out.toString(StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8); + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..c91f8b1 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,13 @@ + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + +