Browse Source

完善同步

master
review512jwy@163.com 3 months ago
parent
commit
1c930cd553
  1. 7
      src/main/java/com/dashboard/aws/lambda/handler/S3ToMySQLHandler.java
  2. 24
      src/main/java/com/dashboard/aws/lambda/service/MySQLService.java

7
src/main/java/com/dashboard/aws/lambda/handler/S3ToMySQLHandler.java

@ -32,6 +32,9 @@ public class S3ToMySQLHandler implements RequestHandler<Map<String, Object>, Str
LocalDate now = DateUtil.resolveEventDate( event, zone); LocalDate now = DateUtil.resolveEventDate( event, zone);
String dateStr = DateUtil.getLastYearSameIsoWeekDayStr(now); String dateStr = DateUtil.getLastYearSameIsoWeekDayStr(now);
// --- 解析出 fileDate(用于删除早于该日期的数据) ---
LocalDate fileDate = LocalDate.parse(dateStr);
// 查询符合条件的企业ID // 查询符合条件的企业ID
List<Long> companyIds = mysqlService.getActiveCompanyIds(); List<Long> companyIds = mysqlService.getActiveCompanyIds();
logger.info("company id list: {}", companyIds); logger.info("company id list: {}", companyIds);
@ -42,6 +45,10 @@ public class S3ToMySQLHandler implements RequestHandler<Map<String, Object>, Str
String schema = "data_center_dongjian_" + companyId; String schema = "data_center_dongjian_" + companyId;
for (String table : tables) { for (String table : tables) {
//先删除旧数据
int deleted = mysqlService.deleteBeforeDate(schema, table, fileDate);
logger.info("[{}][{}] deleted {} records before {}", schema, table, deleted, fileDate);
// 构造 S3 文件路径 // 构造 S3 文件路径
String s3Key = String.format("%s/%s/%s.csv", table, companyId, dateStr); String s3Key = String.format("%s/%s/%s.csv", table, companyId, dateStr);
logger.info("processing s3 file: {}", s3Key); logger.info("processing s3 file: {}", s3Key);

24
src/main/java/com/dashboard/aws/lambda/service/MySQLService.java

@ -4,6 +4,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.sql.*; import java.sql.*;
import java.time.LocalDate;
import java.util.*; import java.util.*;
public class MySQLService { public class MySQLService {
@ -101,4 +102,27 @@ public class MySQLService {
} }
return total; return total;
} }
public int deleteBeforeDate(String schema, String table, LocalDate fileDate) {
// 计算比较值: yyyyMMdd 格式的整数(例如 20251008)
int cutoff = fileDate.getYear() * 10000 + fileDate.getMonthValue() * 100 + fileDate.getDayOfMonth();
String sql = String.format(
"DELETE FROM %s.%s WHERE (COALESCE(date_year,0) * 10000 + COALESCE(date_month,0) * 100 + COALESCE(date_day,0)) < ?",
schema, table
);
logger.info("deleteBeforeDate sql: {}, cutoff:{}", sql, cutoff);
try (Connection conn = java.sql.DriverManager.getConnection(MYSQL_URL, DB_USER, DB_PASSWORD);
PreparedStatement ps = conn.prepareStatement(sql)) {
ps.setInt(1, cutoff);
int deleted = ps.executeUpdate();
logger.info("deleteBeforeDate: schema={}, table={}, cutoff={}, deleted={}", schema, table, cutoff, deleted);
return deleted;
} catch (SQLException e) {
logger.error("deleteBeforeDate failed for {}/{} cutoff={}", schema, table, cutoff, e);
return 0;
}
}
} }

Loading…
Cancel
Save