14 changed files with 455 additions and 398 deletions
@ -0,0 +1,18 @@ |
|||||
|
ALTER TABLE `dashboard_realtime_accumulate_day` |
||||
|
ADD COLUMN attr_code VARCHAR(50) DEFAULT 'single' COMMENT '一个属性的设备默认single,多属性的比如温湿度,则对应温度[temperature],湿度[humidity]' AFTER device_id, |
||||
|
DROP INDEX uniq_device_date, |
||||
|
ADD UNIQUE INDEX uniq_device_attr_date(device_id, attr_code, date_year, date_month, date_day); |
||||
|
|
||||
|
ALTER TABLE `dashboard_record_accumulate` |
||||
|
ADD COLUMN attr_code VARCHAR(50) DEFAULT 'single' COMMENT '一个属性的设备默认single,多属性的比如温湿度,则对应温度[temperature],湿度[humidity]' AFTER device_id, |
||||
|
ADD INDEX idx_device_attr_date(device_id, attr_code, date_year, date_month, date_day); |
||||
|
|
||||
|
ALTER TABLE dashboard_realtime_measure |
||||
|
DROP PRIMARY KEY, |
||||
|
ADD COLUMN attr_code VARCHAR(50) DEFAULT 'single' COMMENT '一个属性的设备默认single,多属性的比如温湿度,则对应温度[temperature],湿度[humidity]' AFTER device_id, |
||||
|
ADD UNIQUE INDEX (device_id, attr_code); |
||||
|
|
||||
|
ALTER TABLE dashboard_record_measure |
||||
|
ADD COLUMN attr_code VARCHAR(50) DEFAULT 'single' COMMENT '一个属性的设备默认single,多属性的比如温湿度,则对应温度[temperature],湿度[humidity]' AFTER device_id, |
||||
|
ADD INDEX idx_device_attr_date (device_id, attr_code, date_year, date_month, date_day); |
||||
|
|
||||
@ -0,0 +1,11 @@ |
|||||
|
package com.techsor.datacenter.sender.constants; |
||||
|
|
||||
|
public class DeviceAttrCode { |
||||
|
|
||||
|
public static final String COMMON = "single"; |
||||
|
|
||||
|
public static final String MEASURE_TEMPERATURE = "temperature"; |
||||
|
|
||||
|
public static final String MEASURE_HUMIDITY = "humidity"; |
||||
|
|
||||
|
} |
||||
@ -1,65 +1,27 @@ |
|||||
package com.techsor.datacenter.sender.disruptor; |
package com.techsor.datacenter.sender.disruptor; |
||||
|
|
||||
import com.lmax.disruptor.EventHandler; |
|
||||
import com.techsor.datacenter.sender.dao.DashboardAlertDao; |
import com.techsor.datacenter.sender.dao.DashboardAlertDao; |
||||
import lombok.extern.slf4j.Slf4j; |
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
import java.util.ArrayList; |
|
||||
import java.util.List; |
|
||||
|
|
||||
@Slf4j |
@Slf4j |
||||
public class AlertEventHandler implements EventHandler<AlertEvent> { |
public class AlertEventHandler extends BaseBatchEventHandler<AlertEvent> { |
||||
|
|
||||
private final DashboardAlertDao dao; |
private final DashboardAlertDao dao; |
||||
|
|
||||
private final int batchSize; |
|
||||
|
|
||||
private final List<AlertEvent> buffer; |
|
||||
|
|
||||
private long lastFlushTime = System.currentTimeMillis(); |
|
||||
|
|
||||
public AlertEventHandler(DashboardAlertDao dao, int batchSize) { |
public AlertEventHandler(DashboardAlertDao dao, int batchSize) { |
||||
|
super(batchSize); |
||||
this.dao = dao; |
this.dao = dao; |
||||
this.batchSize = batchSize; |
|
||||
this.buffer = new ArrayList<>(batchSize); |
|
||||
} |
} |
||||
|
|
||||
@Override |
@Override |
||||
public void onEvent(AlertEvent e, long seq, boolean endOfBatch) { |
protected AlertEvent copyOf(AlertEvent e) { |
||||
buffer.add(copyOf(e)); |
|
||||
|
|
||||
// 自动 flush:达到批量条数或超过 1 秒未 flush
|
|
||||
if (buffer.size() >= batchSize || System.currentTimeMillis() - lastFlushTime > 1000) { |
|
||||
flush(); |
|
||||
} |
|
||||
|
|
||||
e.clear(); |
|
||||
} |
|
||||
|
|
||||
private AlertEvent copyOf(AlertEvent e) { |
|
||||
AlertEvent a = new AlertEvent(); |
AlertEvent a = new AlertEvent(); |
||||
a.setEntity(e.getEntity()); |
a.setEntity(e.getEntity()); |
||||
return a; |
return a; |
||||
} |
} |
||||
|
|
||||
private void flush() { |
|
||||
if (buffer.isEmpty()) return; |
|
||||
|
|
||||
try { |
|
||||
dao.batchUpsertRawData(buffer); |
|
||||
} catch (Exception ex) { |
|
||||
log.error("alert batch DB failed", ex); |
|
||||
} |
|
||||
|
|
||||
buffer.clear(); |
|
||||
lastFlushTime = System.currentTimeMillis(); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public void onStart() {} |
|
||||
|
|
||||
@Override |
@Override |
||||
public void onShutdown() { |
protected void flushToDb(java.util.List<AlertEvent> list) { |
||||
flush(); // 程序退出 flush
|
dao.batchUpsertRawData(list); |
||||
} |
} |
||||
} |
} |
||||
|
|||||
@ -0,0 +1,80 @@ |
|||||
|
package com.techsor.datacenter.sender.disruptor; |
||||
|
|
||||
|
import com.lmax.disruptor.EventHandler; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* 通用批量写库 Handler:支持 batchSize、定时 flush、线程安全 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public abstract class BaseBatchEventHandler<T> implements EventHandler<T> { |
||||
|
|
||||
|
protected final int batchSize; |
||||
|
|
||||
|
protected final List<T> buffer = new ArrayList<>(); |
||||
|
private final Object lock = new Object(); |
||||
|
|
||||
|
public BaseBatchEventHandler(int batchSize) { |
||||
|
this.batchSize = batchSize; |
||||
|
} |
||||
|
|
||||
|
// 子类实现写库逻辑
|
||||
|
protected abstract void flushToDb(List<T> events); |
||||
|
|
||||
|
// 子类负责 copy(避免对象复用覆盖)
|
||||
|
protected abstract T copyOf(T e); |
||||
|
|
||||
|
@Override |
||||
|
public void onEvent(T event, long seq, boolean endOfBatch) { |
||||
|
synchronized (lock) { |
||||
|
buffer.add(copyOf(event)); |
||||
|
|
||||
|
if (buffer.size() >= batchSize) { |
||||
|
flushLocked(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void flushLocked() { |
||||
|
if (buffer.isEmpty()) return; |
||||
|
|
||||
|
try { |
||||
|
flushToDb(buffer); |
||||
|
} catch (Exception ex) { |
||||
|
log.error("batch flush failed", ex); |
||||
|
} |
||||
|
|
||||
|
buffer.clear(); |
||||
|
} |
||||
|
|
||||
|
protected void flush() { |
||||
|
synchronized (lock) { |
||||
|
flushLocked(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onStart() { |
||||
|
Thread flusher = new Thread(() -> { |
||||
|
while (true) { |
||||
|
try { |
||||
|
Thread.sleep(1000); // 每秒检查一次
|
||||
|
flush(); |
||||
|
} catch (Exception e) { |
||||
|
log.error("timer flush failed", e); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
|
||||
|
flusher.setDaemon(true); |
||||
|
flusher.start(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onShutdown() { |
||||
|
flush(); |
||||
|
} |
||||
|
} |
||||
@ -1,69 +1,32 @@ |
|||||
package com.techsor.datacenter.sender.disruptor; |
package com.techsor.datacenter.sender.disruptor; |
||||
|
|
||||
import com.lmax.disruptor.EventHandler; |
|
||||
import com.techsor.datacenter.sender.dao.DashboardStatisticsDao; |
import com.techsor.datacenter.sender.dao.DashboardStatisticsDao; |
||||
import lombok.extern.slf4j.Slf4j; |
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
import java.util.ArrayList; |
|
||||
import java.util.List; |
|
||||
|
|
||||
@Slf4j |
@Slf4j |
||||
public class MeasureEventHandler implements EventHandler<MeasureEvent> { |
public class MeasureEventHandler extends BaseBatchEventHandler<MeasureEvent> { |
||||
|
|
||||
private final DashboardStatisticsDao dao; |
private final DashboardStatisticsDao dao; |
||||
|
|
||||
private final int batchSize; |
|
||||
|
|
||||
private final List<MeasureEvent> buffer; |
|
||||
|
|
||||
private long lastFlushTime = System.currentTimeMillis(); |
|
||||
|
|
||||
public MeasureEventHandler(DashboardStatisticsDao dao, int batchSize) { |
public MeasureEventHandler(DashboardStatisticsDao dao, int batchSize) { |
||||
|
super(batchSize); |
||||
this.dao = dao; |
this.dao = dao; |
||||
this.batchSize = batchSize; |
|
||||
this.buffer = new ArrayList<>(batchSize); |
|
||||
} |
} |
||||
|
|
||||
@Override |
@Override |
||||
public void onEvent(MeasureEvent e, long seq, boolean endOfBatch) { |
protected MeasureEvent copyOf(MeasureEvent e) { |
||||
buffer.add(copyOf(e)); |
|
||||
|
|
||||
// 自动 flush
|
|
||||
if (buffer.size() >= batchSize || System.currentTimeMillis() - lastFlushTime > 1000) { |
|
||||
flush(); |
|
||||
} |
|
||||
|
|
||||
e.clear(); |
|
||||
} |
|
||||
|
|
||||
private MeasureEvent copyOf(MeasureEvent e) { |
|
||||
MeasureEvent m = new MeasureEvent(); |
MeasureEvent m = new MeasureEvent(); |
||||
m.setUploadValue(e.getUploadValue()); |
m.setUploadValue(e.getUploadValue()); |
||||
m.setDeviceId(e.getDeviceId()); |
m.setDeviceId(e.getDeviceId()); |
||||
|
m.setAttrCode(e.getAttrCode()); |
||||
m.setInfo(e.getInfo()); |
m.setInfo(e.getInfo()); |
||||
m.setMinValue(e.getMinValue()); |
m.setMinValue(e.getMinValue()); |
||||
m.setMaxValue(e.getMaxValue()); |
m.setMaxValue(e.getMaxValue()); |
||||
return m; |
return m; |
||||
} |
} |
||||
|
|
||||
private void flush() { |
|
||||
if (buffer.isEmpty()) return; |
|
||||
|
|
||||
try { |
|
||||
dao.measureBatchInsert(buffer); |
|
||||
} catch (Exception ex) { |
|
||||
log.error("batch DB failed", ex); |
|
||||
} |
|
||||
|
|
||||
buffer.clear(); |
|
||||
lastFlushTime = System.currentTimeMillis(); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public void onStart() {} |
|
||||
|
|
||||
@Override |
@Override |
||||
public void onShutdown() { |
protected void flushToDb(java.util.List<MeasureEvent> list) { |
||||
flush(); // 程序退出 flush
|
dao.measureBatchInsert(list); |
||||
} |
} |
||||
} |
} |
||||
|
|||||
Loading…
Reference in new issue