C3MA/DD3-LoRa-Bridge-MultiSender
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use compact binary payload for LoRa batches

Browse Source
This commit is contained in:
acidburns
2026-02-01 22:37:21 +01:00
parent d27b68c1cc
commit e24798eb55
3 changed files with 501 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
src/main.cpp
View File

@@ -2,6 +2,7 @@
#include "config.h"
#include "data_model.h"
#include "json_codec.h"
#include "payload_codec.h"
#include "compressor.h"
#include "lora_transport.h"
#include "meter_driver.h"
@@ -12,8 +13,8 @@
#include "web_server.h"
#include "display_ui.h"
#include "test_mode.h"
#include <ArduinoJson.h>
#include <stdarg.h>
#include <math.h>
#ifdef ARDUINO_ARCH_ESP32
#include <esp_task_wdt.h>
#include <esp_system.h>
@@ -51,7 +52,6 @@ static bool g_receiver_discovery_sent = false;
static constexpr size_t BATCH_HEADER_SIZE = 6;
static constexpr size_t BATCH_CHUNK_PAYLOAD = LORA_MAX_PAYLOAD - BATCH_HEADER_SIZE;
static constexpr size_t BATCH_MAX_COMPRESSED = 4096;
static constexpr size_t BATCH_MAX_DECOMPRESSED = 8192;
static constexpr uint32_t BATCH_RX_MARGIN_MS = 800;
struct BatchBuffer {
@@ -102,25 +102,6 @@ static void serial_debug_printf(const char *fmt, ...) {
Serial.println(buf);
}
static void serial_debug_print_json(const String &json) {
if (!SERIAL_DEBUG_MODE || !SERIAL_DEBUG_DUMP_JSON) {
return;
}
const char *data = json.c_str();
size_t len = json.length();
const size_t chunk = 128;
for (size_t i = 0; i < len; i += chunk) {
size_t n = len - i;
if (n > chunk) {
n = chunk;
}
Serial.write(reinterpret_cast<const uint8_t *>(data + i), n);
watchdog_kick();
delay(0);
}
Serial.write('\n');
}
static uint16_t g_last_batch_id_rx[NUM_SENDERS] = {};
struct BatchRxState {
@@ -283,6 +264,63 @@ static uint16_t read_u16_le(const uint8_t *src) {
return static_cast<uint16_t>(src[0]) | (static_cast<uint16_t>(src[1]) << 8);
}
static uint16_t sender_id_from_short_id(uint16_t short_id) {
for (uint8_t i = 0; i < NUM_SENDERS; ++i) {
if (EXPECTED_SENDER_IDS[i] == short_id) {
return static_cast<uint16_t>(i + 1);
}
}
return 0;
}
static uint16_t short_id_from_sender_id(uint16_t sender_id) {
if (sender_id == 0 || sender_id > NUM_SENDERS) {
return 0;
}
return EXPECTED_SENDER_IDS[sender_id - 1];
}
static uint32_t kwh_to_wh_from_float(float value) {
if (isnan(value)) {
return 0;
}
double wh = static_cast<double>(value) * 1000.0;
if (wh < 0.0) {
wh = 0.0;
}
if (wh > static_cast<double>(UINT32_MAX)) {
wh = static_cast<double>(UINT32_MAX);
}
return static_cast<uint32_t>(llround(wh));
}
static bool float_to_i16_w(float value, int16_t &out) {
if (isnan(value)) {
out = 0;
return true;
}
long rounded = lroundf(value);
if (rounded < INT16_MIN || rounded > INT16_MAX) {
return false;
}
out = static_cast<int16_t>(rounded);
return true;
}
static uint16_t battery_mv_from_voltage(float value) {
if (isnan(value) || value <= 0.0f) {
return 0;
}
long mv = lroundf(value * 1000.0f);
if (mv < 0) {
mv = 0;
}
if (mv > UINT16_MAX) {
mv = UINT16_MAX;
}
return static_cast<uint16_t>(mv);
}
static uint32_t compute_batch_rx_timeout_ms(uint16_t total_len, uint8_t chunk_count) {
if (total_len == 0 || chunk_count == 0) {
return 10000;
@@ -306,22 +344,6 @@ static uint32_t compute_batch_ack_timeout_ms(size_t payload_len) {
return timeout_ms < 10000 ? 10000 : timeout_ms;
}
static bool inject_batch_meta(String &json, int16_t rssi_dbm, float snr_db, uint32_t rx_ts_utc) {
DynamicJsonDocument doc(8192);
DeserializationError err = deserializeJson(doc, json);
if (err) {
return false;
}
JsonObject meta = doc.createNestedObject("meta");
meta["rssi"] = rssi_dbm;
meta["snr"] = snr_db;
meta["rx_ts"] = rx_ts_utc;
json = "";
return serializeJson(doc, json) > 0;
}
static bool send_batch_payload(const uint8_t *data, size_t len, uint32_t ts_for_display, uint16_t batch_id) {
if (!data || len == 0 || len > BATCH_MAX_COMPRESSED) {
return false;
@@ -411,41 +433,48 @@ static bool send_inflight_batch(uint32_t ts_for_display) {
if (!g_inflight_active || g_inflight_count == 0) {
return false;
}
uint32_t json_start = millis();
String json;
if (!meterBatchToJson(g_inflight_samples, g_inflight_count, g_inflight_batch_id, json, &g_sender_faults, g_sender_last_error)) {
return false;
}
uint32_t json_ms = millis() - json_start;
if (SERIAL_DEBUG_MODE) {
serial_debug_printf("tx: batch_id=%u count=%u json_len=%u", g_inflight_batch_id, g_inflight_count, static_cast<unsigned>(json.length()));
if (json_ms > 200) {
serial_debug_printf("tx: json encode took %lums", static_cast<unsigned long>(json_ms));
BatchInput input = {};
input.sender_id = sender_id_from_short_id(g_short_id);
input.batch_id = g_inflight_batch_id;
input.t_last = g_inflight_samples[g_inflight_count - 1].ts_utc;
uint32_t dt_s = METER_SAMPLE_INTERVAL_MS / 1000;
input.dt_s = dt_s > 0 ? static_cast<uint8_t>(dt_s) : 1;
input.n = g_inflight_count;
input.battery_mV = battery_mv_from_voltage(g_inflight_samples[g_inflight_count - 1].battery_voltage_v);
for (uint8_t i = 0; i < g_inflight_count; ++i) {
input.energy_wh[i] = kwh_to_wh_from_float(g_inflight_samples[i].energy_total_kwh);
if (!float_to_i16_w(g_inflight_samples[i].phase_power_w[0], input.p1_w[i]) ||
!float_to_i16_w(g_inflight_samples[i].phase_power_w[1], input.p2_w[i]) ||
!float_to_i16_w(g_inflight_samples[i].phase_power_w[2], input.p3_w[i])) {
return false;
}
serial_debug_print_json(json);
}
static uint8_t compressed[BATCH_MAX_COMPRESSED];
size_t compressed_len = 0;
uint32_t compress_start = millis();
if (!compressBuffer(reinterpret_cast<const uint8_t *>(json.c_str()), json.length(), compressed, sizeof(compressed), compressed_len)) {
static uint8_t encoded[BATCH_MAX_COMPRESSED];
size_t encoded_len = 0;
uint32_t encode_start = millis();
if (!encode_batch(input, encoded, sizeof(encoded), &encoded_len)) {
return false;
}
uint32_t compress_ms = millis() - compress_start;
if (SERIAL_DEBUG_MODE && compress_ms > 200) {
serial_debug_printf("tx: compress took %lums", static_cast<unsigned long>(compress_ms));
uint32_t encode_ms = millis() - encode_start;
if (SERIAL_DEBUG_MODE) {
serial_debug_printf("tx: batch_id=%u count=%u bin_len=%u", g_inflight_batch_id, g_inflight_count,
static_cast<unsigned>(encoded_len));
if (encode_ms > 200) {
serial_debug_printf("tx: encode took %lums", static_cast<unsigned long>(encode_ms));
}
}
g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(compressed_len);
g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(encoded_len);
uint32_t send_start = millis();
bool ok = send_batch_payload(compressed, compressed_len, ts_for_display, g_inflight_batch_id);
bool ok = send_batch_payload(encoded, encoded_len, ts_for_display, g_inflight_batch_id);
uint32_t send_ms = millis() - send_start;
if (SERIAL_DEBUG_MODE && send_ms > 1000) {
serial_debug_printf("tx: send batch took %lums", static_cast<unsigned long>(send_ms));
}
if (ok) {
g_last_batch_send_ms = millis();
serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast<unsigned>(compressed_len));
serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast<unsigned>(encoded_len));
} else {
serial_debug_printf("tx: send failed batch_id=%u", g_inflight_batch_id);
}
@@ -498,7 +527,7 @@ static void reset_batch_rx() {
g_batch_rx.timeout_ms = 0;
}
static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool &decode_error, uint16_t &out_batch_id) {
static bool process_batch_packet(const LoraPacket &pkt, BatchInput &out_batch, bool &decode_error, uint16_t &out_batch_id) {
decode_error = false;
if (pkt.payload_len < BATCH_HEADER_SIZE) {
return false;
@@ -545,20 +574,11 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool &
g_batch_rx.last_rx_ms = now_ms;
if (g_batch_rx.next_index == g_batch_rx.expected_chunks && g_batch_rx.received_len == g_batch_rx.total_len) {
static uint8_t decompressed[BATCH_MAX_DECOMPRESSED];
size_t decompressed_len = 0;
if (!decompressBuffer(g_batch_rx.buffer, g_batch_rx.received_len, decompressed, sizeof(decompressed) - 1, decompressed_len)) {
if (!decode_batch(g_batch_rx.buffer, g_batch_rx.received_len, &out_batch)) {
decode_error = true;
reset_batch_rx();
return false;
}
if (decompressed_len >= sizeof(decompressed)) {
decode_error = true;
reset_batch_rx();
return false;
}
decompressed[decompressed_len] = '\0';
out_json = String(reinterpret_cast<const char *>(decompressed));
out_batch_id = batch_id;
reset_batch_rx();
return true;
@@ -570,6 +590,9 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool &
void setup() {
Serial.begin(115200);
delay(200);
#ifdef PAYLOAD_CODEC_TEST
payload_codec_self_test();
#endif
watchdog_init();
g_boot_ms = millis();
@@ -806,15 +829,10 @@ static void receiver_loop() {
}
}
} else if (pkt.payload_type == PayloadType::MeterBatch) {
String json;
BatchInput batch = {};
bool decode_error = false;
uint16_t batch_id = 0;
if (process_batch_packet(pkt, json, decode_error, batch_id)) {
uint32_t rx_ts_utc = time_get_utc();
if (rx_ts_utc == 0) {
rx_ts_utc = millis() / 1000;
}
inject_batch_meta(json, pkt.rssi_dbm, pkt.snr_db, rx_ts_utc);
if (process_batch_packet(pkt, batch, decode_error, batch_id)) {
MeterData samples[METER_BATCH_MAX_SAMPLES];
size_t count = 0;
int8_t sender_idx = -1;
@@ -827,13 +845,59 @@ static void receiver_loop() {
bool duplicate = sender_idx >= 0 && g_last_batch_id_rx[sender_idx] == batch_id;
if (duplicate) {
send_batch_ack(batch_id, pkt.device_id_short);
} else if (jsonToMeterBatch(json, samples, METER_BATCH_MAX_SAMPLES, count)) {
} else {
count = batch.n;
if (count == 0 || count > METER_BATCH_MAX_SAMPLES) {
note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode);
display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms);
return;
}
uint16_t short_id = pkt.device_id_short;
if (short_id == 0) {
short_id = short_id_from_sender_id(batch.sender_id);
}
uint64_t span = static_cast<uint64_t>(batch.dt_s) * static_cast<uint64_t>(count - 1);
if (batch.t_last < span) {
note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode);
display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms);
return;
}
uint32_t t_first = batch.t_last - static_cast<uint32_t>(span);
float bat_v = batch.battery_mV > 0 ? static_cast<float>(batch.battery_mV) / 1000.0f : NAN;
for (size_t s = 0; s < count; ++s) {
MeterData &data = samples[s];
data = {};
data.short_id = short_id;
if (short_id != 0) {
snprintf(data.device_id, sizeof(data.device_id), "dd3-%04X", short_id);
} else {
snprintf(data.device_id, sizeof(data.device_id), "dd3-0000");
}
data.ts_utc = t_first + static_cast<uint32_t>(s) * batch.dt_s;
data.energy_total_kwh = static_cast<float>(batch.energy_wh[s]) / 1000.0f;
data.phase_power_w[0] = static_cast<float>(batch.p1_w[s]);
data.phase_power_w[1] = static_cast<float>(batch.p2_w[s]);
data.phase_power_w[2] = static_cast<float>(batch.p3_w[s]);
data.total_power_w = data.phase_power_w[0] + data.phase_power_w[1] + data.phase_power_w[2];
data.battery_voltage_v = bat_v;
if (!isnan(bat_v)) {
data.battery_percent = battery_percent_from_voltage(bat_v);
} else {
data.battery_percent = 0;
}
data.valid = true;
data.link_valid = true;
data.link_rssi_dbm = pkt.rssi_dbm;
data.link_snr_db = pkt.snr_db;
data.err_meter_read = 0;
data.err_decode = 0;
data.err_lora_tx = 0;
data.last_error = FaultType::None;
}
if (sender_idx >= 0) {
web_server_set_last_batch(static_cast<uint8_t>(sender_idx), samples, count);
for (size_t s = 0; s < count; ++s) {
samples[s].link_valid = true;
samples[s].link_rssi_dbm = pkt.rssi_dbm;
samples[s].link_snr_db = pkt.snr_db;
samples[s].short_id = pkt.device_id_short;
mqtt_publish_state(samples[s]);
}
@@ -856,9 +920,6 @@ static void receiver_loop() {
g_last_batch_id_rx[sender_idx] = batch_id;
send_batch_ack(batch_id, pkt.device_id_short);
}
} else {
note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode);
display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms);
}
} else if (decode_error) {
note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode);