From e24798eb558d2bce6430b3a7ba5868eb188151b9 Mon Sep 17 00:00:00 2001 From: acidburns Date: Sun, 1 Feb 2026 22:37:21 +0100 Subject: [PATCH 01/29] Use compact binary payload for LoRa batches --- src/main.cpp | 227 ++++++++++++++++++----------- src/payload_codec.cpp | 325 ++++++++++++++++++++++++++++++++++++++++++ src/payload_codec.h | 32 +++++ 3 files changed, 501 insertions(+), 83 deletions(-) create mode 100644 src/payload_codec.cpp create mode 100644 src/payload_codec.h diff --git a/src/main.cpp b/src/main.cpp index c795fe3..28738ab 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2,6 +2,7 @@ #include "config.h" #include "data_model.h" #include "json_codec.h" +#include "payload_codec.h" #include "compressor.h" #include "lora_transport.h" #include "meter_driver.h" @@ -12,8 +13,8 @@ #include "web_server.h" #include "display_ui.h" #include "test_mode.h" -#include #include +#include #ifdef ARDUINO_ARCH_ESP32 #include #include @@ -51,7 +52,6 @@ static bool g_receiver_discovery_sent = false; static constexpr size_t BATCH_HEADER_SIZE = 6; static constexpr size_t BATCH_CHUNK_PAYLOAD = LORA_MAX_PAYLOAD - BATCH_HEADER_SIZE; static constexpr size_t BATCH_MAX_COMPRESSED = 4096; -static constexpr size_t BATCH_MAX_DECOMPRESSED = 8192; static constexpr uint32_t BATCH_RX_MARGIN_MS = 800; struct BatchBuffer { @@ -102,25 +102,6 @@ static void serial_debug_printf(const char *fmt, ...) { Serial.println(buf); } -static void serial_debug_print_json(const String &json) { - if (!SERIAL_DEBUG_MODE || !SERIAL_DEBUG_DUMP_JSON) { - return; - } - const char *data = json.c_str(); - size_t len = json.length(); - const size_t chunk = 128; - for (size_t i = 0; i < len; i += chunk) { - size_t n = len - i; - if (n > chunk) { - n = chunk; - } - Serial.write(reinterpret_cast(data + i), n); - watchdog_kick(); - delay(0); - } - Serial.write('\n'); -} - static uint16_t g_last_batch_id_rx[NUM_SENDERS] = {}; struct BatchRxState { @@ -283,6 +264,63 @@ static uint16_t read_u16_le(const uint8_t *src) { return static_cast(src[0]) | (static_cast(src[1]) << 8); } +static uint16_t sender_id_from_short_id(uint16_t short_id) { + for (uint8_t i = 0; i < NUM_SENDERS; ++i) { + if (EXPECTED_SENDER_IDS[i] == short_id) { + return static_cast(i + 1); + } + } + return 0; +} + +static uint16_t short_id_from_sender_id(uint16_t sender_id) { + if (sender_id == 0 || sender_id > NUM_SENDERS) { + return 0; + } + return EXPECTED_SENDER_IDS[sender_id - 1]; +} + +static uint32_t kwh_to_wh_from_float(float value) { + if (isnan(value)) { + return 0; + } + double wh = static_cast(value) * 1000.0; + if (wh < 0.0) { + wh = 0.0; + } + if (wh > static_cast(UINT32_MAX)) { + wh = static_cast(UINT32_MAX); + } + return static_cast(llround(wh)); +} + +static bool float_to_i16_w(float value, int16_t &out) { + if (isnan(value)) { + out = 0; + return true; + } + long rounded = lroundf(value); + if (rounded < INT16_MIN || rounded > INT16_MAX) { + return false; + } + out = static_cast(rounded); + return true; +} + +static uint16_t battery_mv_from_voltage(float value) { + if (isnan(value) || value <= 0.0f) { + return 0; + } + long mv = lroundf(value * 1000.0f); + if (mv < 0) { + mv = 0; + } + if (mv > UINT16_MAX) { + mv = UINT16_MAX; + } + return static_cast(mv); +} + static uint32_t compute_batch_rx_timeout_ms(uint16_t total_len, uint8_t chunk_count) { if (total_len == 0 || chunk_count == 0) { return 10000; @@ -306,22 +344,6 @@ static uint32_t compute_batch_ack_timeout_ms(size_t payload_len) { return timeout_ms < 10000 ? 10000 : timeout_ms; } -static bool inject_batch_meta(String &json, int16_t rssi_dbm, float snr_db, uint32_t rx_ts_utc) { - DynamicJsonDocument doc(8192); - DeserializationError err = deserializeJson(doc, json); - if (err) { - return false; - } - - JsonObject meta = doc.createNestedObject("meta"); - meta["rssi"] = rssi_dbm; - meta["snr"] = snr_db; - meta["rx_ts"] = rx_ts_utc; - - json = ""; - return serializeJson(doc, json) > 0; -} - static bool send_batch_payload(const uint8_t *data, size_t len, uint32_t ts_for_display, uint16_t batch_id) { if (!data || len == 0 || len > BATCH_MAX_COMPRESSED) { return false; @@ -411,41 +433,48 @@ static bool send_inflight_batch(uint32_t ts_for_display) { if (!g_inflight_active || g_inflight_count == 0) { return false; } - uint32_t json_start = millis(); - String json; - if (!meterBatchToJson(g_inflight_samples, g_inflight_count, g_inflight_batch_id, json, &g_sender_faults, g_sender_last_error)) { - return false; - } - uint32_t json_ms = millis() - json_start; - if (SERIAL_DEBUG_MODE) { - serial_debug_printf("tx: batch_id=%u count=%u json_len=%u", g_inflight_batch_id, g_inflight_count, static_cast(json.length())); - if (json_ms > 200) { - serial_debug_printf("tx: json encode took %lums", static_cast(json_ms)); + BatchInput input = {}; + input.sender_id = sender_id_from_short_id(g_short_id); + input.batch_id = g_inflight_batch_id; + input.t_last = g_inflight_samples[g_inflight_count - 1].ts_utc; + uint32_t dt_s = METER_SAMPLE_INTERVAL_MS / 1000; + input.dt_s = dt_s > 0 ? static_cast(dt_s) : 1; + input.n = g_inflight_count; + input.battery_mV = battery_mv_from_voltage(g_inflight_samples[g_inflight_count - 1].battery_voltage_v); + for (uint8_t i = 0; i < g_inflight_count; ++i) { + input.energy_wh[i] = kwh_to_wh_from_float(g_inflight_samples[i].energy_total_kwh); + if (!float_to_i16_w(g_inflight_samples[i].phase_power_w[0], input.p1_w[i]) || + !float_to_i16_w(g_inflight_samples[i].phase_power_w[1], input.p2_w[i]) || + !float_to_i16_w(g_inflight_samples[i].phase_power_w[2], input.p3_w[i])) { + return false; } - serial_debug_print_json(json); } - static uint8_t compressed[BATCH_MAX_COMPRESSED]; - size_t compressed_len = 0; - uint32_t compress_start = millis(); - if (!compressBuffer(reinterpret_cast(json.c_str()), json.length(), compressed, sizeof(compressed), compressed_len)) { + static uint8_t encoded[BATCH_MAX_COMPRESSED]; + size_t encoded_len = 0; + uint32_t encode_start = millis(); + if (!encode_batch(input, encoded, sizeof(encoded), &encoded_len)) { return false; } - uint32_t compress_ms = millis() - compress_start; - if (SERIAL_DEBUG_MODE && compress_ms > 200) { - serial_debug_printf("tx: compress took %lums", static_cast(compress_ms)); + uint32_t encode_ms = millis() - encode_start; + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("tx: batch_id=%u count=%u bin_len=%u", g_inflight_batch_id, g_inflight_count, + static_cast(encoded_len)); + if (encode_ms > 200) { + serial_debug_printf("tx: encode took %lums", static_cast(encode_ms)); + } } - g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(compressed_len); + g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(encoded_len); uint32_t send_start = millis(); - bool ok = send_batch_payload(compressed, compressed_len, ts_for_display, g_inflight_batch_id); + bool ok = send_batch_payload(encoded, encoded_len, ts_for_display, g_inflight_batch_id); uint32_t send_ms = millis() - send_start; if (SERIAL_DEBUG_MODE && send_ms > 1000) { serial_debug_printf("tx: send batch took %lums", static_cast(send_ms)); } if (ok) { g_last_batch_send_ms = millis(); - serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast(compressed_len)); + serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast(encoded_len)); } else { serial_debug_printf("tx: send failed batch_id=%u", g_inflight_batch_id); } @@ -498,7 +527,7 @@ static void reset_batch_rx() { g_batch_rx.timeout_ms = 0; } -static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool &decode_error, uint16_t &out_batch_id) { +static bool process_batch_packet(const LoraPacket &pkt, BatchInput &out_batch, bool &decode_error, uint16_t &out_batch_id) { decode_error = false; if (pkt.payload_len < BATCH_HEADER_SIZE) { return false; @@ -545,20 +574,11 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool & g_batch_rx.last_rx_ms = now_ms; if (g_batch_rx.next_index == g_batch_rx.expected_chunks && g_batch_rx.received_len == g_batch_rx.total_len) { - static uint8_t decompressed[BATCH_MAX_DECOMPRESSED]; - size_t decompressed_len = 0; - if (!decompressBuffer(g_batch_rx.buffer, g_batch_rx.received_len, decompressed, sizeof(decompressed) - 1, decompressed_len)) { + if (!decode_batch(g_batch_rx.buffer, g_batch_rx.received_len, &out_batch)) { decode_error = true; reset_batch_rx(); return false; } - if (decompressed_len >= sizeof(decompressed)) { - decode_error = true; - reset_batch_rx(); - return false; - } - decompressed[decompressed_len] = '\0'; - out_json = String(reinterpret_cast(decompressed)); out_batch_id = batch_id; reset_batch_rx(); return true; @@ -570,6 +590,9 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool & void setup() { Serial.begin(115200); delay(200); +#ifdef PAYLOAD_CODEC_TEST + payload_codec_self_test(); +#endif watchdog_init(); g_boot_ms = millis(); @@ -806,15 +829,10 @@ static void receiver_loop() { } } } else if (pkt.payload_type == PayloadType::MeterBatch) { - String json; + BatchInput batch = {}; bool decode_error = false; uint16_t batch_id = 0; - if (process_batch_packet(pkt, json, decode_error, batch_id)) { - uint32_t rx_ts_utc = time_get_utc(); - if (rx_ts_utc == 0) { - rx_ts_utc = millis() / 1000; - } - inject_batch_meta(json, pkt.rssi_dbm, pkt.snr_db, rx_ts_utc); + if (process_batch_packet(pkt, batch, decode_error, batch_id)) { MeterData samples[METER_BATCH_MAX_SAMPLES]; size_t count = 0; int8_t sender_idx = -1; @@ -827,13 +845,59 @@ static void receiver_loop() { bool duplicate = sender_idx >= 0 && g_last_batch_id_rx[sender_idx] == batch_id; if (duplicate) { send_batch_ack(batch_id, pkt.device_id_short); - } else if (jsonToMeterBatch(json, samples, METER_BATCH_MAX_SAMPLES, count)) { + } else { + count = batch.n; + if (count == 0 || count > METER_BATCH_MAX_SAMPLES) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + return; + } + uint16_t short_id = pkt.device_id_short; + if (short_id == 0) { + short_id = short_id_from_sender_id(batch.sender_id); + } + uint64_t span = static_cast(batch.dt_s) * static_cast(count - 1); + if (batch.t_last < span) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + return; + } + uint32_t t_first = batch.t_last - static_cast(span); + float bat_v = batch.battery_mV > 0 ? static_cast(batch.battery_mV) / 1000.0f : NAN; + for (size_t s = 0; s < count; ++s) { + MeterData &data = samples[s]; + data = {}; + data.short_id = short_id; + if (short_id != 0) { + snprintf(data.device_id, sizeof(data.device_id), "dd3-%04X", short_id); + } else { + snprintf(data.device_id, sizeof(data.device_id), "dd3-0000"); + } + data.ts_utc = t_first + static_cast(s) * batch.dt_s; + data.energy_total_kwh = static_cast(batch.energy_wh[s]) / 1000.0f; + data.phase_power_w[0] = static_cast(batch.p1_w[s]); + data.phase_power_w[1] = static_cast(batch.p2_w[s]); + data.phase_power_w[2] = static_cast(batch.p3_w[s]); + data.total_power_w = data.phase_power_w[0] + data.phase_power_w[1] + data.phase_power_w[2]; + data.battery_voltage_v = bat_v; + if (!isnan(bat_v)) { + data.battery_percent = battery_percent_from_voltage(bat_v); + } else { + data.battery_percent = 0; + } + data.valid = true; + data.link_valid = true; + data.link_rssi_dbm = pkt.rssi_dbm; + data.link_snr_db = pkt.snr_db; + data.err_meter_read = 0; + data.err_decode = 0; + data.err_lora_tx = 0; + data.last_error = FaultType::None; + } + if (sender_idx >= 0) { web_server_set_last_batch(static_cast(sender_idx), samples, count); for (size_t s = 0; s < count; ++s) { - samples[s].link_valid = true; - samples[s].link_rssi_dbm = pkt.rssi_dbm; - samples[s].link_snr_db = pkt.snr_db; samples[s].short_id = pkt.device_id_short; mqtt_publish_state(samples[s]); } @@ -856,9 +920,6 @@ static void receiver_loop() { g_last_batch_id_rx[sender_idx] = batch_id; send_batch_ack(batch_id, pkt.device_id_short); } - } else { - note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); - display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); } } else if (decode_error) { note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); diff --git a/src/payload_codec.cpp b/src/payload_codec.cpp new file mode 100644 index 0000000..f962b3b --- /dev/null +++ b/src/payload_codec.cpp @@ -0,0 +1,325 @@ +#include "payload_codec.h" +#include + +static constexpr uint16_t kMagic = 0xDDB3; +static constexpr uint8_t kSchema = 1; +static constexpr uint8_t kFlags = 0x01; +static constexpr size_t kMaxSamples = 30; + +static void write_u16_le(uint8_t *dst, uint16_t value) { + dst[0] = static_cast(value & 0xFF); + dst[1] = static_cast((value >> 8) & 0xFF); +} + +static void write_u32_le(uint8_t *dst, uint32_t value) { + dst[0] = static_cast(value & 0xFF); + dst[1] = static_cast((value >> 8) & 0xFF); + dst[2] = static_cast((value >> 16) & 0xFF); + dst[3] = static_cast((value >> 24) & 0xFF); +} + +static uint16_t read_u16_le(const uint8_t *src) { + return static_cast(src[0]) | (static_cast(src[1]) << 8); +} + +static uint32_t read_u32_le(const uint8_t *src) { + return static_cast(src[0]) | + (static_cast(src[1]) << 8) | + (static_cast(src[2]) << 16) | + (static_cast(src[3]) << 24); +} + +size_t uleb128_encode(uint32_t v, uint8_t *out, size_t cap) { + size_t i = 0; + do { + if (i >= cap) { + return 0; + } + uint8_t byte = static_cast(v & 0x7F); + v >>= 7; + if (v != 0) { + byte |= 0x80; + } + out[i++] = byte; + } while (v != 0); + return i; +} + +bool uleb128_decode(const uint8_t *in, size_t len, size_t *pos, uint32_t *v) { + if (!in || !pos || !v) { + return false; + } + uint32_t result = 0; + uint8_t shift = 0; + size_t p = *pos; + for (uint8_t i = 0; i < 5; ++i) { + if (p >= len) { + return false; + } + uint8_t byte = in[p++]; + if (i == 4 && (byte & 0xF0) != 0) { + return false; + } + result |= static_cast(byte & 0x7F) << shift; + if ((byte & 0x80) == 0) { + *pos = p; + *v = result; + return true; + } + shift = static_cast(shift + 7); + } + return false; +} + +uint32_t zigzag32(int32_t x) { + return (static_cast(x) << 1) ^ static_cast(x >> 31); +} + +int32_t unzigzag32(uint32_t u) { + return static_cast((u >> 1) ^ (static_cast(-static_cast(u & 1)))); +} + +size_t svarint_encode(int32_t x, uint8_t *out, size_t cap) { + uint32_t zz = zigzag32(x); + return uleb128_encode(zz, out, cap); +} + +bool svarint_decode(const uint8_t *in, size_t len, size_t *pos, int32_t *x) { + uint32_t u = 0; + if (!uleb128_decode(in, len, pos, &u)) { + return false; + } + *x = unzigzag32(u); + return true; +} + +static bool ensure_capacity(size_t needed, size_t cap, size_t pos) { + return pos + needed <= cap; +} + +bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *out_len) { + if (!out || !out_len) { + return false; + } + if (in.n == 0 || in.n > kMaxSamples) { + return false; + } + if (in.dt_s == 0) { + return false; + } + size_t pos = 0; + if (!ensure_capacity(16, out_cap, pos)) { + return false; + } + write_u16_le(&out[pos], kMagic); + pos += 2; + out[pos++] = kSchema; + out[pos++] = kFlags; + write_u16_le(&out[pos], in.sender_id); + pos += 2; + write_u16_le(&out[pos], in.batch_id); + pos += 2; + write_u32_le(&out[pos], in.t_last); + pos += 4; + out[pos++] = in.dt_s; + out[pos++] = in.n; + write_u16_le(&out[pos], in.battery_mV); + pos += 2; + + if (!ensure_capacity(4, out_cap, pos)) { + return false; + } + write_u32_le(&out[pos], in.energy_wh[0]); + pos += 4; + for (uint8_t i = 1; i < in.n; ++i) { + if (in.energy_wh[i] < in.energy_wh[i - 1]) { + return false; + } + uint32_t delta = in.energy_wh[i] - in.energy_wh[i - 1]; + size_t wrote = uleb128_encode(delta, &out[pos], out_cap - pos); + if (wrote == 0) { + return false; + } + pos += wrote; + } + + auto encode_phase = [&](const int16_t *phase) -> bool { + if (!ensure_capacity(2, out_cap, pos)) { + return false; + } + write_u16_le(&out[pos], static_cast(phase[0])); + pos += 2; + for (uint8_t i = 1; i < in.n; ++i) { + int32_t delta = static_cast(phase[i]) - static_cast(phase[i - 1]); + size_t wrote = svarint_encode(delta, &out[pos], out_cap - pos); + if (wrote == 0) { + return false; + } + pos += wrote; + } + return true; + }; + + if (!encode_phase(in.p1_w)) { + return false; + } + if (!encode_phase(in.p2_w)) { + return false; + } + if (!encode_phase(in.p3_w)) { + return false; + } + + *out_len = pos; + return true; +} + +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out) { + if (!buf || !out) { + return false; + } + size_t pos = 0; + if (len < 16) { + return false; + } + uint16_t magic = read_u16_le(&buf[pos]); + pos += 2; + uint8_t schema = buf[pos++]; + uint8_t flags = buf[pos++]; + if (magic != kMagic || schema != kSchema || (flags & 0x01) == 0) { + return false; + } + out->sender_id = read_u16_le(&buf[pos]); + pos += 2; + out->batch_id = read_u16_le(&buf[pos]); + pos += 2; + out->t_last = read_u32_le(&buf[pos]); + pos += 4; + out->dt_s = buf[pos++]; + out->n = buf[pos++]; + out->battery_mV = read_u16_le(&buf[pos]); + pos += 2; + + if (out->n == 0 || out->n > kMaxSamples || out->dt_s == 0) { + return false; + } + if (pos + 4 > len) { + return false; + } + out->energy_wh[0] = read_u32_le(&buf[pos]); + pos += 4; + for (uint8_t i = 1; i < out->n; ++i) { + uint32_t delta = 0; + if (!uleb128_decode(buf, len, &pos, &delta)) { + return false; + } + uint64_t sum = static_cast(out->energy_wh[i - 1]) + static_cast(delta); + if (sum > UINT32_MAX) { + return false; + } + out->energy_wh[i] = static_cast(sum); + } + + auto decode_phase = [&](int16_t *phase) -> bool { + if (pos + 2 > len) { + return false; + } + phase[0] = static_cast(read_u16_le(&buf[pos])); + pos += 2; + int32_t prev = static_cast(phase[0]); + for (uint8_t i = 1; i < out->n; ++i) { + int32_t delta = 0; + if (!svarint_decode(buf, len, &pos, &delta)) { + return false; + } + int32_t value = prev + delta; + if (value < INT16_MIN || value > INT16_MAX) { + return false; + } + phase[i] = static_cast(value); + prev = value; + } + return true; + }; + + if (!decode_phase(out->p1_w)) { + return false; + } + if (!decode_phase(out->p2_w)) { + return false; + } + if (!decode_phase(out->p3_w)) { + return false; + } + + for (uint8_t i = out->n; i < kMaxSamples; ++i) { + out->energy_wh[i] = 0; + out->p1_w[i] = 0; + out->p2_w[i] = 0; + out->p3_w[i] = 0; + } + + return pos == len; +} + +#ifdef PAYLOAD_CODEC_TEST +bool payload_codec_self_test() { + BatchInput in = {}; + in.sender_id = 1; + in.batch_id = 42; + in.t_last = 1700000000; + in.dt_s = 1; + in.n = 5; + in.battery_mV = 3750; + in.energy_wh[0] = 100000; + in.energy_wh[1] = 100001; + in.energy_wh[2] = 100050; + in.energy_wh[3] = 100050; + in.energy_wh[4] = 100200; + in.p1_w[0] = -120; + in.p1_w[1] = -90; + in.p1_w[2] = 1910; + in.p1_w[3] = -90; + in.p1_w[4] = 500; + in.p2_w[0] = 50; + in.p2_w[1] = -1950; + in.p2_w[2] = 60; + in.p2_w[3] = 2060; + in.p2_w[4] = -10; + in.p3_w[0] = 0; + in.p3_w[1] = 10; + in.p3_w[2] = -1990; + in.p3_w[3] = 10; + in.p3_w[4] = 20; + + uint8_t buf[256]; + size_t len = 0; + if (!encode_batch(in, buf, sizeof(buf), &len)) { + Serial.println("payload_codec_self_test: encode failed"); + return false; + } + + BatchInput out = {}; + if (!decode_batch(buf, len, &out)) { + Serial.println("payload_codec_self_test: decode failed"); + return false; + } + + if (out.sender_id != in.sender_id || out.batch_id != in.batch_id || out.t_last != in.t_last || + out.dt_s != in.dt_s || out.n != in.n || out.battery_mV != in.battery_mV) { + Serial.println("payload_codec_self_test: header mismatch"); + return false; + } + + for (uint8_t i = 0; i < in.n; ++i) { + if (out.energy_wh[i] != in.energy_wh[i] || out.p1_w[i] != in.p1_w[i] || out.p2_w[i] != in.p2_w[i] || + out.p3_w[i] != in.p3_w[i]) { + Serial.println("payload_codec_self_test: sample mismatch"); + return false; + } + } + + Serial.printf("payload_codec_self_test: ok len=%u\n", static_cast(len)); + return true; +} +#endif diff --git a/src/payload_codec.h b/src/payload_codec.h new file mode 100644 index 0000000..1098089 --- /dev/null +++ b/src/payload_codec.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +struct BatchInput { + uint16_t sender_id; + uint16_t batch_id; + uint32_t t_last; + uint8_t dt_s; + uint8_t n; + uint16_t battery_mV; + uint32_t energy_wh[30]; + int16_t p1_w[30]; + int16_t p2_w[30]; + int16_t p3_w[30]; +}; + +bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *out_len); +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out); + +size_t uleb128_encode(uint32_t v, uint8_t *out, size_t cap); +bool uleb128_decode(const uint8_t *in, size_t len, size_t *pos, uint32_t *v); + +uint32_t zigzag32(int32_t x); +int32_t unzigzag32(uint32_t u); + +size_t svarint_encode(int32_t x, uint8_t *out, size_t cap); +bool svarint_decode(const uint8_t *in, size_t len, size_t *pos, int32_t *x); + +#ifdef PAYLOAD_CODEC_TEST +bool payload_codec_self_test(); +#endif From e5c4e04ff9a1623b880bed4e4a4d3b0ca3359253 Mon Sep 17 00:00:00 2001 From: acidburns Date: Sun, 1 Feb 2026 22:42:26 +0100 Subject: [PATCH 02/29] Update README for binary batch payload and SF11 --- README.md | 79 +++++++++++++++++++++++------------------------- include/config.h | 2 +- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index e6f59ee..7cd0664 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DD3 LoRa Bridge (Multi-Sender) -Unified firmware for LilyGO T3 v1.6.1 (ESP32 + SX1276 + SSD1306) that runs as **Sender** or **Receiver** based on a GPIO jumper. Senders read DD3 smart meter values, compress JSON, and transmit over LoRa. The receiver validates packets, publishes to MQTT, provides a web UI, and shows per-sender status on the OLED. +Unified firmware for LilyGO T3 v1.6.1 (ESP32 + SX1276 + SSD1306) that runs as **Sender** or **Receiver** based on a GPIO jumper. Senders read DD3 smart meter values and transmit compact binary batches over LoRa. The receiver validates packets, publishes to MQTT, provides a web UI, and shows per-sender status on the OLED. ## Hardware Board: **LilyGO T3 LoRa32 v1.6.1** (ESP32 + SX1276 + SSD1306 128x64 + LiPo) @@ -46,7 +46,7 @@ Variants: - Total power: 1-0:16.7.0*255 - Phase power: 36.7 / 56.7 / 76.7 - Reads battery voltage and estimates SoC. -- Builds JSON payload, compresses, wraps in LoRa packet, transmits. +- Builds compact binary batch payload, wraps in LoRa packet, transmits. - Light sleeps between meter reads; batches are sent every 30s. - Listens for LoRa time sync packets to set UTC clock. - Uses DS3231 RTC after boot if no time sync has arrived yet. @@ -59,9 +59,8 @@ void sender_loop() { read_battery(data); // VBAT + SoC if (time_to_send_batch()) { - json = meterBatchToJson(samples, batch_id); // bat_v per batch, t_first/t_last included - compressed = compressBuffer(json); - lora_send(packet(MeterBatch, compressed)); + payload = encode_batch(samples, batch_id); // compact binary batch + lora_send(packet(MeterBatch, payload)); } display_set_last_meter(data); @@ -79,14 +78,14 @@ void sender_loop() { bool meter_read(MeterData &data); // parse OBIS fields void read_battery(MeterData &data); // ADC -> volts + percent bool meterDataToJson(const MeterData&, String&); -bool compressBuffer(const uint8_t*, size_t, uint8_t*, size_t, size_t&); +bool compressBuffer(const uint8_t*, size_t, uint8_t*, size_t, size_t&); // MeterData only bool lora_send(const LoraPacket &pkt); // add header + CRC16 and transmit ``` ### Receiver (USB-powered) - WiFi STA connect using stored config; if not available/fails, starts AP. - NTP sync (UTC) and local display in Europe/Berlin. -- Receives LoRa packets, verifies CRC16, decompresses, parses JSON. +- Receives LoRa packets, verifies CRC16, decompresses MeterData JSON, decodes binary batches. - Publishes meter JSON to MQTT. - Sends ACKs for MeterBatch packets and de-duplicates by batch_id. - Web UI: @@ -105,8 +104,8 @@ void receiver_loop() { mqtt_publish_state(data); } } else if (pkt.type == MeterBatch) { - json = reassemble_and_decompress_batch(pkt); - for (sample in jsonToMeterBatch(json)) { // uses t_first/t_last for jittered timestamps + batch = reassemble_and_decode_batch(pkt); + for (sample in batch) { update_sender_status(sample); mqtt_publish_state(sample); } @@ -128,7 +127,7 @@ void receiver_loop() { ```cpp bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms); bool jsonToMeterData(const String &json, MeterData &data); -bool jsonToMeterBatch(const String &json, MeterData *samples, size_t max, size_t &count); +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out); bool mqtt_publish_state(const MeterData &data); void web_server_loop(); // AP or STA UI void time_send_timesync(uint16_t self_id); @@ -170,16 +169,16 @@ Packet layout: [1] role (0=sender, 1=receiver) [2..3] device_id_short (uint16) [4] payload_type (0=meter, 1=test, 2=time_sync, 3=meter_batch, 4=ack) -[5..N-3] compressed payload +[5..N-3] payload bytes (compressed JSON for MeterData, binary for MeterBatch/Test/TimeSync) [N-2..N-1] CRC16 (bytes 0..N-3) ``` LoRa radio settings: - Frequency: **433 MHz** or **868 MHz** (set by build env via `LORA_FREQUENCY_HZ`) -- SF10, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 +- SF11, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 ## Data Format -JSON payload (sender + MQTT): +MeterData JSON (sender + MQTT): ```json { @@ -195,36 +194,31 @@ JSON payload (sender + MQTT): } ``` -MeterBatch JSON (compressed over LoRa) uses per-field arrays with integer units for easier ingestion: +### Binary MeterBatch Payload (LoRa) +Fixed header (little-endian): +- `magic` u16 = 0xDDB3 +- `schema` u8 = 1 +- `flags` u8 = 0x01 (bit0 = signed phases) +- `sender_id` u16 (1..NUM_SENDERS, maps to `EXPECTED_SENDER_IDS`) +- `batch_id` u16 +- `t_last` u32 (unix seconds of last sample) +- `dt_s` u8 (seconds, >0) +- `n` u8 (sample count, <=30) +- `battery_mV` u16 -```json -{ - "schema": 1, - "sender": "s01", - "batch_id": 1842, - "t0": 1738288000, - "t_first": 1738288000, - "t_last": 1738288030, - "dt_s": 1, - "n": 3, - "e_wh": [123456700, 123456701, 123456701], - "p_w": [930, 940, 950], - "p1_w": [480, 490, 500], - "p2_w": [450, 450, 450], - "p3_w": [0, 0, 0], - "bat_v": 3.92, - "meta": { - "rssi": -92, - "snr": 7.5, - "rx_ts": 1738288031 - } -} -``` +Body: +- `E0` u32 (absolute energy in Wh) +- `dE[1..n-1]` ULEB128 (delta vs previous, >=0) +- `P1_0` s16 (absolute W) +- `dP1[1..n-1]` signed varint (ZigZag + ULEB128) +- `P2_0` s16 +- `dP2[1..n-1]` signed varint +- `P3_0` s16 +- `dP3[1..n-1]` signed varint Notes: -- `sender` maps to `EXPECTED_SENDER_IDS` order (`s01` = first sender). -- `meta` is injected by the receiver after batch reassembly. -- `bat_v` is a single batch-level value (percent is calculated locally). +- Receiver reconstructs timestamps from `t_last` and `dt_s`. +- Total power is computed on receiver as `p1 + p2 + p3`. ## Device IDs - Derived from WiFi STA MAC. @@ -291,9 +285,9 @@ Key timing settings in `include/config.h`: - `LORA_SEND_BYPASS` (debug only) ## Limits & Known Constraints -- **Compression**: uses lightweight RLE (good for JSON but not optimal). +- **Compression**: MeterData uses lightweight RLE (good for JSON but not optimal). - **OBIS parsing**: supports IEC 62056-21 ASCII (Mode D); may need tuning for some meters. -- **Payload size**: single JSON frames < 256 bytes (ArduinoJson static doc); batch frames are chunked and reassembled. +- **Payload size**: single JSON frames < 256 bytes (ArduinoJson static doc); binary batch frames are chunked and reassembled (typically 1 chunk). - **Battery ADC**: uses simple linear calibration constant in `power_manager.cpp`. - **OLED**: no hardware reset line is used (matches working reference). - **Batch ACKs**: sender waits for ACK after a batch and retries up to `BATCH_MAX_RETRIES` with `BATCH_ACK_TIMEOUT_MS` between attempts. @@ -304,6 +298,7 @@ Key timing settings in `include/config.h`: - `include/json_codec.h`, `src/json_codec.cpp`: JSON encode/decode - `include/compressor.h`, `src/compressor.cpp`: RLE compression - `include/lora_transport.h`, `src/lora_transport.cpp`: LoRa packet + CRC +- `src/payload_codec.h`, `src/payload_codec.cpp`: binary batch encoder/decoder - `include/meter_driver.h`, `src/meter_driver.cpp`: IEC 62056-21 ASCII parse - `include/power_manager.h`, `src/power_manager.cpp`: ADC + sleep - `include/time_manager.h`, `src/time_manager.cpp`: NTP + time sync diff --git a/include/config.h b/include/config.h index c9e8fc9..59cfc60 100644 --- a/include/config.h +++ b/include/config.h @@ -49,7 +49,7 @@ constexpr uint8_t PIN_METER_RX = 34; #define LORA_FREQUENCY_HZ 433E6 #endif constexpr long LORA_FREQUENCY = LORA_FREQUENCY_HZ; -constexpr uint8_t LORA_SPREADING_FACTOR = 10; +constexpr uint8_t LORA_SPREADING_FACTOR = 11; constexpr long LORA_BANDWIDTH = 125E3; constexpr uint8_t LORA_CODING_RATE = 5; constexpr uint8_t LORA_SYNC_WORD = 0x34; From 16c1b90b1e5230527ac0505b653f6a31647c818b Mon Sep 17 00:00:00 2001 From: acidburns Date: Sun, 1 Feb 2026 22:54:07 +0100 Subject: [PATCH 03/29] Add payload codec test envs and enable serial debug --- include/config.h | 4 ++-- platformio.ini | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/include/config.h b/include/config.h index 59cfc60..4f5b724 100644 --- a/include/config.h +++ b/include/config.h @@ -73,8 +73,8 @@ constexpr uint8_t BATCH_QUEUE_DEPTH = 10; constexpr BatchRetryPolicy BATCH_RETRY_POLICY = BatchRetryPolicy::Keep; constexpr uint32_t WATCHDOG_TIMEOUT_SEC = 120; constexpr bool ENABLE_HA_DISCOVERY = true; -constexpr bool SERIAL_DEBUG_MODE = false; -constexpr bool SERIAL_DEBUG_DUMP_JSON = false; +constexpr bool SERIAL_DEBUG_MODE = true; +constexpr bool SERIAL_DEBUG_DUMP_JSON = true; constexpr bool LORA_SEND_BYPASS = false; constexpr uint8_t NUM_SENDERS = 1; diff --git a/platformio.ini b/platformio.ini index 277abd6..aaf3db0 100644 --- a/platformio.ini +++ b/platformio.ini @@ -58,3 +58,30 @@ lib_deps = build_flags = -DENABLE_TEST_MODE -DLORA_FREQUENCY_HZ=868E6 + +[env:lilygo-t3-v1-6-1-payload-test] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DPAYLOAD_CODEC_TEST + +[env:lilygo-t3-v1-6-1-868-payload-test] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DPAYLOAD_CODEC_TEST + -DLORA_FREQUENCY_HZ=868E6 From 13f2f02e4205056cd7bfeb4e0f427f6b7d530d5b Mon Sep 17 00:00:00 2001 From: acidburns Date: Sun, 1 Feb 2026 23:38:43 +0100 Subject: [PATCH 04/29] Tidy sender page layout and use SF12 --- include/config.h | 2 +- src/display_ui.cpp | 32 ++++++++++++++++++-------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/config.h b/include/config.h index 4f5b724..250e727 100644 --- a/include/config.h +++ b/include/config.h @@ -49,7 +49,7 @@ constexpr uint8_t PIN_METER_RX = 34; #define LORA_FREQUENCY_HZ 433E6 #endif constexpr long LORA_FREQUENCY = LORA_FREQUENCY_HZ; -constexpr uint8_t LORA_SPREADING_FACTOR = 11; +constexpr uint8_t LORA_SPREADING_FACTOR = 12; constexpr long LORA_BANDWIDTH = 125E3; constexpr uint8_t LORA_CODING_RATE = 5; constexpr uint8_t LORA_SYNC_WORD = 0x34; diff --git a/src/display_ui.cpp b/src/display_ui.cpp index ac5a013..1327004 100644 --- a/src/display_ui.cpp +++ b/src/display_ui.cpp @@ -268,16 +268,10 @@ static void render_receiver_status() { display.printf("Time: %s", time_buf); uint32_t latest = 0; - bool link_valid = false; - int16_t link_rssi = 0; - float link_snr = 0.0f; if (g_statuses) { for (uint8_t i = 0; i < g_status_count; ++i) { if (g_statuses[i].has_data && g_statuses[i].last_update_ts_utc > latest) { latest = g_statuses[i].last_update_ts_utc; - link_valid = g_statuses[i].last_data.link_valid; - link_rssi = g_statuses[i].last_data.link_rssi_dbm; - link_snr = g_statuses[i].last_data.link_snr_db; } } } @@ -291,9 +285,6 @@ static void render_receiver_status() { localtime_r(&t, &timeinfo); display.printf("Upd %02d:%02d", timeinfo.tm_hour, timeinfo.tm_min); } - if (link_valid) { - display.printf(" R:%d S:%.1f", link_rssi, link_snr); - } render_last_error_line(56); display.display(); @@ -343,14 +334,27 @@ static void render_receiver_sender(uint8_t index) { display.setCursor(0, 12); display.printf("E %.1f kWh", status.last_data.energy_total_kwh); - display.setCursor(0, 24); - display.printf("P %.0fW", status.last_data.total_power_w); - display.setCursor(0, 36); + display.setCursor(0, 22); display.printf("L1 %.0fW", status.last_data.phase_power_w[0]); - display.setCursor(0, 48); + display.setCursor(0, 32); display.printf("L2 %.0fW", status.last_data.phase_power_w[1]); - display.setCursor(0, 56); + display.setCursor(0, 42); display.printf("L3 %.0fW", status.last_data.phase_power_w[2]); + display.setCursor(0, 52); + display.print("P"); + char p_buf[16]; + snprintf(p_buf, sizeof(p_buf), "%.0fW", status.last_data.total_power_w); + int16_t x1 = 0; + int16_t y1 = 0; + uint16_t w = 0; + uint16_t h = 0; + display.getTextBounds(p_buf, 0, 0, &x1, &y1, &w, &h); + int16_t x = static_cast(display.width() - w); + if (x < 0) { + x = 0; + } + display.setCursor(x, 52); + display.print(p_buf); display.display(); } From a03c2cdb07b93e34696eb2f61aefe8193f83aa03 Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 00:00:29 +0100 Subject: [PATCH 05/29] Include sender error counters in batch payload --- src/main.cpp | 12 ++++++++---- src/payload_codec.cpp | 19 ++++++++++++++++--- src/payload_codec.h | 4 ++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 28738ab..dcbaf46 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -441,6 +441,10 @@ static bool send_inflight_batch(uint32_t ts_for_display) { input.dt_s = dt_s > 0 ? static_cast(dt_s) : 1; input.n = g_inflight_count; input.battery_mV = battery_mv_from_voltage(g_inflight_samples[g_inflight_count - 1].battery_voltage_v); + input.err_m = g_sender_faults.meter_read_fail > 255 ? 255 : static_cast(g_sender_faults.meter_read_fail); + input.err_d = g_sender_faults.decode_fail > 255 ? 255 : static_cast(g_sender_faults.decode_fail); + input.err_tx = g_sender_faults.lora_tx_fail > 255 ? 255 : static_cast(g_sender_faults.lora_tx_fail); + input.err_last = static_cast(g_sender_last_error); for (uint8_t i = 0; i < g_inflight_count; ++i) { input.energy_wh[i] = kwh_to_wh_from_float(g_inflight_samples[i].energy_total_kwh); if (!float_to_i16_w(g_inflight_samples[i].phase_power_w[0], input.p1_w[i]) || @@ -889,10 +893,10 @@ static void receiver_loop() { data.link_valid = true; data.link_rssi_dbm = pkt.rssi_dbm; data.link_snr_db = pkt.snr_db; - data.err_meter_read = 0; - data.err_decode = 0; - data.err_lora_tx = 0; - data.last_error = FaultType::None; + data.err_meter_read = batch.err_m; + data.err_decode = batch.err_d; + data.err_lora_tx = batch.err_tx; + data.last_error = static_cast(batch.err_last); } if (sender_idx >= 0) { diff --git a/src/payload_codec.cpp b/src/payload_codec.cpp index f962b3b..7781f78 100644 --- a/src/payload_codec.cpp +++ b/src/payload_codec.cpp @@ -108,7 +108,7 @@ bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *ou return false; } size_t pos = 0; - if (!ensure_capacity(16, out_cap, pos)) { + if (!ensure_capacity(20, out_cap, pos)) { return false; } write_u16_le(&out[pos], kMagic); @@ -125,6 +125,10 @@ bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *ou out[pos++] = in.n; write_u16_le(&out[pos], in.battery_mV); pos += 2; + out[pos++] = in.err_m; + out[pos++] = in.err_d; + out[pos++] = in.err_tx; + out[pos++] = in.err_last; if (!ensure_capacity(4, out_cap, pos)) { return false; @@ -179,7 +183,7 @@ bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out) { return false; } size_t pos = 0; - if (len < 16) { + if (len < 20) { return false; } uint16_t magic = read_u16_le(&buf[pos]); @@ -199,6 +203,10 @@ bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out) { out->n = buf[pos++]; out->battery_mV = read_u16_le(&buf[pos]); pos += 2; + out->err_m = buf[pos++]; + out->err_d = buf[pos++]; + out->err_tx = buf[pos++]; + out->err_last = buf[pos++]; if (out->n == 0 || out->n > kMaxSamples || out->dt_s == 0) { return false; @@ -271,6 +279,10 @@ bool payload_codec_self_test() { in.dt_s = 1; in.n = 5; in.battery_mV = 3750; + in.err_m = 2; + in.err_d = 1; + in.err_tx = 3; + in.err_last = 2; in.energy_wh[0] = 100000; in.energy_wh[1] = 100001; in.energy_wh[2] = 100050; @@ -306,7 +318,8 @@ bool payload_codec_self_test() { } if (out.sender_id != in.sender_id || out.batch_id != in.batch_id || out.t_last != in.t_last || - out.dt_s != in.dt_s || out.n != in.n || out.battery_mV != in.battery_mV) { + out.dt_s != in.dt_s || out.n != in.n || out.battery_mV != in.battery_mV || + out.err_m != in.err_m || out.err_d != in.err_d || out.err_tx != in.err_tx || out.err_last != in.err_last) { Serial.println("payload_codec_self_test: header mismatch"); return false; } diff --git a/src/payload_codec.h b/src/payload_codec.h index 1098089..36f388e 100644 --- a/src/payload_codec.h +++ b/src/payload_codec.h @@ -9,6 +9,10 @@ struct BatchInput { uint8_t dt_s; uint8_t n; uint16_t battery_mV; + uint8_t err_m; + uint8_t err_d; + uint8_t err_tx; + uint8_t err_last; uint32_t energy_wh[30]; int16_t p1_w[30]; int16_t p2_w[30]; From 5085b9ad3dfe0b0c7fcf97dfaf59e44e386c34ae Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 00:00:55 +0100 Subject: [PATCH 06/29] Improve receiver web UI fields and manual --- src/web_server.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/web_server.cpp b/src/web_server.cpp index e56bd6b..ac4456e 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -54,7 +54,11 @@ static String render_sender_block(const SenderStatus &status) { } s += "" + String(status.last_data.device_id) + ""; if (status.has_data && status.last_data.link_valid) { - s += " R:" + String(status.last_data.link_rssi_dbm) + " S:" + String(status.last_data.link_snr_db, 1); + s += " RSSI:" + String(status.last_data.link_rssi_dbm) + " SNR:" + String(status.last_data.link_snr_db, 1); + } + if (status.has_data) { + s += " err_tx:" + String(status.last_data.err_lora_tx); + s += " err_last:" + String(static_cast(status.last_data.last_error)); } s += format_faults(idx); s += "
"; @@ -63,7 +67,9 @@ static String render_sender_block(const SenderStatus &status) { } else { s += "Energy: " + String(status.last_data.energy_total_kwh, 3) + " kWh
"; s += "Power: " + String(status.last_data.total_power_w, 1) + " W
"; - s += "Battery: " + String(status.last_data.battery_voltage_v, 2) + " V (" + String(status.last_data.battery_percent) + ")"; + s += "P1/P2/P3: " + String(status.last_data.phase_power_w[0], 1) + " / " + String(status.last_data.phase_power_w[1], 1) + + " / " + String(status.last_data.phase_power_w[2], 1) + " W
"; + s += "Battery: " + String(status.last_data.battery_percent) + "% (" + String(status.last_data.battery_voltage_v, 2) + " V)"; } s += ""; return s; @@ -175,10 +181,12 @@ static void handle_manual() { html += "
  • Energy: total kWh since meter start.
    • "; html += "
  • Power: total active power in W.
    • "; html += "
  • P1/P2/P3: phase power in W.
    • "; - html += "
  • bat_v: battery voltage (V), bat_pct: estimated percent.
    • "; + html += "
  • Battery: percent with voltage in V.
    • "; html += "
  • RSSI/SNR: LoRa link quality from last packet.
    • "; - html += "
  • err_tx: LoRa TX error count; err_last: last error code.
    • "; - html += "
  • faults m/d/tx: meter read/decode/tx counters.
    • "; + html += "
  • err_tx: sender-side LoRa TX error counter.
    • "; + html += "
  • err_last: last error code (0=None, 1=MeterRead, 2=Decode, 3=LoraTx).
    • "; + html += "
  • faults m/d/tx: receiver-side counters (meter read fails, decode fails, LoRa TX fails).
    • "; + html += "
  • faults last: last receiver-side error code (same mapping as err_last).
    • "; html += ""; html += html_footer(); server.send(200, "text/html", html); From f3af5b3f1c51b06d731117b093e15c2afe3e2568 Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 00:22:35 +0100 Subject: [PATCH 07/29] Add SD logging and update docs - Add optional microSD CSV logging per sender/day on receiver - Wire logger into receiver packet handling - Document new batch header fields, build envs, and SD logging - Make sender links open in a new tab --- README.md | 19 +++++++- include/config.h | 6 ++- src/main.cpp | 4 ++ src/sd_logger.cpp | 114 +++++++++++++++++++++++++++++++++++++++++++++ src/sd_logger.h | 7 +++ src/web_server.cpp | 3 +- 6 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 src/sd_logger.cpp create mode 100644 src/sd_logger.h diff --git a/README.md b/README.md index 7cd0664..3565a94 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ Packet layout: LoRa radio settings: - Frequency: **433 MHz** or **868 MHz** (set by build env via `LORA_FREQUENCY_HZ`) -- SF11, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 +- SF12, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 ## Data Format MeterData JSON (sender + MQTT): @@ -205,6 +205,10 @@ Fixed header (little-endian): - `dt_s` u8 (seconds, >0) - `n` u8 (sample count, <=30) - `battery_mV` u16 +- `err_m` u8 (meter read failures, sender-side counter) +- `err_d` u8 (decode failures, sender-side counter) +- `err_tx` u8 (LoRa TX failures, sender-side counter) +- `err_last` u8 (last error code: 0=None, 1=MeterRead, 2=Decode, 3=LoraTx) Body: - `E0` u32 (absolute energy in Wh) @@ -219,6 +223,7 @@ Body: Notes: - Receiver reconstructs timestamps from `t_last` and `dt_s`. - Total power is computed on receiver as `p1 + p2 + p3`. +- Sender error counters are carried in the batch header and applied to all samples. ## Device IDs - Derived from WiFi STA MAC. @@ -250,6 +255,7 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - `/`: status overview - `/wifi`: WiFi/MQTT/NTP config (AP and STA) - `/sender/`: per-sender details +- Sender IDs on `/` are clickable (open sender page in a new tab). ## MQTT - Topic: `smartmeter//state` @@ -272,6 +278,8 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - `lilygo-t3-v1-6-1-test`: test build with `ENABLE_TEST_MODE` - `lilygo-t3-v1-6-1-868`: production build for 868 MHz modules - `lilygo-t3-v1-6-1-868-test`: test build for 868 MHz modules +- `lilygo-t3-v1-6-1-payload-test`: build with `PAYLOAD_CODEC_TEST` +- `lilygo-t3-v1-6-1-868-payload-test`: 868 MHz build with `PAYLOAD_CODEC_TEST` ## Config Knobs Key timing settings in `include/config.h`: @@ -283,6 +291,7 @@ Key timing settings in `include/config.h`: - `BATCH_RETRY_POLICY` (keep or drop on retry exhaustion) - `SERIAL_DEBUG_MODE` / `SERIAL_DEBUG_DUMP_JSON` - `LORA_SEND_BYPASS` (debug only) + - `ENABLE_SD_LOGGING` / `PIN_SD_CS` ## Limits & Known Constraints - **Compression**: MeterData uses lightweight RLE (good for JSON but not optimal). @@ -292,6 +301,14 @@ Key timing settings in `include/config.h`: - **OLED**: no hardware reset line is used (matches working reference). - **Batch ACKs**: sender waits for ACK after a batch and retries up to `BATCH_MAX_RETRIES` with `BATCH_ACK_TIMEOUT_MS` between attempts. +## SD Logging (Receiver) +Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. + +- Path: `/dd3//YYYY-MM-DD.csv` +- Columns: + `ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last` +- `err_last` is written as text (`meter`, `decode`, `loratx`) only on the last sample of a batch that reports an error. + ## Files & Modules - `include/config.h`, `src/config.cpp`: pins, radio settings, sender IDs - `include/data_model.h`, `src/data_model.cpp`: MeterData + ID init diff --git a/include/config.h b/include/config.h index 250e727..a3bab78 100644 --- a/include/config.h +++ b/include/config.h @@ -73,9 +73,11 @@ constexpr uint8_t BATCH_QUEUE_DEPTH = 10; constexpr BatchRetryPolicy BATCH_RETRY_POLICY = BatchRetryPolicy::Keep; constexpr uint32_t WATCHDOG_TIMEOUT_SEC = 120; constexpr bool ENABLE_HA_DISCOVERY = true; -constexpr bool SERIAL_DEBUG_MODE = true; -constexpr bool SERIAL_DEBUG_DUMP_JSON = true; +constexpr bool SERIAL_DEBUG_MODE = false; +constexpr bool SERIAL_DEBUG_DUMP_JSON = false; constexpr bool LORA_SEND_BYPASS = false; +constexpr bool ENABLE_SD_LOGGING = false; +constexpr uint8_t PIN_SD_CS = 25; constexpr uint8_t NUM_SENDERS = 1; inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { diff --git a/src/main.cpp b/src/main.cpp index dcbaf46..9dc364f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,6 +13,7 @@ #include "web_server.h" #include "display_ui.h" #include "test_mode.h" +#include "sd_logger.h" #include #include #ifdef ARDUINO_ARCH_ESP32 @@ -625,6 +626,7 @@ void setup() { update_battery_cache(); } else { power_receiver_init(); + sd_logger_init(); wifi_manager_init(); init_sender_statuses(); display_set_sender_statuses(g_sender_statuses, NUM_SENDERS); @@ -806,6 +808,7 @@ static void receiver_loop() { data.link_valid = true; data.link_rssi_dbm = pkt.rssi_dbm; data.link_snr_db = pkt.snr_db; + sd_logger_log_sample(data, data.last_error != FaultType::None); for (uint8_t i = 0; i < NUM_SENDERS; ++i) { if (pkt.device_id_short == EXPECTED_SENDER_IDS[i]) { data.short_id = pkt.device_id_short; @@ -897,6 +900,7 @@ static void receiver_loop() { data.err_decode = batch.err_d; data.err_lora_tx = batch.err_tx; data.last_error = static_cast(batch.err_last); + sd_logger_log_sample(data, (s + 1 == count) && data.last_error != FaultType::None); } if (sender_idx >= 0) { diff --git a/src/sd_logger.cpp b/src/sd_logger.cpp new file mode 100644 index 0000000..ceabfcd --- /dev/null +++ b/src/sd_logger.cpp @@ -0,0 +1,114 @@ +#include "sd_logger.h" +#include "config.h" +#include +#include +#include + +static bool g_sd_ready = false; + +static const char *fault_text(FaultType fault) { + switch (fault) { + case FaultType::MeterRead: + return "meter"; + case FaultType::Decode: + return "decode"; + case FaultType::LoraTx: + return "loratx"; + default: + return ""; + } +} + +static bool ensure_dir(const String &path) { + if (SD.exists(path)) { + return true; + } + return SD.mkdir(path); +} + +static String format_date_utc(uint32_t ts_utc) { + time_t t = static_cast(ts_utc); + struct tm tm_utc; + gmtime_r(&t, &tm_utc); + char buf[16]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02d", + tm_utc.tm_year + 1900, + tm_utc.tm_mon + 1, + tm_utc.tm_mday); + return String(buf); +} + +void sd_logger_init() { + if (!ENABLE_SD_LOGGING) { + g_sd_ready = false; + return; + } + g_sd_ready = SD.begin(PIN_SD_CS); +} + +bool sd_logger_is_ready() { + return g_sd_ready; +} + +void sd_logger_log_sample(const MeterData &data, bool include_error_text) { + if (!g_sd_ready || data.ts_utc == 0) { + return; + } + + String root_dir = "/dd3"; + if (!ensure_dir(root_dir)) { + return; + } + + String sender_dir = root_dir + "/" + String(data.device_id); + if (!ensure_dir(sender_dir)) { + return; + } + + String filename = sender_dir + "/" + format_date_utc(data.ts_utc) + ".csv"; + bool new_file = !SD.exists(filename); + File f = SD.open(filename, FILE_APPEND); + if (!f) { + return; + } + + if (new_file) { + f.println("ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last"); + } + + f.print(data.ts_utc); + f.print(','); + f.print(data.total_power_w, 1); + f.print(','); + f.print(data.phase_power_w[0], 1); + f.print(','); + f.print(data.phase_power_w[1], 1); + f.print(','); + f.print(data.phase_power_w[2], 1); + f.print(','); + f.print(data.energy_total_kwh, 3); + f.print(','); + f.print(data.battery_voltage_v, 2); + f.print(','); + f.print(data.battery_percent); + f.print(','); + f.print(data.link_rssi_dbm); + f.print(','); + if (isnan(data.link_snr_db)) { + f.print(""); + } else { + f.print(data.link_snr_db, 1); + } + f.print(','); + f.print(data.err_meter_read); + f.print(','); + f.print(data.err_decode); + f.print(','); + f.print(data.err_lora_tx); + f.print(','); + if (include_error_text && data.last_error != FaultType::None) { + f.print(fault_text(data.last_error)); + } + f.println(); + f.close(); +} diff --git a/src/sd_logger.h b/src/sd_logger.h new file mode 100644 index 0000000..60b1611 --- /dev/null +++ b/src/sd_logger.h @@ -0,0 +1,7 @@ +#pragma once + +#include "data_model.h" + +void sd_logger_init(); +bool sd_logger_is_ready(); +void sd_logger_log_sample(const MeterData &data, bool include_error_text); diff --git a/src/web_server.cpp b/src/web_server.cpp index ac4456e..4682c5d 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -52,7 +52,8 @@ static String render_sender_block(const SenderStatus &status) { } } } - s += "" + String(status.last_data.device_id) + ""; + String device_id = status.last_data.device_id; + s += "" + device_id + ""; if (status.has_data && status.last_data.link_valid) { s += " RSSI:" + String(status.last_data.link_rssi_dbm) + " SNR:" + String(status.last_data.link_snr_db, 1); } From d32ae30014635865db7a5b2e5caaf08f8e930203 Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 00:23:52 +0100 Subject: [PATCH 08/29] Move AP credentials to config and clarify STA UI access --- README.md | 5 +++-- include/config.h | 2 ++ src/main.cpp | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3565a94..9f6f3bb 100644 --- a/README.md +++ b/README.md @@ -249,13 +249,14 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - Uses deep sleep between cycles (`SENDER_WAKE_INTERVAL_SEC`). ## Web UI -- AP SSID: `DD3-Bridge-` -- AP password: `changeme123` +- AP SSID: `DD3-Bridge-` (prefix configurable) +- AP password: `changeme123` (configurable) - Endpoints: - `/`: status overview - `/wifi`: WiFi/MQTT/NTP config (AP and STA) - `/sender/`: per-sender details - Sender IDs on `/` are clickable (open sender page in a new tab). +- In STA mode, the UI is also available via the board’s IP/hostname on your WiFi network. ## MQTT - Topic: `smartmeter//state` diff --git a/include/config.h b/include/config.h index a3bab78..7da81f1 100644 --- a/include/config.h +++ b/include/config.h @@ -78,6 +78,8 @@ constexpr bool SERIAL_DEBUG_DUMP_JSON = false; constexpr bool LORA_SEND_BYPASS = false; constexpr bool ENABLE_SD_LOGGING = false; constexpr uint8_t PIN_SD_CS = 25; +constexpr const char *AP_SSID_PREFIX = "DD3-Bridge-"; +constexpr const char *AP_PASSWORD = "changeme123"; constexpr uint8_t NUM_SENDERS = 1; inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { diff --git a/src/main.cpp b/src/main.cpp index 9dc364f..488fa37 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -642,8 +642,8 @@ void setup() { } else { g_ap_mode = true; char ap_ssid[32]; - snprintf(ap_ssid, sizeof(ap_ssid), "DD3-Bridge-%04X", g_short_id); - wifi_start_ap(ap_ssid, "changeme123"); + snprintf(ap_ssid, sizeof(ap_ssid), "%s%04X", AP_SSID_PREFIX, g_short_id); + wifi_start_ap(ap_ssid, AP_PASSWORD); if (g_cfg.ntp_server_1.isEmpty()) { g_cfg.ntp_server_1 = "pool.ntp.org"; } From b5477262ea1d1ae3b4368b55f86c55abf6a96777 Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 01:43:54 +0100 Subject: [PATCH 09/29] Add SD history UI and pin remap - Add SD history chart + download listing to web UI - Use HSPI for SD and fix SD pin mapping - Swap role/OLED control pins and update role detection - Update README pin mapping and SD/history docs --- README.md | 23 ++- include/config.h | 17 +- src/config.cpp | 2 +- src/display_ui.cpp | 9 +- src/main.cpp | 2 +- src/sd_logger.cpp | 16 +- src/web_server.cpp | 430 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 484 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 9f6f3bb..66f099d 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,23 @@ Variants: - SCL: GPIO22 - RST: **not used** (SSD1306 init uses `-1` reset pin) - I2C address: 0x3C +- microSD (on-board) + - CS: GPIO13 + - MOSI: GPIO15 + - SCK: GPIO14 + - MISO: GPIO2 - I2C RTC (DS3231) - SDA: GPIO21 - SCL: GPIO22 - I2C address: 0x68 - Battery ADC: GPIO35 (via on-board divider) -- **Role select**: GPIO13 (INPUT_PULLDOWN) - - LOW = Sender - - HIGH = Receiver -- **OLED control**: GPIO14 (INPUT_PULLDOWN) +- **Role select**: GPIO14 (INPUT_PULLDOWN, sampled at boot) + - HIGH = Sender + - LOW/floating = Receiver +- **OLED control**: GPIO13 (INPUT_PULLDOWN, sender only) - HIGH = force OLED on - LOW = allow auto-off after timeout + - Not used on receiver (OLED always on) - Smart meter UART RX: GPIO34 (input-only, always connected) ### Notes on GPIOs @@ -257,6 +263,8 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - `/sender/`: per-sender details - Sender IDs on `/` are clickable (open sender page in a new tab). - In STA mode, the UI is also available via the board’s IP/hostname on your WiFi network. +- Main page shows SD card file listing (downloadable). +- Sender page includes a history chart (power) with configurable range/resolution/mode. ## MQTT - Topic: `smartmeter//state` @@ -293,6 +301,8 @@ Key timing settings in `include/config.h`: - `SERIAL_DEBUG_MODE` / `SERIAL_DEBUG_DUMP_JSON` - `LORA_SEND_BYPASS` (debug only) - `ENABLE_SD_LOGGING` / `PIN_SD_CS` + - `SD_HISTORY_MAX_DAYS` / `SD_HISTORY_MIN_RES_MIN` + - `SD_HISTORY_MAX_BINS` / `SD_HISTORY_TIME_BUDGET_MS` ## Limits & Known Constraints - **Compression**: MeterData uses lightweight RLE (good for JSON but not optimal). @@ -309,6 +319,9 @@ Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. - Columns: `ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last` - `err_last` is written as text (`meter`, `decode`, `loratx`) only on the last sample of a batch that reports an error. +- Files are downloadable from the main UI page. +- History chart on sender page stream-parses CSVs and bins data in the background. +- SD uses the on-board microSD SPI pins (CS=13, MOSI=15, SCK=14, MISO=2). ## Files & Modules - `include/config.h`, `src/config.cpp`: pins, radio settings, sender IDs @@ -331,7 +344,7 @@ Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. 1. Set role jumper on GPIO13: - LOW: sender - HIGH: receiver -2. OLED control on GPIO14: +2. OLED control on GPIO13: - HIGH: always on - LOW: auto-off after 10 minutes 3. Build and upload: diff --git a/include/config.h b/include/config.h index 7da81f1..5bcc3c7 100644 --- a/include/config.h +++ b/include/config.h @@ -39,8 +39,8 @@ constexpr uint8_t OLED_HEIGHT = 64; constexpr uint8_t PIN_BAT_ADC = 35; -constexpr uint8_t PIN_ROLE = 13; -constexpr uint8_t PIN_OLED_CTRL = 14; +constexpr uint8_t PIN_ROLE = 14; +constexpr uint8_t PIN_OLED_CTRL = 13; constexpr uint8_t PIN_METER_RX = 34; @@ -73,11 +73,18 @@ constexpr uint8_t BATCH_QUEUE_DEPTH = 10; constexpr BatchRetryPolicy BATCH_RETRY_POLICY = BatchRetryPolicy::Keep; constexpr uint32_t WATCHDOG_TIMEOUT_SEC = 120; constexpr bool ENABLE_HA_DISCOVERY = true; -constexpr bool SERIAL_DEBUG_MODE = false; +constexpr bool SERIAL_DEBUG_MODE = true; constexpr bool SERIAL_DEBUG_DUMP_JSON = false; constexpr bool LORA_SEND_BYPASS = false; -constexpr bool ENABLE_SD_LOGGING = false; -constexpr uint8_t PIN_SD_CS = 25; +constexpr bool ENABLE_SD_LOGGING = true; +constexpr uint8_t PIN_SD_CS = 13; +constexpr uint8_t PIN_SD_MOSI = 15; +constexpr uint8_t PIN_SD_MISO = 2; +constexpr uint8_t PIN_SD_SCK = 14; +constexpr uint16_t SD_HISTORY_MAX_DAYS = 30; +constexpr uint16_t SD_HISTORY_MIN_RES_MIN = 1; +constexpr uint16_t SD_HISTORY_MAX_BINS = 4000; +constexpr uint16_t SD_HISTORY_TIME_BUDGET_MS = 10; constexpr const char *AP_SSID_PREFIX = "DD3-Bridge-"; constexpr const char *AP_PASSWORD = "changeme123"; diff --git a/src/config.cpp b/src/config.cpp index cf2fe9d..0632bce 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -2,5 +2,5 @@ DeviceRole detect_role() { pinMode(PIN_ROLE, INPUT_PULLDOWN); - return digitalRead(PIN_ROLE) == HIGH ? DeviceRole::Receiver : DeviceRole::Sender; + return digitalRead(PIN_ROLE) == HIGH ? DeviceRole::Sender : DeviceRole::Receiver; } diff --git a/src/display_ui.cpp b/src/display_ui.cpp index 1327004..e22f9ba 100644 --- a/src/display_ui.cpp +++ b/src/display_ui.cpp @@ -69,7 +69,9 @@ void display_power_down() { } void display_init() { - pinMode(PIN_OLED_CTRL, INPUT_PULLDOWN); + if (g_role == DeviceRole::Sender) { + pinMode(PIN_OLED_CTRL, INPUT_PULLDOWN); + } Wire.begin(PIN_OLED_SDA, PIN_OLED_SCL); Wire.setClock(100000); g_display_ready = display.begin(SSD1306_SWITCHCAPVCC, OLED_I2C_ADDR); @@ -373,7 +375,10 @@ void display_tick() { } return; } - bool ctrl_high = digitalRead(PIN_OLED_CTRL) == HIGH; + bool ctrl_high = false; + if (g_role == DeviceRole::Sender) { + ctrl_high = digitalRead(PIN_OLED_CTRL) == HIGH; + } bool in_boot_window = (millis() - g_boot_ms) < OLED_AUTO_OFF_MS; if (g_role == DeviceRole::Receiver) { diff --git a/src/main.cpp b/src/main.cpp index 488fa37..32e3fc8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -603,6 +603,7 @@ void setup() { g_boot_ms = millis(); g_role = detect_role(); init_device_ids(g_short_id, g_device_id, sizeof(g_device_id)); + display_set_role(g_role); if (SERIAL_DEBUG_MODE) { #ifdef ARDUINO_ARCH_ESP32 serial_debug_printf("boot: reset_reason=%d", static_cast(esp_reset_reason())); @@ -615,7 +616,6 @@ void setup() { display_init(); time_rtc_init(); time_try_load_from_rtc(); - display_set_role(g_role); display_set_self_ids(g_short_id, g_device_id); if (g_role == DeviceRole::Sender) { diff --git a/src/sd_logger.cpp b/src/sd_logger.cpp index ceabfcd..085e2b4 100644 --- a/src/sd_logger.cpp +++ b/src/sd_logger.cpp @@ -5,6 +5,7 @@ #include static bool g_sd_ready = false; +static SPIClass *g_sd_spi = nullptr; static const char *fault_text(FaultType fault) { switch (fault) { @@ -43,7 +44,20 @@ void sd_logger_init() { g_sd_ready = false; return; } - g_sd_ready = SD.begin(PIN_SD_CS); + if (!g_sd_spi) { + g_sd_spi = new SPIClass(HSPI); + } + g_sd_spi->begin(PIN_SD_SCK, PIN_SD_MISO, PIN_SD_MOSI, PIN_SD_CS); + g_sd_ready = SD.begin(PIN_SD_CS, *g_sd_spi); + if (SERIAL_DEBUG_MODE) { + if (g_sd_ready) { + uint8_t type = SD.cardType(); + uint64_t size = SD.cardSize(); + Serial.printf("sd: ok type=%u size=%llu\n", static_cast(type), static_cast(size)); + } else { + Serial.println("sd: init failed"); + } + } } bool sd_logger_is_ready() { diff --git a/src/web_server.cpp b/src/web_server.cpp index 4682c5d..7fdc81b 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -2,6 +2,13 @@ #include #include "wifi_manager.h" #include "config.h" +#include "sd_logger.h" +#include "time_manager.h" +#include +#include +#include +#include +#include static WebServer server(80); static const SenderStatus *g_statuses = nullptr; @@ -13,6 +20,37 @@ static const FaultType *g_sender_last_errors = nullptr; static MeterData g_last_batch[NUM_SENDERS][METER_BATCH_MAX_SAMPLES]; static uint8_t g_last_batch_count[NUM_SENDERS] = {}; +struct HistoryBin { + uint32_t ts; + float value; + uint32_t count; +}; + +enum class HistoryMode : uint8_t { + Avg = 0, + Max = 1 +}; + +struct HistoryJob { + bool active; + bool done; + bool error; + String error_msg; + String device_id; + HistoryMode mode; + uint32_t start_ts; + uint32_t end_ts; + uint32_t res_sec; + uint32_t bins_count; + uint32_t bins_filled; + uint16_t day_index; + File file; + HistoryBin *bins; +}; + +static HistoryJob g_history = {}; +static constexpr size_t SD_LIST_MAX_FILES = 200; + static String html_header(const String &title) { String h = ""; h += "" + title + ""; @@ -40,6 +78,143 @@ static String format_faults(uint8_t idx) { return s; } +static void history_reset() { + if (g_history.file) { + g_history.file.close(); + } + if (g_history.bins) { + delete[] g_history.bins; + } + g_history = {}; +} + +static String history_date_from_epoch(uint32_t ts_utc) { + time_t t = static_cast(ts_utc); + struct tm tm_utc; + gmtime_r(&t, &tm_utc); + char buf[16]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02d", tm_utc.tm_year + 1900, tm_utc.tm_mon + 1, tm_utc.tm_mday); + return String(buf); +} + +static bool history_open_next_file() { + if (!g_history.active || g_history.done || g_history.error) { + return false; + } + if (g_history.file) { + g_history.file.close(); + } + uint32_t day_ts = g_history.start_ts + static_cast(g_history.day_index) * 86400UL; + if (day_ts > g_history.end_ts) { + g_history.done = true; + return false; + } + String path = String("/dd3/") + g_history.device_id + "/" + history_date_from_epoch(day_ts) + ".csv"; + g_history.file = SD.open(path.c_str(), FILE_READ); + g_history.day_index++; + return true; +} + +static bool history_parse_line(const char *line, uint32_t &ts_out, float &p_out) { + if (!line || line[0] < '0' || line[0] > '9') { + return false; + } + const char *comma = strchr(line, ','); + if (!comma) { + return false; + } + char ts_buf[16]; + size_t ts_len = static_cast(comma - line); + if (ts_len >= sizeof(ts_buf)) { + return false; + } + memcpy(ts_buf, line, ts_len); + ts_buf[ts_len] = '\0'; + char *end = nullptr; + uint32_t ts = static_cast(strtoul(ts_buf, &end, 10)); + if (end == ts_buf) { + return false; + } + const char *p_start = comma + 1; + const char *p_end = strchr(p_start, ','); + char p_buf[16]; + size_t p_len = p_end ? static_cast(p_end - p_start) : strlen(p_start); + if (p_len >= sizeof(p_buf)) { + return false; + } + memcpy(p_buf, p_start, p_len); + p_buf[p_len] = '\0'; + char *endp = nullptr; + float p = strtof(p_buf, &endp); + if (endp == p_buf) { + return false; + } + ts_out = ts; + p_out = p; + return true; +} + +static void history_tick() { + if (!g_history.active || g_history.done || g_history.error) { + return; + } + if (!sd_logger_is_ready()) { + g_history.error = true; + g_history.error_msg = "sd_not_ready"; + return; + } + + uint32_t start_ms = millis(); + while (millis() - start_ms < SD_HISTORY_TIME_BUDGET_MS) { + if (!g_history.file) { + if (!history_open_next_file()) { + if (g_history.done) { + g_history.active = false; + } + return; + } + } + if (!g_history.file.available()) { + g_history.file.close(); + continue; + } + + char line[160]; + size_t n = g_history.file.readBytesUntil('\n', line, sizeof(line) - 1); + line[n] = '\0'; + if (n == 0) { + continue; + } + uint32_t ts = 0; + float p = 0.0f; + if (!history_parse_line(line, ts, p)) { + continue; + } + if (ts < g_history.start_ts || ts > g_history.end_ts) { + continue; + } + uint32_t idx = (ts - g_history.start_ts) / g_history.res_sec; + if (idx >= g_history.bins_count) { + continue; + } + HistoryBin &bin = g_history.bins[idx]; + if (bin.count == 0) { + bin.ts = g_history.start_ts + idx * g_history.res_sec; + bin.value = p; + bin.count = 1; + g_history.bins_filled++; + } else if (g_history.mode == HistoryMode::Avg) { + bin.value += p; + bin.count++; + } else { + if (p > bin.value) { + bin.value = p; + } + bin.count++; + } + } +} + static String render_sender_block(const SenderStatus &status) { String s; s += "
    "; @@ -76,6 +251,42 @@ static String render_sender_block(const SenderStatus &status) { return s; } +static void append_sd_listing(String &html, const String &dir_path, uint8_t depth, size_t &count) { + if (count >= SD_LIST_MAX_FILES || depth > 4) { + return; + } + File dir = SD.open(dir_path.c_str()); + if (!dir || !dir.isDirectory()) { + return; + } + File entry = dir.openNextFile(); + while (entry && count < SD_LIST_MAX_FILES) { + String name = entry.name(); + String full_path = name; + if (!full_path.startsWith(dir_path)) { + if (!dir_path.endsWith("/")) { + full_path = dir_path + "/" + name; + } else { + full_path = dir_path + name; + } + } + if (entry.isDirectory()) { + html += "
  • " + full_path + "/
    • "; + append_sd_listing(html, full_path, depth + 1, count); + } else { + String href = full_path; + if (!href.startsWith("/")) { + href = "/" + href; + } + html += "
  • " + full_path + ""; + html += " (" + String(entry.size()) + " bytes)
    • "; + count++; + } + entry = dir.openNextFile(); + } + dir.close(); +} + static void handle_root() { String html = html_header("DD3 Bridge Status"); html += g_is_ap ? "

    Mode: AP

    " : "

    Mode: STA

    "; @@ -86,6 +297,18 @@ static void handle_root() { } } + if (sd_logger_is_ready()) { + html += "

    SD Files

      "; + size_t count = 0; + append_sd_listing(html, "/dd3", 0, count); + if (count >= SD_LIST_MAX_FILES) { + html += "
    • Listing truncated...
      • "; + } + html += "
    "; + } else { + html += "

    SD: not ready

    "; + } + html += "

    Configure WiFi/MQTT/NTP

    "; html += "

    Manual

    "; html += html_footer(); @@ -143,6 +366,63 @@ static void handle_sender() { if (device_id.equalsIgnoreCase(g_statuses[i].last_data.device_id)) { String html = html_header("Sender " + device_id); html += render_sender_block(g_statuses[i]); + html += "

    History (Power)

    "; + html += "
    "; + html += "Days: "; + html += "Res(min): "; + html += " "; + html += ""; + html += "
    "; + html += ""; + html += "
    "; + html += ""; if (g_last_batch_count[i] > 0) { html += "

    Last batch (" + String(g_last_batch_count[i]) + " samples)

    "; html += ""; @@ -193,6 +473,149 @@ static void handle_manual() { server.send(200, "text/html", html); } +static void handle_history_start() { + if (!sd_logger_is_ready()) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"sd_not_ready\"}"); + return; + } + if (!time_is_synced()) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"time_not_synced\"}"); + return; + } + String device_id = server.arg("device_id"); + uint16_t days = static_cast(server.arg("days").toInt()); + uint16_t res_min = static_cast(server.arg("res").toInt()); + String mode_str = server.arg("mode"); + if (device_id.length() == 0 || days == 0 || res_min == 0) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"bad_params\"}"); + return; + } + if (days > SD_HISTORY_MAX_DAYS) { + days = SD_HISTORY_MAX_DAYS; + } + if (res_min < SD_HISTORY_MIN_RES_MIN) { + res_min = SD_HISTORY_MIN_RES_MIN; + } + uint32_t bins = (static_cast(days) * 24UL * 60UL) / res_min; + if (bins == 0 || bins > SD_HISTORY_MAX_BINS) { + String resp = String("{\"ok\":false,\"error\":\"too_many_bins\",\"max_bins\":") + SD_HISTORY_MAX_BINS + "}"; + server.send(200, "application/json", resp); + return; + } + + history_reset(); + g_history.active = true; + g_history.done = false; + g_history.error = false; + g_history.device_id = device_id; + g_history.mode = (mode_str == "max") ? HistoryMode::Max : HistoryMode::Avg; + g_history.res_sec = static_cast(res_min) * 60UL; + g_history.bins_count = bins; + g_history.day_index = 0; + g_history.bins = new (std::nothrow) HistoryBin[bins]; + if (!g_history.bins) { + g_history.error = true; + g_history.error_msg = "oom"; + server.send(200, "application/json", "{\"ok\":false,\"error\":\"oom\"}"); + return; + } + for (uint32_t i = 0; i < bins; ++i) { + g_history.bins[i] = {}; + } + g_history.end_ts = time_get_utc(); + uint32_t span = static_cast(days) * 86400UL; + g_history.start_ts = g_history.end_ts > span ? (g_history.end_ts - span) : 0; + if (g_history.res_sec > 0) { + g_history.start_ts = (g_history.start_ts / g_history.res_sec) * g_history.res_sec; + } + + String resp = String("{\"ok\":true,\"bins\":") + bins + "}"; + server.send(200, "application/json", resp); +} + +static void handle_history_data() { + String device_id = server.arg("device_id"); + if (!g_history.bins || device_id.length() == 0 || device_id != g_history.device_id) { + server.send(200, "application/json", "{\"ready\":false,\"error\":\"no_job\"}"); + return; + } + if (g_history.error) { + String resp = String("{\"ready\":false,\"error\":\"") + g_history.error_msg + "\"}"; + server.send(200, "application/json", resp); + return; + } + if (g_history.active && !g_history.done) { + uint32_t progress = g_history.bins_count == 0 ? 0 : (g_history.bins_filled * 100UL / g_history.bins_count); + String resp = String("{\"ready\":false,\"progress\":") + progress + "}"; + server.send(200, "application/json", resp); + return; + } + + server.setContentLength(CONTENT_LENGTH_UNKNOWN); + server.send(200, "application/json", ""); + server.sendContent("{\"ready\":true,\"series\":["); + bool first = true; + for (uint32_t i = 0; i < g_history.bins_count; ++i) { + const HistoryBin &bin = g_history.bins[i]; + if (!first) { + server.sendContent(","); + } + first = false; + float value = NAN; + if (bin.count > 0) { + value = (g_history.mode == HistoryMode::Avg) ? (bin.value / static_cast(bin.count)) : bin.value; + } + if (bin.count == 0) { + server.sendContent(String("[") + bin.ts + ",null]"); + } else { + server.sendContent(String("[") + bin.ts + "," + String(value, 2) + "]"); + } + } + server.sendContent("]}"); +} + +static void handle_sd_download() { + if (!sd_logger_is_ready()) { + server.send(404, "text/plain", "SD not ready"); + return; + } + String path = server.arg("path"); + if (path.startsWith("dd3/")) { + path = "/" + path; + } + if (!path.startsWith("/dd3/")) { + server.send(400, "text/plain", "Invalid path"); + return; + } + File f = SD.open(path.c_str(), FILE_READ); + if (!f) { + server.send(404, "text/plain", "Not found"); + return; + } + size_t size = f.size(); + String filename = path.substring(path.lastIndexOf('/') + 1); + server.sendHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + server.setContentLength(size); + const char *content_type = "application/octet-stream"; + if (filename.endsWith(".csv")) { + content_type = "text/csv"; + } else if (filename.endsWith(".txt")) { + content_type = "text/plain"; + } + server.send(200, content_type, ""); + WiFiClient client = server.client(); + uint8_t buf[512]; + while (f.available()) { + size_t n = f.read(buf, sizeof(buf)); + if (n == 0) { + break; + } + client.write(buf, n); + delay(0); + } + f.close(); +} + void web_server_set_config(const WifiMqttConfig &config) { g_config = config; } @@ -222,6 +645,9 @@ void web_server_begin_ap(const SenderStatus *statuses, uint8_t count) { server.on("/", handle_root); server.on("/manual", handle_manual); + server.on("/history/start", handle_history_start); + server.on("/history/data", handle_history_data); + server.on("/sd/download", handle_sd_download); server.on("/wifi", HTTP_GET, handle_wifi_get); server.on("/wifi", HTTP_POST, handle_wifi_post); server.on("/sender/", handle_sender); @@ -243,6 +669,9 @@ void web_server_begin_sta(const SenderStatus *statuses, uint8_t count) { server.on("/", handle_root); server.on("/manual", handle_manual); server.on("/sender/", handle_sender); + server.on("/history/start", handle_history_start); + server.on("/history/data", handle_history_data); + server.on("/sd/download", handle_sd_download); server.on("/wifi", HTTP_GET, handle_wifi_get); server.on("/wifi", HTTP_POST, handle_wifi_post); server.onNotFound([]() { @@ -256,5 +685,6 @@ void web_server_begin_sta(const SenderStatus *statuses, uint8_t count) { } void web_server_loop() { + history_tick(); server.handleClient(); } From 0e12b406de4c8bcc4f682d98f46957933872dd06 Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 2 Feb 2026 21:07:37 +0100 Subject: [PATCH 10/29] Harden web UI auth, input handling, and SD path validation - Add optional Basic Auth with NVS-backed credentials and STA/AP flags; protect status, wifi, history, and download routes - Stop pre-filling WiFi/MQTT/Web UI password fields; keep stored secrets on blank and add clear-password checkboxes - Add HTML escaping + URL encoding helpers and apply to user-controlled strings; add unit test - Harden /sd/download path validation (prefix, length, dotdot, slashes) and log rejections - Enforce protocol version in LoRa receive and release GPIO14 before SD init - Update README security, SD, and GPIO sharing notes --- README.md | 23 ++- include/config.h | 4 + include/html_util.h | 6 + include/wifi_manager.h | 2 + src/html_util.cpp | 49 ++++++ src/lora_transport.cpp | 3 + src/main.cpp | 1 + src/web_server.cpp | 170 ++++++++++++++++++--- src/wifi_manager.cpp | 11 +- test/test_html_escape/test_html_escape.cpp | 21 +++ 10 files changed, 260 insertions(+), 30 deletions(-) create mode 100644 include/html_util.h create mode 100644 src/html_util.cpp create mode 100644 test/test_html_escape/test_html_escape.cpp diff --git a/README.md b/README.md index 66f099d..409ccff 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ Variants: - SCL: GPIO22 - I2C address: 0x68 - Battery ADC: GPIO35 (via on-board divider) -- **Role select**: GPIO14 (INPUT_PULLDOWN, sampled at boot) +- **Role select**: GPIO14 (INPUT_PULLDOWN, sampled at boot, **shared with SD SCK**) - HIGH = Sender - LOW/floating = Receiver -- **OLED control**: GPIO13 (INPUT_PULLDOWN, sender only) +- **OLED control**: GPIO13 (INPUT_PULLDOWN, sender only, **shared with SD CS**) - HIGH = force OLED on - LOW = allow auto-off after timeout - Not used on receiver (OLED always on) @@ -43,6 +43,9 @@ Variants: ### Notes on GPIOs - GPIO34/35/36/39 are input-only and have **no internal pullups/pulldowns**. - Strap pins (GPIO0/2/4/5/12/15) can affect boot; avoid for role or control jumpers. +- GPIO14 is shared between role select and SD SCK. **Do not attach the role jumper in Receiver mode if the SD card is connected/used**, and never force GPIO14 high when using SD. +- GPIO13 is shared between OLED control and SD CS. Avoid driving OLED control when SD is active. +- Receiver firmware releases GPIO14 to `INPUT` (no pulldown) after boot before SD SPI init. ## Firmware Roles ### Sender (battery-powered) @@ -262,10 +265,20 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - `/wifi`: WiFi/MQTT/NTP config (AP and STA) - `/sender/`: per-sender details - Sender IDs on `/` are clickable (open sender page in a new tab). -- In STA mode, the UI is also available via the board’s IP/hostname on your WiFi network. +- In STA mode, the UI is also available via the board's IP/hostname on your WiFi network. - Main page shows SD card file listing (downloadable). - Sender page includes a history chart (power) with configurable range/resolution/mode. +## Security +- Basic Auth is supported for the web UI. In STA mode it is enabled by default; AP mode is optional. +- Config flags in `include/config.h`: + - `WEB_AUTH_REQUIRE_STA` (default `true`) + - `WEB_AUTH_REQUIRE_AP` (default `false`) + - `WEB_AUTH_DEFAULT_USER` / `WEB_AUTH_DEFAULT_PASS` +- Web credentials are stored in NVS. `/wifi`, `/sd/download`, `/history/*`, `/`, `/sender/*`, and `/manual` require auth when enabled. +- Password inputs are not prefilled. Leaving a password blank keeps the stored value; use the "clear password" checkbox to erase it. +- User-controlled strings are HTML-escaped before embedding in pages. + ## MQTT - Topic: `smartmeter//state` - QoS 0 @@ -303,6 +316,7 @@ Key timing settings in `include/config.h`: - `ENABLE_SD_LOGGING` / `PIN_SD_CS` - `SD_HISTORY_MAX_DAYS` / `SD_HISTORY_MIN_RES_MIN` - `SD_HISTORY_MAX_BINS` / `SD_HISTORY_TIME_BUDGET_MS` + - `WEB_AUTH_REQUIRE_STA` / `WEB_AUTH_REQUIRE_AP` / `WEB_AUTH_DEFAULT_USER` / `WEB_AUTH_DEFAULT_PASS` ## Limits & Known Constraints - **Compression**: MeterData uses lightweight RLE (good for JSON but not optimal). @@ -320,6 +334,7 @@ Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. `ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last` - `err_last` is written as text (`meter`, `decode`, `loratx`) only on the last sample of a batch that reports an error. - Files are downloadable from the main UI page. +- Downloads only allow absolute paths under `/dd3/`, reject `..`, backslashes, and repeated slashes, and enforce a max path length. - History chart on sender page stream-parses CSVs and bins data in the background. - SD uses the on-board microSD SPI pins (CS=13, MOSI=15, SCK=14, MISO=2). @@ -341,7 +356,7 @@ Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. - `src/main.cpp`: role detection and main loop ## Quick Start -1. Set role jumper on GPIO13: +1. Set role jumper on GPIO14: - LOW: sender - HIGH: receiver 2. OLED control on GPIO13: diff --git a/include/config.h b/include/config.h index 5bcc3c7..6c50496 100644 --- a/include/config.h +++ b/include/config.h @@ -87,6 +87,10 @@ constexpr uint16_t SD_HISTORY_MAX_BINS = 4000; constexpr uint16_t SD_HISTORY_TIME_BUDGET_MS = 10; constexpr const char *AP_SSID_PREFIX = "DD3-Bridge-"; constexpr const char *AP_PASSWORD = "changeme123"; +constexpr bool WEB_AUTH_REQUIRE_STA = true; +constexpr bool WEB_AUTH_REQUIRE_AP = false; +constexpr const char *WEB_AUTH_DEFAULT_USER = "admin"; +constexpr const char *WEB_AUTH_DEFAULT_PASS = "admin"; constexpr uint8_t NUM_SENDERS = 1; inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { diff --git a/include/html_util.h b/include/html_util.h new file mode 100644 index 0000000..5ed1db4 --- /dev/null +++ b/include/html_util.h @@ -0,0 +1,6 @@ +#pragma once + +#include + +String html_escape(const String &input); +String url_encode_component(const String &input); diff --git a/include/wifi_manager.h b/include/wifi_manager.h index e9012ca..74f288c 100644 --- a/include/wifi_manager.h +++ b/include/wifi_manager.h @@ -12,6 +12,8 @@ struct WifiMqttConfig { String mqtt_pass; String ntp_server_1; String ntp_server_2; + String web_user; + String web_pass; bool valid; }; diff --git a/src/html_util.cpp b/src/html_util.cpp new file mode 100644 index 0000000..dff470f --- /dev/null +++ b/src/html_util.cpp @@ -0,0 +1,49 @@ +#include "html_util.h" + +String html_escape(const String &input) { + String out; + out.reserve(input.length() + 8); + for (size_t i = 0; i < input.length(); ++i) { + char c = input[i]; + switch (c) { + case '&': + out += "&"; + break; + case '<': + out += "<"; + break; + case '>': + out += ">"; + break; + case '"': + out += """; + break; + case '\'': + out += "'"; + break; + default: + out += c; + break; + } + } + return out; +} + +String url_encode_component(const String &input) { + String out; + out.reserve(input.length() * 3); + const char *hex = "0123456789ABCDEF"; + for (size_t i = 0; i < input.length(); ++i) { + unsigned char c = static_cast(input[i]); + bool safe = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~'; + if (safe) { + out += static_cast(c); + } else { + out += '%'; + out += hex[(c >> 4) & 0x0F]; + out += hex[c & 0x0F]; + } + } + return out; +} diff --git a/src/lora_transport.cpp b/src/lora_transport.cpp index 7ea0612..97032f6 100644 --- a/src/lora_transport.cpp +++ b/src/lora_transport.cpp @@ -85,6 +85,9 @@ bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms) { if (crc_calc != crc_rx) { return false; } + if (buffer[0] != PROTOCOL_VERSION) { + return false; + } pkt.protocol_version = buffer[0]; pkt.role = static_cast(buffer[1]); diff --git a/src/main.cpp b/src/main.cpp index 32e3fc8..e173807 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -626,6 +626,7 @@ void setup() { update_battery_cache(); } else { power_receiver_init(); + pinMode(PIN_ROLE, INPUT); // release pulldown before SD uses GPIO14 as SCK sd_logger_init(); wifi_manager_init(); init_sender_statuses(); diff --git a/src/web_server.cpp b/src/web_server.cpp index 7fdc81b..c5d2bc7 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -4,6 +4,7 @@ #include "config.h" #include "sd_logger.h" #include "time_manager.h" +#include "html_util.h" #include #include #include @@ -15,6 +16,8 @@ static const SenderStatus *g_statuses = nullptr; static uint8_t g_status_count = 0; static WifiMqttConfig g_config; static bool g_is_ap = false; +static String g_web_user; +static String g_web_pass; static const FaultCounters *g_sender_faults = nullptr; static const FaultType *g_sender_last_errors = nullptr; static MeterData g_last_batch[NUM_SENDERS][METER_BATCH_MAX_SAMPLES]; @@ -50,11 +53,30 @@ struct HistoryJob { static HistoryJob g_history = {}; static constexpr size_t SD_LIST_MAX_FILES = 200; +static constexpr size_t SD_DOWNLOAD_MAX_PATH = 160; + +static bool auth_required() { + return g_is_ap ? WEB_AUTH_REQUIRE_AP : WEB_AUTH_REQUIRE_STA; +} + +static bool ensure_auth() { + if (!auth_required()) { + return true; + } + const char *user = g_web_user.c_str(); + const char *pass = g_web_pass.c_str(); + if (server.authenticate(user, pass)) { + return true; + } + server.requestAuthentication(BASIC_AUTH, "DD3", "Authentication required"); + return false; +} static String html_header(const String &title) { + String safe_title = html_escape(title); String h = ""; - h += "" + title + ""; - h += "

    " + title + "

    "; + h += "" + safe_title + ""; + h += "

    " + safe_title + "

    "; return h; } @@ -78,6 +100,46 @@ static String format_faults(uint8_t idx) { return s; } +static bool sanitize_sd_download_path(String &path, String &error) { + path.trim(); + if (path.length() == 0) { + error = "empty"; + return false; + } + if (path.startsWith("dd3/")) { + path = "/" + path; + } + if (path.length() > SD_DOWNLOAD_MAX_PATH) { + error = "too_long"; + return false; + } + if (!path.startsWith("/dd3/")) { + error = "prefix"; + return false; + } + if (path.indexOf("..") >= 0) { + error = "dotdot"; + return false; + } + if (path.indexOf('\\') >= 0) { + error = "backslash"; + return false; + } + if (path.indexOf("//") >= 0) { + error = "repeated_slash"; + return false; + } + return true; +} + +static bool checkbox_checked(const char *name) { + if (!server.hasArg(name)) { + return false; + } + String val = server.arg(name); + return val == "on" || val == "true" || val == "1"; +} + static void history_reset() { if (g_history.file) { g_history.file.close(); @@ -228,7 +290,9 @@ static String render_sender_block(const SenderStatus &status) { } } String device_id = status.last_data.device_id; - s += "" + device_id + ""; + String device_id_safe = html_escape(device_id); + String device_id_url = url_encode_component(device_id); + s += "" + device_id_safe + ""; if (status.has_data && status.last_data.link_valid) { s += " RSSI:" + String(status.last_data.link_rssi_dbm) + " SNR:" + String(status.last_data.link_snr_db, 1); } @@ -271,14 +335,15 @@ static void append_sd_listing(String &html, const String &dir_path, uint8_t dept } } if (entry.isDirectory()) { - html += "
  • " + full_path + "/
    • "; + html += "
  • " + html_escape(full_path) + "/
    • "; append_sd_listing(html, full_path, depth + 1, count); } else { String href = full_path; if (!href.startsWith("/")) { href = "/" + href; } - html += "
  • " + full_path + ""; + String href_enc = url_encode_component(href); + html += "
  • " + html_escape(full_path) + ""; html += " (" + String(entry.size()) + " bytes)
    • "; count++; } @@ -288,6 +353,9 @@ static void append_sd_listing(String &html, const String &dir_path, uint8_t dept } static void handle_root() { + if (!ensure_auth()) { + return; + } String html = html_header("DD3 Bridge Status"); html += g_is_ap ? "

    Mode: AP

    " : "

    Mode: STA

    "; @@ -316,16 +384,26 @@ static void handle_root() { } static void handle_wifi_get() { + if (!ensure_auth()) { + return; + } String html = html_header("WiFi/MQTT Config"); html += "
    "; - html += "SSID:
    "; - html += "Password:
    "; - html += "MQTT Host:
    "; + html += "SSID:
    "; + html += "Password: "; + html += "
    "; + html += "MQTT Host:
    "; html += "MQTT Port:
    "; - html += "MQTT User:
    "; - html += "MQTT Pass:
    "; - html += "NTP Server 1:
    "; - html += "NTP Server 2:
    "; + html += "MQTT User:
    "; + html += "MQTT Pass: "; + html += "
    "; + html += "NTP Server 1:
    "; + html += "NTP Server 2:
    "; + html += "
    "; + html += "Web UI User:
    "; + html += "Web UI Pass: "; + html += "
    "; + html += "
    Leaving password blank keeps the existing one.
    "; html += ""; html += ""; html += html_footer(); @@ -333,15 +411,38 @@ static void handle_wifi_get() { } static void handle_wifi_post() { - WifiMqttConfig cfg; - cfg.ntp_server_1 = "pool.ntp.org"; - cfg.ntp_server_2 = "time.nist.gov"; + if (!ensure_auth()) { + return; + } + WifiMqttConfig cfg = g_config; + cfg.ntp_server_1 = g_config.ntp_server_1.length() > 0 ? g_config.ntp_server_1 : "pool.ntp.org"; + cfg.ntp_server_2 = g_config.ntp_server_2.length() > 0 ? g_config.ntp_server_2 : "time.nist.gov"; cfg.ssid = server.arg("ssid"); - cfg.password = server.arg("pass"); + String wifi_pass = server.arg("pass"); + if (checkbox_checked("clear_wifi_pass")) { + cfg.password = ""; + } else if (wifi_pass.length() > 0) { + cfg.password = wifi_pass; + } cfg.mqtt_host = server.arg("mqhost"); cfg.mqtt_port = static_cast(server.arg("mqport").toInt()); cfg.mqtt_user = server.arg("mquser"); - cfg.mqtt_pass = server.arg("mqpass"); + String mqtt_pass = server.arg("mqpass"); + if (checkbox_checked("clear_mqtt_pass")) { + cfg.mqtt_pass = ""; + } else if (mqtt_pass.length() > 0) { + cfg.mqtt_pass = mqtt_pass; + } + String web_user = server.arg("webuser"); + if (web_user.length() > 0) { + cfg.web_user = web_user; + } + String web_pass = server.arg("webpass"); + if (checkbox_checked("clear_web_pass")) { + cfg.web_pass = ""; + } else if (web_pass.length() > 0) { + cfg.web_pass = web_pass; + } if (server.arg("ntp1").length() > 0) { cfg.ntp_server_1 = server.arg("ntp1"); } @@ -349,6 +450,9 @@ static void handle_wifi_post() { cfg.ntp_server_2 = server.arg("ntp2"); } cfg.valid = true; + g_config = cfg; + g_web_user = cfg.web_user; + g_web_pass = cfg.web_pass; wifi_save_config(cfg); server.send(200, "text/html", "Saved. Rebooting..."); delay(1000); @@ -356,12 +460,16 @@ static void handle_wifi_post() { } static void handle_sender() { + if (!ensure_auth()) { + return; + } if (!g_statuses) { server.send(404, "text/plain", "No senders"); return; } String uri = server.uri(); String device_id = uri.substring(String("/sender/").length()); + String device_id_url = url_encode_component(device_id); for (uint8_t i = 0; i < g_status_count; ++i) { if (device_id.equalsIgnoreCase(g_statuses[i].last_data.device_id)) { String html = html_header("Sender " + device_id); @@ -376,6 +484,7 @@ static void handle_sender() { html += ""; html += ""; html += "