diff --git a/README.md b/README.md index e6f59ee..59060c7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DD3 LoRa Bridge (Multi-Sender) -Unified firmware for LilyGO T3 v1.6.1 (ESP32 + SX1276 + SSD1306) that runs as **Sender** or **Receiver** based on a GPIO jumper. Senders read DD3 smart meter values, compress JSON, and transmit over LoRa. The receiver validates packets, publishes to MQTT, provides a web UI, and shows per-sender status on the OLED. +Unified firmware for LilyGO T3 v1.6.1 (ESP32 + SX1276 + SSD1306) that runs as **Sender** or **Receiver** based on a GPIO jumper. Senders read DD3 smart meter values and transmit compact binary batches over LoRa. The receiver validates packets, publishes to MQTT, provides a web UI, and shows per-sender status on the OLED. ## Hardware Board: **LilyGO T3 LoRa32 v1.6.1** (ESP32 + SX1276 + SSD1306 128x64 + LiPo) @@ -21,22 +21,31 @@ Variants: - SCL: GPIO22 - RST: **not used** (SSD1306 init uses `-1` reset pin) - I2C address: 0x3C +- microSD (on-board) + - CS: GPIO13 + - MOSI: GPIO15 + - SCK: GPIO14 + - MISO: GPIO2 - I2C RTC (DS3231) - SDA: GPIO21 - SCL: GPIO22 - I2C address: 0x68 - Battery ADC: GPIO35 (via on-board divider) -- **Role select**: GPIO13 (INPUT_PULLDOWN) - - LOW = Sender - - HIGH = Receiver -- **OLED control**: GPIO14 (INPUT_PULLDOWN) +- **Role select**: GPIO14 (INPUT_PULLDOWN, sampled at boot, **shared with SD SCK**) + - HIGH = Sender + - LOW/floating = Receiver +- **OLED control**: GPIO13 (INPUT_PULLDOWN, sender only, **shared with SD CS**) - HIGH = force OLED on - LOW = allow auto-off after timeout + - Not used on receiver (OLED always on) - Smart meter UART RX: GPIO34 (input-only, always connected) ### Notes on GPIOs - GPIO34/35/36/39 are input-only and have **no internal pullups/pulldowns**. - Strap pins (GPIO0/2/4/5/12/15) can affect boot; avoid for role or control jumpers. +- GPIO14 is shared between role select and SD SCK. **Do not attach the role jumper in Receiver mode if the SD card is connected/used**, and never force GPIO14 high when using SD. +- GPIO13 is shared between OLED control and SD CS. Avoid driving OLED control when SD is active. +- Receiver firmware releases GPIO14 to `INPUT` (no pulldown) after boot before SD SPI init. ## Firmware Roles ### Sender (battery-powered) @@ -45,8 +54,9 @@ Variants: - Energy total: 1-0:1.8.0*255 - Total power: 1-0:16.7.0*255 - Phase power: 36.7 / 56.7 / 76.7 +- Meter input is parsed via a non-blocking RX state machine; the last valid frame is reused for 1 Hz sampling. - Reads battery voltage and estimates SoC. -- Builds JSON payload, compresses, wraps in LoRa packet, transmits. +- Builds compact binary batch payload, wraps in LoRa packet, transmits. - Light sleeps between meter reads; batches are sent every 30s. - Listens for LoRa time sync packets to set UTC clock. - Uses DS3231 RTC after boot if no time sync has arrived yet. @@ -59,9 +69,8 @@ void sender_loop() { read_battery(data); // VBAT + SoC if (time_to_send_batch()) { - json = meterBatchToJson(samples, batch_id); // bat_v per batch, t_first/t_last included - compressed = compressBuffer(json); - lora_send(packet(MeterBatch, compressed)); + payload = encode_batch(samples, batch_id); // compact binary batch + lora_send(packet(MeterBatch, payload)); } display_set_last_meter(data); @@ -79,14 +88,14 @@ void sender_loop() { bool meter_read(MeterData &data); // parse OBIS fields void read_battery(MeterData &data); // ADC -> volts + percent bool meterDataToJson(const MeterData&, String&); -bool compressBuffer(const uint8_t*, size_t, uint8_t*, size_t, size_t&); +bool compressBuffer(const uint8_t*, size_t, uint8_t*, size_t, size_t&); // MeterData only bool lora_send(const LoraPacket &pkt); // add header + CRC16 and transmit ``` ### Receiver (USB-powered) - WiFi STA connect using stored config; if not available/fails, starts AP. - NTP sync (UTC) and local display in Europe/Berlin. -- Receives LoRa packets, verifies CRC16, decompresses, parses JSON. +- Receives LoRa packets, verifies CRC16, decompresses MeterData JSON, decodes binary batches. - Publishes meter JSON to MQTT. - Sends ACKs for MeterBatch packets and de-duplicates by batch_id. - Web UI: @@ -105,8 +114,8 @@ void receiver_loop() { mqtt_publish_state(data); } } else if (pkt.type == MeterBatch) { - json = reassemble_and_decompress_batch(pkt); - for (sample in jsonToMeterBatch(json)) { // uses t_first/t_last for jittered timestamps + batch = reassemble_and_decode_batch(pkt); + for (sample in batch) { update_sender_status(sample); mqtt_publish_state(sample); } @@ -114,7 +123,7 @@ void receiver_loop() { } if (time_to_send_timesync()) { - time_send_timesync(self_short_id); // 60s for first 10 min, then hourly if RTC is present + time_send_timesync(self_short_id); // always every 60s (receiver is mains-powered) } mqtt_loop(); @@ -124,11 +133,13 @@ void receiver_loop() { } ``` +Receiver keeps the SX1276 in continuous RX, re-entering RX after any transmit (ACK or time sync). + **Key receiver functions**: ```cpp bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms); bool jsonToMeterData(const String &json, MeterData &data); -bool jsonToMeterBatch(const String &json, MeterData *samples, size_t max, size_t &count); +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out); bool mqtt_publish_state(const MeterData &data); void web_server_loop(); // AP or STA UI void time_send_timesync(uint16_t self_id); @@ -170,16 +181,17 @@ Packet layout: [1] role (0=sender, 1=receiver) [2..3] device_id_short (uint16) [4] payload_type (0=meter, 1=test, 2=time_sync, 3=meter_batch, 4=ack) -[5..N-3] compressed payload +[5..N-3] payload bytes (compressed JSON for MeterData, binary for MeterBatch/Test/TimeSync) [N-2..N-1] CRC16 (bytes 0..N-3) ``` LoRa radio settings: - Frequency: **433 MHz** or **868 MHz** (set by build env via `LORA_FREQUENCY_HZ`) -- SF10, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 +- SF12, BW 125 kHz, CR 4/5, CRC on, Sync Word 0x34 +- When `SERIAL_DEBUG_MODE` is enabled, LoRa TX logs include timing breakdowns for `idle/begin/write/end` to diagnose long transmit times. ## Data Format -JSON payload (sender + MQTT): +MeterData JSON (sender + MQTT): ```json { @@ -191,40 +203,48 @@ JSON payload (sender + MQTT): "p2_w": 450.00, "p3_w": 0.00, "bat_v": 3.92, - "bat_pct": 78 + "bat_pct": 78, + "rx_reject": 0, + "rx_reject_text": "none" } ``` -MeterBatch JSON (compressed over LoRa) uses per-field arrays with integer units for easier ingestion: +### Binary MeterBatch Payload (LoRa) +Fixed header (little-endian): +- `magic` u16 = 0xDDB3 +- `schema` u8 = 2 +- `flags` u8 = 0x01 (bit0 = signed phases) +- `sender_id` u16 (1..NUM_SENDERS, maps to `EXPECTED_SENDER_IDS`) +- `batch_id` u16 +- `t_last` u32 (unix seconds of last sample) +- `dt_s` u8 (seconds, >0) +- `n` u8 (sample count, <=30) +- `battery_mV` u16 +- `err_m` u8 (meter read failures, sender-side counter) +- `err_d` u8 (decode failures, sender-side counter) +- `err_tx` u8 (LoRa TX failures, sender-side counter) +- `err_last` u8 (last error code: 0=None, 1=MeterRead, 2=Decode, 3=LoraTx, 4=TimeSync) +- `err_rx_reject` u8 (last RX reject reason) +- `err_rx_reject` u8 (last RX reject reason: 0=None, 1=crc_fail, 2=bad_protocol_version, 3=wrong_role, 4=wrong_payload_type, 5=length_mismatch, 6=device_id_mismatch, 7=batch_id_mismatch) +- MQTT faults payload also includes `err_last_text` (string) and `err_last_age` (seconds). -```json -{ - "schema": 1, - "sender": "s01", - "batch_id": 1842, - "t0": 1738288000, - "t_first": 1738288000, - "t_last": 1738288030, - "dt_s": 1, - "n": 3, - "e_wh": [123456700, 123456701, 123456701], - "p_w": [930, 940, 950], - "p1_w": [480, 490, 500], - "p2_w": [450, 450, 450], - "p3_w": [0, 0, 0], - "bat_v": 3.92, - "meta": { - "rssi": -92, - "snr": 7.5, - "rx_ts": 1738288031 - } -} -``` +Body: +- `E0` u32 (absolute energy in Wh) +- `dE[1..n-1]` ULEB128 (delta vs previous, >=0) +- `P1_0` s16 (absolute W) +- `dP1[1..n-1]` signed varint (ZigZag + ULEB128) +- `P2_0` s16 +- `dP2[1..n-1]` signed varint +- `P3_0` s16 +- `dP3[1..n-1]` signed varint Notes: -- `sender` maps to `EXPECTED_SENDER_IDS` order (`s01` = first sender). -- `meta` is injected by the receiver after batch reassembly. -- `bat_v` is a single batch-level value (percent is calculated locally). +- Receiver reconstructs timestamps from `t_last` and `dt_s`. +- Total power is computed on receiver as `p1 + p2 + p3`. +- Sender error counters are carried in the batch header and applied to all samples. +- Receiver ACKs MeterBatch as soon as the batch is reassembled, before MQTT/web/UI work, to avoid missing the sender ACK window. +- Receiver repeats ACKs (`ACK_REPEAT_COUNT`) spaced by `ACK_REPEAT_DELAY_MS` to cover sender RX latency. +- Sender ACK RX window is derived from LoRa airtime (bounded min/max) and retried once if the first window misses. ## Device IDs - Derived from WiFi STA MAC. @@ -239,23 +259,43 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; ``` ## OLED Behavior -- Sender: OLED stays **ON for 10 seconds** on each wake, then powers down for sleep. +- Sender: OLED stays on for `OLED_AUTO_OFF_MS` after boot or last activity. +- Activity is detected while `PIN_OLED_CTRL` is held high, or on the high->low edge when the control is released. - Receiver: OLED is always on (no auto-off). - Pages rotate every 4s. ## Power & Battery -- Sender disables WiFi/BLE, reads VBAT via ADC, uses linear SoC map: - - 3.0 V = 0% +- Sender disables WiFi/BLE, reads VBAT via ADC, and converts voltage to % using a LiPo curve: - 4.2 V = 100% + - 2.9 V = 0% + - linear interpolation between curve points - Uses deep sleep between cycles (`SENDER_WAKE_INTERVAL_SEC`). +- Sender CPU is throttled to 80 MHz and LoRa RX is only enabled in short windows (ACK wait or time-sync). +- Battery sampling averages 5 ADC reads and updates at most once per `BATTERY_SAMPLE_INTERVAL_MS` (default 60s). +- `BATTERY_CAL` applies a scale factor to match measured VBAT. +- When `SERIAL_DEBUG_MODE` is enabled, each ADC read logs the 5 raw samples, average, and computed voltage. ## Web UI -- AP SSID: `DD3-Bridge-` -- AP password: `changeme123` +- AP SSID: `DD3-Bridge-` (prefix configurable) +- AP password: `changeme123` (configurable) - Endpoints: - `/`: status overview - `/wifi`: WiFi/MQTT/NTP config (AP and STA) - `/sender/`: per-sender details +- Sender IDs on `/` are clickable (open sender page in a new tab). +- In STA mode, the UI is also available via the board's IP/hostname on your WiFi network. +- Main page shows SD card file listing (downloadable). +- Sender page includes a history chart (power) with configurable range/resolution/mode. + +## Security +- Basic Auth is supported for the web UI. In STA mode it is enabled by default; AP mode is optional. +- Config flags in `include/config.h`: + - `WEB_AUTH_REQUIRE_STA` (default `true`) + - `WEB_AUTH_REQUIRE_AP` (default `false`) + - `WEB_AUTH_DEFAULT_USER` / `WEB_AUTH_DEFAULT_PASS` +- Web credentials are stored in NVS. `/wifi`, `/sd/download`, `/history/*`, `/`, `/sender/*`, and `/manual` require auth when enabled. +- Password inputs are not prefilled. Leaving a password blank keeps the stored value; use the "clear password" checkbox to erase it. +- User-controlled strings are HTML-escaped before embedding in pages. ## MQTT - Topic: `smartmeter//state` @@ -271,39 +311,71 @@ inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { 0xF19C }; - Enable/disable with `ENABLE_DS3231` in `include/config.h`. - Receiver time sync packets set the RTC. - On boot, if no LoRa time sync has arrived yet, the sender uses the RTC time as the initial `ts_utc`. -- When no RTC is present or enabled, the receiver keeps sending time sync every 60 seconds. +- Receiver keeps sending time sync every 60 seconds. +- If a sender’s timestamps drift from receiver time by more than `TIME_SYNC_DRIFT_THRESHOLD_SEC`, the receiver enters a burst mode (every `TIME_SYNC_BURST_INTERVAL_MS` for `TIME_SYNC_BURST_DURATION_MS`). +- Sender raises a local `TimeSync` error if it has not received a time beacon for `TIME_SYNC_ERROR_TIMEOUT_MS` (default 2 days). This is shown on the sender OLED only and is not sent over LoRa. +- RTC loads are validated (reject out-of-range epochs) so LoRa TimeSync can recover if the RTC is wrong. +- Sender uses a short “fast acquisition” mode on boot (until first LoRa TimeSync) with wider RX windows to avoid phase-miss. ## Build Environments -- `lilygo-t3-v1-6-1`: production build +- `lilygo-t3-v1-6-1`: production build (debug on) - `lilygo-t3-v1-6-1-test`: test build with `ENABLE_TEST_MODE` -- `lilygo-t3-v1-6-1-868`: production build for 868 MHz modules +- `lilygo-t3-v1-6-1-868`: production build for 868 MHz modules (debug on) - `lilygo-t3-v1-6-1-868-test`: test build for 868 MHz modules +- `lilygo-t3-v1-6-1-payload-test`: build with `PAYLOAD_CODEC_TEST` +- `lilygo-t3-v1-6-1-868-payload-test`: 868 MHz build with `PAYLOAD_CODEC_TEST` +- `lilygo-t3-v1-6-1-prod`: production build with serial debug off +- `lilygo-t3-v1-6-1-868-prod`: 868 MHz production build with serial debug off ## Config Knobs Key timing settings in `include/config.h`: -- `METER_SAMPLE_INTERVAL_MS` -- `METER_SEND_INTERVAL_MS` -- `BATCH_ACK_TIMEOUT_MS` -- `BATCH_MAX_RETRIES` + - `METER_SAMPLE_INTERVAL_MS` + - `METER_SEND_INTERVAL_MS` + - `BATTERY_SAMPLE_INTERVAL_MS` + - `BATTERY_CAL` + - `BATCH_ACK_TIMEOUT_MS` + - `BATCH_MAX_RETRIES` - `BATCH_QUEUE_DEPTH` - `BATCH_RETRY_POLICY` (keep or drop on retry exhaustion) - - `SERIAL_DEBUG_MODE` / `SERIAL_DEBUG_DUMP_JSON` + - `SERIAL_DEBUG_MODE_FLAG` (build flag) / `SERIAL_DEBUG_DUMP_JSON` - `LORA_SEND_BYPASS` (debug only) + - `ENABLE_SD_LOGGING` / `PIN_SD_CS` + - `SENDER_TIMESYNC_WINDOW_MS` + - `SENDER_TIMESYNC_CHECK_SEC_FAST` / `SENDER_TIMESYNC_CHECK_SEC_SLOW` + - `TIME_SYNC_DRIFT_THRESHOLD_SEC` + - `TIME_SYNC_BURST_INTERVAL_MS` / `TIME_SYNC_BURST_DURATION_MS` + - `TIME_SYNC_ERROR_TIMEOUT_MS` + - `SD_HISTORY_MAX_DAYS` / `SD_HISTORY_MIN_RES_MIN` + - `SD_HISTORY_MAX_BINS` / `SD_HISTORY_TIME_BUDGET_MS` + - `WEB_AUTH_REQUIRE_STA` / `WEB_AUTH_REQUIRE_AP` / `WEB_AUTH_DEFAULT_USER` / `WEB_AUTH_DEFAULT_PASS` ## Limits & Known Constraints -- **Compression**: uses lightweight RLE (good for JSON but not optimal). +- **Compression**: MeterData uses lightweight RLE (good for JSON but not optimal). - **OBIS parsing**: supports IEC 62056-21 ASCII (Mode D); may need tuning for some meters. -- **Payload size**: single JSON frames < 256 bytes (ArduinoJson static doc); batch frames are chunked and reassembled. -- **Battery ADC**: uses simple linear calibration constant in `power_manager.cpp`. +- **Payload size**: single JSON frames < 256 bytes (ArduinoJson static doc); binary batch frames are chunked and reassembled (typically 1 chunk). +- **Battery ADC**: uses a divider (R44/R45 = 100K/100K) with a configurable `BATTERY_CAL` scale and LiPo % curve. - **OLED**: no hardware reset line is used (matches working reference). - **Batch ACKs**: sender waits for ACK after a batch and retries up to `BATCH_MAX_RETRIES` with `BATCH_ACK_TIMEOUT_MS` between attempts. +## SD Logging (Receiver) +Optional CSV logging to microSD (FAT32) when `ENABLE_SD_LOGGING = true`. + +- Path: `/dd3//YYYY-MM-DD.csv` +- Columns: + `ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last` +- `err_last` is written as text (`meter`, `decode`, `loratx`) only on the last sample of a batch that reports an error. +- Files are downloadable from the main UI page. +- Downloads only allow absolute paths under `/dd3/`, reject `..`, backslashes, and repeated slashes, and enforce a max path length. +- History chart on sender page stream-parses CSVs and bins data in the background. +- SD uses the on-board microSD SPI pins (CS=13, MOSI=15, SCK=14, MISO=2). + ## Files & Modules - `include/config.h`, `src/config.cpp`: pins, radio settings, sender IDs - `include/data_model.h`, `src/data_model.cpp`: MeterData + ID init - `include/json_codec.h`, `src/json_codec.cpp`: JSON encode/decode - `include/compressor.h`, `src/compressor.cpp`: RLE compression - `include/lora_transport.h`, `src/lora_transport.cpp`: LoRa packet + CRC +- `src/payload_codec.h`, `src/payload_codec.cpp`: binary batch encoder/decoder - `include/meter_driver.h`, `src/meter_driver.cpp`: IEC 62056-21 ASCII parse - `include/power_manager.h`, `src/power_manager.cpp`: ADC + sleep - `include/time_manager.h`, `src/time_manager.cpp`: NTP + time sync @@ -315,10 +387,10 @@ Key timing settings in `include/config.h`: - `src/main.cpp`: role detection and main loop ## Quick Start -1. Set role jumper on GPIO13: +1. Set role jumper on GPIO14: - LOW: sender - HIGH: receiver -2. OLED control on GPIO14: +2. OLED control on GPIO13: - HIGH: always on - LOW: auto-off after 10 minutes 3. Build and upload: diff --git a/include/config.h b/include/config.h index c9e8fc9..00c9d14 100644 --- a/include/config.h +++ b/include/config.h @@ -39,8 +39,8 @@ constexpr uint8_t OLED_HEIGHT = 64; constexpr uint8_t PIN_BAT_ADC = 35; -constexpr uint8_t PIN_ROLE = 13; -constexpr uint8_t PIN_OLED_CTRL = 14; +constexpr uint8_t PIN_ROLE = 14; +constexpr uint8_t PIN_OLED_CTRL = 13; constexpr uint8_t PIN_METER_RX = 34; @@ -49,7 +49,7 @@ constexpr uint8_t PIN_METER_RX = 34; #define LORA_FREQUENCY_HZ 433E6 #endif constexpr long LORA_FREQUENCY = LORA_FREQUENCY_HZ; -constexpr uint8_t LORA_SPREADING_FACTOR = 10; +constexpr uint8_t LORA_SPREADING_FACTOR = 12; constexpr long LORA_BANDWIDTH = 125E3; constexpr uint8_t LORA_CODING_RATE = 5; constexpr uint8_t LORA_SYNC_WORD = 0x34; @@ -60,22 +60,51 @@ constexpr uint32_t SENDER_WAKE_INTERVAL_SEC = 30; constexpr uint32_t TIME_SYNC_INTERVAL_SEC = 60; constexpr uint32_t TIME_SYNC_SLOW_INTERVAL_SEC = 3600; constexpr uint32_t TIME_SYNC_FAST_WINDOW_MS = 10UL * 60UL * 1000UL; +constexpr uint32_t SENDER_TIMESYNC_WINDOW_MS = 300; +constexpr uint32_t SENDER_TIMESYNC_CHECK_SEC_FAST = 60; +constexpr uint32_t SENDER_TIMESYNC_CHECK_SEC_SLOW = 3600; +constexpr uint32_t TIME_SYNC_DRIFT_THRESHOLD_SEC = 10; +constexpr uint32_t TIME_SYNC_BURST_INTERVAL_MS = 10000; +constexpr uint32_t TIME_SYNC_BURST_DURATION_MS = 10UL * 60UL * 1000UL; +constexpr uint32_t TIME_SYNC_ERROR_TIMEOUT_MS = 2UL * 24UL * 60UL * 60UL * 1000UL; constexpr bool ENABLE_DS3231 = true; constexpr uint32_t OLED_PAGE_INTERVAL_MS = 4000; constexpr uint32_t OLED_AUTO_OFF_MS = 10UL * 60UL * 1000UL; constexpr uint32_t SENDER_OLED_READ_MS = 10000; constexpr uint32_t METER_SAMPLE_INTERVAL_MS = 1000; constexpr uint32_t METER_SEND_INTERVAL_MS = 30000; +constexpr uint32_t BATTERY_SAMPLE_INTERVAL_MS = 60000; +constexpr float BATTERY_CAL = 1.083f; constexpr uint32_t BATCH_ACK_TIMEOUT_MS = 3000; +constexpr uint8_t ACK_REPEAT_COUNT = 3; +constexpr uint32_t ACK_REPEAT_DELAY_MS = 200; constexpr uint8_t BATCH_MAX_RETRIES = 2; constexpr uint8_t METER_BATCH_MAX_SAMPLES = 30; constexpr uint8_t BATCH_QUEUE_DEPTH = 10; constexpr BatchRetryPolicy BATCH_RETRY_POLICY = BatchRetryPolicy::Keep; constexpr uint32_t WATCHDOG_TIMEOUT_SEC = 120; constexpr bool ENABLE_HA_DISCOVERY = true; -constexpr bool SERIAL_DEBUG_MODE = false; +#ifndef SERIAL_DEBUG_MODE_FLAG +#define SERIAL_DEBUG_MODE_FLAG 0 +#endif +constexpr bool SERIAL_DEBUG_MODE = SERIAL_DEBUG_MODE_FLAG != 0; constexpr bool SERIAL_DEBUG_DUMP_JSON = false; constexpr bool LORA_SEND_BYPASS = false; +constexpr bool ENABLE_SD_LOGGING = true; +constexpr uint8_t PIN_SD_CS = 13; +constexpr uint8_t PIN_SD_MOSI = 15; +constexpr uint8_t PIN_SD_MISO = 2; +constexpr uint8_t PIN_SD_SCK = 14; +constexpr uint16_t SD_HISTORY_MAX_DAYS = 30; +constexpr uint16_t SD_HISTORY_MIN_RES_MIN = 1; +constexpr uint16_t SD_HISTORY_MAX_BINS = 4000; +constexpr uint16_t SD_HISTORY_TIME_BUDGET_MS = 10; +constexpr const char *AP_SSID_PREFIX = "DD3-Bridge-"; +constexpr const char *AP_PASSWORD = "changeme123"; +constexpr bool WEB_AUTH_REQUIRE_STA = true; +constexpr bool WEB_AUTH_REQUIRE_AP = false; +constexpr const char *WEB_AUTH_DEFAULT_USER = "admin"; +constexpr const char *WEB_AUTH_DEFAULT_PASS = "admin"; constexpr uint8_t NUM_SENDERS = 1; inline constexpr uint16_t EXPECTED_SENDER_IDS[NUM_SENDERS] = { diff --git a/include/data_model.h b/include/data_model.h index 94ed471..cb30367 100644 --- a/include/data_model.h +++ b/include/data_model.h @@ -6,7 +6,19 @@ enum class FaultType : uint8_t { None = 0, MeterRead = 1, Decode = 2, - LoraTx = 3 + LoraTx = 3, + TimeSync = 4 +}; + +enum class RxRejectReason : uint8_t { + None = 0, + CrcFail = 1, + BadProtocol = 2, + WrongRole = 3, + WrongPayloadType = 4, + LengthMismatch = 5, + DeviceIdMismatch = 6, + BatchIdMismatch = 7 }; struct FaultCounters { @@ -32,6 +44,7 @@ struct MeterData { uint32_t err_decode; uint32_t err_lora_tx; FaultType last_error; + uint8_t rx_reject_reason; }; struct SenderStatus { @@ -41,3 +54,4 @@ struct SenderStatus { }; void init_device_ids(uint16_t &short_id, char *device_id, size_t device_id_len); +const char *rx_reject_reason_text(RxRejectReason reason); diff --git a/include/html_util.h b/include/html_util.h new file mode 100644 index 0000000..02c96b4 --- /dev/null +++ b/include/html_util.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +String html_escape(const String &input); +String url_encode_component(const String &input); +bool sanitize_device_id(const String &input, String &out_device_id); diff --git a/include/lora_transport.h b/include/lora_transport.h index bb445fb..97e3b75 100644 --- a/include/lora_transport.h +++ b/include/lora_transport.h @@ -2,6 +2,7 @@ #include #include "config.h" +#include "data_model.h" constexpr size_t LORA_MAX_PAYLOAD = 230; @@ -19,5 +20,9 @@ struct LoraPacket { void lora_init(); bool lora_send(const LoraPacket &pkt); bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms); +RxRejectReason lora_get_last_rx_reject_reason(); +void lora_idle(); void lora_sleep(); +void lora_receive_continuous(); +bool lora_receive_window(LoraPacket &pkt, uint32_t timeout_ms); uint32_t lora_airtime_ms(size_t packet_len); diff --git a/include/meter_driver.h b/include/meter_driver.h index 25d4761..a38cf73 100644 --- a/include/meter_driver.h +++ b/include/meter_driver.h @@ -5,3 +5,5 @@ void meter_init(); bool meter_read(MeterData &data); +bool meter_poll_frame(const char *&frame, size_t &len); +bool meter_parse_frame(const char *frame, size_t len, MeterData &data); diff --git a/include/power_manager.h b/include/power_manager.h index 110c377..0a4aeea 100644 --- a/include/power_manager.h +++ b/include/power_manager.h @@ -5,6 +5,7 @@ void power_sender_init(); void power_receiver_init(); +void power_configure_unused_pins_sender(); void read_battery(MeterData &data); uint8_t battery_percent_from_voltage(float voltage_v); void light_sleep_ms(uint32_t ms); diff --git a/include/wifi_manager.h b/include/wifi_manager.h index e9012ca..74f288c 100644 --- a/include/wifi_manager.h +++ b/include/wifi_manager.h @@ -12,6 +12,8 @@ struct WifiMqttConfig { String mqtt_pass; String ntp_server_1; String ntp_server_2; + String web_user; + String web_pass; bool valid; }; diff --git a/platformio.ini b/platformio.ini index 277abd6..99e3165 100644 --- a/platformio.ini +++ b/platformio.ini @@ -18,6 +18,8 @@ lib_deps = adafruit/Adafruit SSD1306@^2.5.9 adafruit/Adafruit GFX Library@^1.11.9 knolleary/PubSubClient@^2.8 +build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 [env:lilygo-t3-v1-6-1-test] platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip @@ -30,6 +32,7 @@ lib_deps = adafruit/Adafruit GFX Library@^1.11.9 knolleary/PubSubClient@^2.8 build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 -DENABLE_TEST_MODE [env:lilygo-t3-v1-6-1-868] @@ -43,6 +46,7 @@ lib_deps = adafruit/Adafruit GFX Library@^1.11.9 knolleary/PubSubClient@^2.8 build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 -DLORA_FREQUENCY_HZ=868E6 [env:lilygo-t3-v1-6-1-868-test] @@ -56,5 +60,62 @@ lib_deps = adafruit/Adafruit GFX Library@^1.11.9 knolleary/PubSubClient@^2.8 build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 -DENABLE_TEST_MODE -DLORA_FREQUENCY_HZ=868E6 + +[env:lilygo-t3-v1-6-1-payload-test] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 + -DPAYLOAD_CODEC_TEST + +[env:lilygo-t3-v1-6-1-868-payload-test] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DSERIAL_DEBUG_MODE_FLAG=1 + -DPAYLOAD_CODEC_TEST + -DLORA_FREQUENCY_HZ=868E6 + +[env:lilygo-t3-v1-6-1-prod] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DSERIAL_DEBUG_MODE_FLAG=0 + +[env:lilygo-t3-v1-6-1-868-prod] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.07/platform-espressif32.zip +board = ttgo-lora32-v1 +framework = arduino +lib_deps = + sandeepmistry/LoRa@^0.8.0 + bblanchon/ArduinoJson@^6.21.5 + adafruit/Adafruit SSD1306@^2.5.9 + adafruit/Adafruit GFX Library@^1.11.9 + knolleary/PubSubClient@^2.8 +build_flags = + -DSERIAL_DEBUG_MODE_FLAG=0 + -DLORA_FREQUENCY_HZ=868E6 diff --git a/src/config.cpp b/src/config.cpp index cf2fe9d..0632bce 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -2,5 +2,5 @@ DeviceRole detect_role() { pinMode(PIN_ROLE, INPUT_PULLDOWN); - return digitalRead(PIN_ROLE) == HIGH ? DeviceRole::Receiver : DeviceRole::Sender; + return digitalRead(PIN_ROLE) == HIGH ? DeviceRole::Sender : DeviceRole::Receiver; } diff --git a/src/data_model.cpp b/src/data_model.cpp index f1bc694..fbdc0e0 100644 --- a/src/data_model.cpp +++ b/src/data_model.cpp @@ -8,3 +8,24 @@ void init_device_ids(uint16_t &short_id, char *device_id, size_t device_id_len) short_id = (static_cast(mac[4]) << 8) | mac[5]; snprintf(device_id, device_id_len, "dd3-%04X", short_id); } + +const char *rx_reject_reason_text(RxRejectReason reason) { + switch (reason) { + case RxRejectReason::CrcFail: + return "crc_fail"; + case RxRejectReason::BadProtocol: + return "bad_protocol_version"; + case RxRejectReason::WrongRole: + return "wrong_role"; + case RxRejectReason::WrongPayloadType: + return "wrong_payload_type"; + case RxRejectReason::LengthMismatch: + return "length_mismatch"; + case RxRejectReason::DeviceIdMismatch: + return "device_id_mismatch"; + case RxRejectReason::BatchIdMismatch: + return "batch_id_mismatch"; + default: + return "none"; + } +} diff --git a/src/display_ui.cpp b/src/display_ui.cpp index ac5a013..c276d46 100644 --- a/src/display_ui.cpp +++ b/src/display_ui.cpp @@ -36,10 +36,9 @@ static bool g_mqtt_ok = false; static bool g_oled_on = true; static bool g_prev_ctrl_high = false; -static uint32_t g_oled_off_start = 0; static uint32_t g_last_page_ms = 0; static uint8_t g_page = 0; -static uint32_t g_boot_ms = 0; +static uint32_t g_last_activity_ms = 0; static bool g_display_ready = false; static uint32_t g_last_init_attempt_ms = 0; static bool g_last_oled_on = true; @@ -69,7 +68,9 @@ void display_power_down() { } void display_init() { - pinMode(PIN_OLED_CTRL, INPUT_PULLDOWN); + if (g_role == DeviceRole::Sender) { + pinMode(PIN_OLED_CTRL, INPUT_PULLDOWN); + } Wire.begin(PIN_OLED_SDA, PIN_OLED_SCL); Wire.setClock(100000); g_display_ready = display.begin(SSD1306_SWITCHCAPVCC, OLED_I2C_ADDR); @@ -81,7 +82,7 @@ void display_init() { display.display(); } g_last_init_attempt_ms = millis(); - g_boot_ms = millis(); + g_last_activity_ms = millis(); } void display_set_role(DeviceRole role) { @@ -171,6 +172,8 @@ static bool render_last_error_line(uint8_t y) { label = "decode"; } else if (g_last_error == FaultType::LoraTx) { label = "lora"; + } else if (g_last_error == FaultType::TimeSync) { + label = "timesync"; } display.setCursor(0, y); display.printf("Err: %s %lus", label, static_cast(age_seconds(g_last_error_ts, g_last_error_ms))); @@ -268,16 +271,10 @@ static void render_receiver_status() { display.printf("Time: %s", time_buf); uint32_t latest = 0; - bool link_valid = false; - int16_t link_rssi = 0; - float link_snr = 0.0f; if (g_statuses) { for (uint8_t i = 0; i < g_status_count; ++i) { if (g_statuses[i].has_data && g_statuses[i].last_update_ts_utc > latest) { latest = g_statuses[i].last_update_ts_utc; - link_valid = g_statuses[i].last_data.link_valid; - link_rssi = g_statuses[i].last_data.link_rssi_dbm; - link_snr = g_statuses[i].last_data.link_snr_db; } } } @@ -291,9 +288,6 @@ static void render_receiver_status() { localtime_r(&t, &timeinfo); display.printf("Upd %02d:%02d", timeinfo.tm_hour, timeinfo.tm_min); } - if (link_valid) { - display.printf(" R:%d S:%.1f", link_rssi, link_snr); - } render_last_error_line(56); display.display(); @@ -343,14 +337,27 @@ static void render_receiver_sender(uint8_t index) { display.setCursor(0, 12); display.printf("E %.1f kWh", status.last_data.energy_total_kwh); - display.setCursor(0, 24); - display.printf("P %.0fW", status.last_data.total_power_w); - display.setCursor(0, 36); + display.setCursor(0, 22); display.printf("L1 %.0fW", status.last_data.phase_power_w[0]); - display.setCursor(0, 48); + display.setCursor(0, 32); display.printf("L2 %.0fW", status.last_data.phase_power_w[1]); - display.setCursor(0, 56); + display.setCursor(0, 42); display.printf("L3 %.0fW", status.last_data.phase_power_w[2]); + display.setCursor(0, 52); + display.print("P"); + char p_buf[16]; + snprintf(p_buf, sizeof(p_buf), "%.0fW", status.last_data.total_power_w); + int16_t x1 = 0; + int16_t y1 = 0; + uint16_t w = 0; + uint16_t h = 0; + display.getTextBounds(p_buf, 0, 0, &x1, &y1, &w, &h); + int16_t x = static_cast(display.width() - w); + if (x < 0) { + x = 0; + } + display.setCursor(x, 52); + display.print(p_buf); display.display(); } @@ -369,29 +376,21 @@ void display_tick() { } return; } - bool ctrl_high = digitalRead(PIN_OLED_CTRL) == HIGH; + bool ctrl_high = false; + if (g_role == DeviceRole::Sender) { + ctrl_high = digitalRead(PIN_OLED_CTRL) == HIGH; + } - bool in_boot_window = (millis() - g_boot_ms) < OLED_AUTO_OFF_MS; + uint32_t now_ms = millis(); + bool ctrl_falling_edge = g_prev_ctrl_high && !ctrl_high; if (g_role == DeviceRole::Receiver) { g_oled_on = true; - g_oled_off_start = 0; - } else if (in_boot_window) { - g_oled_on = true; + g_last_activity_ms = now_ms; } else { - if (ctrl_high) { - g_oled_on = true; - g_oled_off_start = 0; - } else if (g_prev_ctrl_high && !ctrl_high) { - g_oled_off_start = millis(); - } else if (!g_prev_ctrl_high && !ctrl_high && g_oled_off_start == 0) { - g_oled_off_start = millis(); + if (ctrl_high || ctrl_falling_edge) { + g_last_activity_ms = now_ms; } - - if (!ctrl_high && g_oled_off_start > 0 && millis() - g_oled_off_start > OLED_AUTO_OFF_MS) { - g_oled_on = false; - } - - // fall through to power gating below + g_oled_on = (now_ms - g_last_activity_ms) < OLED_AUTO_OFF_MS; } if (g_oled_on) { diff --git a/src/html_util.cpp b/src/html_util.cpp new file mode 100644 index 0000000..51b79d4 --- /dev/null +++ b/src/html_util.cpp @@ -0,0 +1,98 @@ +#include "html_util.h" + +String html_escape(const String &input) { + String out; + out.reserve(input.length() + 8); + for (size_t i = 0; i < input.length(); ++i) { + char c = input[i]; + switch (c) { + case '&': + out += "&"; + break; + case '<': + out += "<"; + break; + case '>': + out += ">"; + break; + case '"': + out += """; + break; + case '\'': + out += "'"; + break; + default: + out += c; + break; + } + } + return out; +} + +String url_encode_component(const String &input) { + String out; + out.reserve(input.length() * 3); + const char *hex = "0123456789ABCDEF"; + for (size_t i = 0; i < input.length(); ++i) { + unsigned char c = static_cast(input[i]); + bool safe = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~'; + if (safe) { + out += static_cast(c); + } else { + out += '%'; + out += hex[(c >> 4) & 0x0F]; + out += hex[c & 0x0F]; + } + } + return out; +} + +static bool is_hex_char(char c) { + return (c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +static String to_upper_hex4(const String &input) { + String out = input; + out.toUpperCase(); + return out; +} + +bool sanitize_device_id(const String &input, String &out_device_id) { + String trimmed = input; + trimmed.trim(); + if (trimmed.length() == 0) { + return false; + } + if (trimmed.indexOf('/') >= 0 || trimmed.indexOf('\\') >= 0 || trimmed.indexOf("..") >= 0) { + return false; + } + if (trimmed.indexOf('%') >= 0) { + return false; + } + + if (trimmed.length() == 4) { + for (size_t i = 0; i < 4; ++i) { + if (!is_hex_char(trimmed[i])) { + return false; + } + } + out_device_id = String("dd3-") + to_upper_hex4(trimmed); + return true; + } + + if (trimmed.length() == 8 && trimmed.startsWith("dd3-")) { + String hex = trimmed.substring(4); + for (size_t i = 0; i < 4; ++i) { + if (!is_hex_char(hex[i])) { + return false; + } + } + out_device_id = String("dd3-") + to_upper_hex4(hex); + return true; + } + + return false; +} diff --git a/src/json_codec.cpp b/src/json_codec.cpp index e3e88c6..765b53d 100644 --- a/src/json_codec.cpp +++ b/src/json_codec.cpp @@ -118,9 +118,9 @@ bool meterDataToJson(const MeterData &data, String &out_json) { if (data.err_lora_tx > 0) { doc["err_tx"] = data.err_lora_tx; } - if (data.last_error != FaultType::None) { - doc["err_last"] = static_cast(data.last_error); - } + doc["err_last"] = static_cast(data.last_error); + doc["rx_reject"] = data.rx_reject_reason; + doc["rx_reject_text"] = rx_reject_reason_text(static_cast(data.rx_reject_reason)); out_json = ""; size_t len = serializeJson(doc, out_json); @@ -162,6 +162,7 @@ bool jsonToMeterData(const String &json, MeterData &data) { data.err_decode = doc["err_d"] | 0; data.err_lora_tx = doc["err_tx"] | 0; data.last_error = static_cast(doc["err_last"] | 0); + data.rx_reject_reason = static_cast(doc["rx_reject"] | 0); if (strlen(data.device_id) >= 8) { const char *suffix = data.device_id + strlen(data.device_id) - 4; @@ -196,9 +197,7 @@ bool meterBatchToJson(const MeterData *samples, size_t count, uint16_t batch_id, doc["err_tx"] = faults->lora_tx_fail; } } - if (last_error != FaultType::None) { - doc["err_last"] = static_cast(last_error); - } + doc["err_last"] = static_cast(last_error); if (!isnan(samples[count - 1].battery_voltage_v)) { char bat_buf[16]; format_float_2(bat_buf, sizeof(bat_buf), samples[count - 1].battery_voltage_v); diff --git a/src/lora_transport.cpp b/src/lora_transport.cpp index 7ea0612..80443be 100644 --- a/src/lora_transport.cpp +++ b/src/lora_transport.cpp @@ -3,6 +3,26 @@ #include #include +static RxRejectReason g_last_rx_reject_reason = RxRejectReason::None; +static uint32_t g_last_rx_reject_log_ms = 0; + +static void note_reject(RxRejectReason reason) { + g_last_rx_reject_reason = reason; + if (SERIAL_DEBUG_MODE) { + uint32_t now_ms = millis(); + if (now_ms - g_last_rx_reject_log_ms >= 1000) { + g_last_rx_reject_log_ms = now_ms; + Serial.printf("lora_rx: reject reason=%s\n", rx_reject_reason_text(reason)); + } + } +} + +RxRejectReason lora_get_last_rx_reject_reason() { + RxRejectReason reason = g_last_rx_reject_reason; + g_last_rx_reject_reason = RxRejectReason::None; + return reason; +} + static uint16_t crc16_ccitt(const uint8_t *data, size_t len) { uint16_t crc = 0xFFFF; for (size_t i = 0; i < len; ++i) { @@ -33,6 +53,18 @@ bool lora_send(const LoraPacket &pkt) { if (LORA_SEND_BYPASS) { return true; } + uint32_t t0 = 0; + uint32_t t1 = 0; + uint32_t t2 = 0; + uint32_t t3 = 0; + uint32_t t4 = 0; + if (SERIAL_DEBUG_MODE) { + t0 = millis(); + } + LoRa.idle(); + if (SERIAL_DEBUG_MODE) { + t1 = millis(); + } uint8_t buffer[5 + LORA_MAX_PAYLOAD + 2]; size_t idx = 0; buffer[idx++] = pkt.protocol_version; @@ -53,8 +85,24 @@ bool lora_send(const LoraPacket &pkt) { buffer[idx++] = static_cast(crc & 0xFF); LoRa.beginPacket(); + if (SERIAL_DEBUG_MODE) { + t2 = millis(); + } LoRa.write(buffer, idx); - int result = LoRa.endPacket(); + if (SERIAL_DEBUG_MODE) { + t3 = millis(); + } + int result = LoRa.endPacket(false); + if (SERIAL_DEBUG_MODE) { + t4 = millis(); + Serial.printf("lora_tx: idle=%lums begin=%lums write=%lums end=%lums total=%lums len=%u\n", + static_cast(t1 - t0), + static_cast(t2 - t1), + static_cast(t3 - t2), + static_cast(t4 - t3), + static_cast(t4 - t0), + static_cast(idx)); + } return result == 1; } @@ -67,6 +115,7 @@ bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms) { while (LoRa.available()) { LoRa.read(); } + note_reject(RxRejectReason::LengthMismatch); return false; } @@ -77,12 +126,18 @@ bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms) { } if (len < 7) { + note_reject(RxRejectReason::LengthMismatch); return false; } uint16_t crc_calc = crc16_ccitt(buffer, len - 2); uint16_t crc_rx = static_cast(buffer[len - 2] << 8) | buffer[len - 1]; if (crc_calc != crc_rx) { + note_reject(RxRejectReason::CrcFail); + return false; + } + if (buffer[0] != PROTOCOL_VERSION) { + note_reject(RxRejectReason::BadProtocol); return false; } @@ -92,6 +147,7 @@ bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms) { pkt.payload_type = static_cast(buffer[4]); pkt.payload_len = len - 7; if (pkt.payload_len > LORA_MAX_PAYLOAD) { + note_reject(RxRejectReason::LengthMismatch); return false; } memcpy(pkt.payload, &buffer[5], pkt.payload_len); @@ -107,10 +163,28 @@ bool lora_receive(LoraPacket &pkt, uint32_t timeout_ms) { } } +void lora_idle() { + LoRa.idle(); +} + void lora_sleep() { LoRa.sleep(); } +void lora_receive_continuous() { + LoRa.receive(); +} + +bool lora_receive_window(LoraPacket &pkt, uint32_t timeout_ms) { + if (timeout_ms == 0) { + return false; + } + LoRa.receive(); + bool got = lora_receive(pkt, timeout_ms); + LoRa.sleep(); + return got; +} + uint32_t lora_airtime_ms(size_t packet_len) { if (packet_len == 0) { return 0; diff --git a/src/main.cpp b/src/main.cpp index c795fe3..392235f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2,6 +2,7 @@ #include "config.h" #include "data_model.h" #include "json_codec.h" +#include "payload_codec.h" #include "compressor.h" #include "lora_transport.h" #include "meter_driver.h" @@ -12,8 +13,9 @@ #include "web_server.h" #include "display_ui.h" #include "test_mode.h" -#include +#include "sd_logger.h" #include +#include #ifdef ARDUINO_ARCH_ESP32 #include #include @@ -48,10 +50,21 @@ static uint32_t g_sender_last_error_remote_ms[NUM_SENDERS] = {}; static bool g_sender_discovery_sent[NUM_SENDERS] = {}; static bool g_receiver_discovery_sent = false; +struct TimeSyncBurstState { + bool active; + uint32_t start_ms; + uint32_t last_send_ms; + uint32_t last_drift_check_ms; + bool last_drift_ok; +}; + +static TimeSyncBurstState g_timesync_burst[NUM_SENDERS] = {}; +static uint32_t g_sender_last_timesync_rx_ms = 0; +static bool g_sender_timesync_error = false; + static constexpr size_t BATCH_HEADER_SIZE = 6; static constexpr size_t BATCH_CHUNK_PAYLOAD = LORA_MAX_PAYLOAD - BATCH_HEADER_SIZE; static constexpr size_t BATCH_MAX_COMPRESSED = 4096; -static constexpr size_t BATCH_MAX_DECOMPRESSED = 8192; static constexpr uint32_t BATCH_RX_MARGIN_MS = 800; struct BatchBuffer { @@ -87,6 +100,20 @@ static uint8_t g_inflight_count = 0; static uint16_t g_inflight_batch_id = 0; static bool g_inflight_active = false; static uint32_t g_last_debug_log_ms = 0; +static uint32_t g_sender_last_timesync_check_ms = 0; +static uint32_t g_sender_rx_window_ms = 0; +static uint32_t g_sender_sleep_ms = 0; +static uint32_t g_sender_power_log_ms = 0; +static uint8_t g_sender_timesync_mode = 0; +static RxRejectReason g_sender_rx_reject_reason = RxRejectReason::None; +static uint32_t g_sender_rx_reject_log_ms = 0; +static MeterData g_last_meter_data = {}; +static bool g_last_meter_valid = false; +static uint32_t g_last_meter_rx_ms = 0; +static uint32_t g_meter_stale_seconds = 0; +static constexpr uint32_t SENDER_TIMESYNC_ACQUIRE_MS = 10UL * 60UL * 1000UL; +static constexpr uint32_t SENDER_TIMESYNC_ACQUIRE_INTERVAL_SEC = 20; +static constexpr uint32_t SENDER_TIMESYNC_ACQUIRE_WINDOW_MS = 3000; static void watchdog_kick(); @@ -102,23 +129,15 @@ static void serial_debug_printf(const char *fmt, ...) { Serial.println(buf); } -static void serial_debug_print_json(const String &json) { - if (!SERIAL_DEBUG_MODE || !SERIAL_DEBUG_DUMP_JSON) { +static void sender_set_timesync_mode(uint8_t mode) { + if (g_sender_timesync_mode == mode) { return; } - const char *data = json.c_str(); - size_t len = json.length(); - const size_t chunk = 128; - for (size_t i = 0; i < len; i += chunk) { - size_t n = len - i; - if (n > chunk) { - n = chunk; - } - Serial.write(reinterpret_cast(data + i), n); - watchdog_kick(); - delay(0); + g_sender_timesync_mode = mode; + if (SERIAL_DEBUG_MODE) { + const char *label = mode == 2 ? "acquire" : (mode == 1 ? "slow" : "fast"); + serial_debug_printf("timesync: mode=%s", label); } - Serial.write('\n'); } static uint16_t g_last_batch_id_rx[NUM_SENDERS] = {}; @@ -162,6 +181,41 @@ static void update_battery_cache() { g_last_battery_ms = millis(); } +static bool battery_sample_due(uint32_t now_ms) { + return g_last_battery_ms == 0 || now_ms - g_last_battery_ms >= BATTERY_SAMPLE_INTERVAL_MS; +} + +static bool sender_timesync_window_due() { + uint32_t interval_sec = SENDER_TIMESYNC_CHECK_SEC_FAST; + bool in_acquire = (g_sender_last_timesync_rx_ms == 0) && (millis() - g_boot_ms < SENDER_TIMESYNC_ACQUIRE_MS); + bool allow_slow = (millis() - g_boot_ms >= 60000UL) && time_is_synced() && time_rtc_present() && + (g_sender_last_timesync_rx_ms > 0); + // RTC boot time is not evidence of receiving network TimeSync. + if (in_acquire) { + interval_sec = SENDER_TIMESYNC_ACQUIRE_INTERVAL_SEC; + sender_set_timesync_mode(2); + } else if (allow_slow) { + interval_sec = SENDER_TIMESYNC_CHECK_SEC_SLOW; + sender_set_timesync_mode(1); + } else { + sender_set_timesync_mode(0); + } + static uint32_t last_interval_sec = 0; + if (last_interval_sec != interval_sec) { + last_interval_sec = interval_sec; + g_sender_last_timesync_check_ms = millis(); + } + if (g_sender_last_timesync_check_ms == 0) { + g_sender_last_timesync_check_ms = millis() - interval_sec * 1000UL; + } + uint32_t now_ms = millis(); + if (now_ms - g_sender_last_timesync_check_ms >= interval_sec * 1000UL) { + g_sender_last_timesync_check_ms = now_ms; + return true; + } + return false; +} + static bool batch_queue_drop_oldest() { if (g_batch_count == 0) { return false; @@ -180,6 +234,39 @@ static bool batch_queue_drop_oldest() { return dropped_inflight; } +static void receiver_note_timesync_drift(uint8_t sender_idx, uint32_t sender_ts_utc) { + if (sender_idx >= NUM_SENDERS) { + return; + } + if (!time_is_synced() || sender_ts_utc == 0) { + return; + } + uint32_t now_utc = time_get_utc(); + uint32_t diff = now_utc > sender_ts_utc ? now_utc - sender_ts_utc : sender_ts_utc - now_utc; + TimeSyncBurstState &state = g_timesync_burst[sender_idx]; + state.last_drift_check_ms = millis(); + state.last_drift_ok = diff <= TIME_SYNC_DRIFT_THRESHOLD_SEC; + if (!state.last_drift_ok) { + if (!state.active) { + state.active = true; + state.start_ms = millis(); + state.last_send_ms = 0; + } + } +} + +static void sender_note_rx_reject(RxRejectReason reason, const char *context) { + if (reason == RxRejectReason::None) { + return; + } + g_sender_rx_reject_reason = reason; + uint32_t now_ms = millis(); + if (SERIAL_DEBUG_MODE && now_ms - g_sender_rx_reject_log_ms >= 1000) { + g_sender_rx_reject_log_ms = now_ms; + serial_debug_printf("rx_reject: %s reason=%s", context, rx_reject_reason_text(reason)); + } +} + static BatchBuffer *batch_queue_peek() { if (g_batch_count == 0) { return nullptr; @@ -283,6 +370,63 @@ static uint16_t read_u16_le(const uint8_t *src) { return static_cast(src[0]) | (static_cast(src[1]) << 8); } +static uint16_t sender_id_from_short_id(uint16_t short_id) { + for (uint8_t i = 0; i < NUM_SENDERS; ++i) { + if (EXPECTED_SENDER_IDS[i] == short_id) { + return static_cast(i + 1); + } + } + return 0; +} + +static uint16_t short_id_from_sender_id(uint16_t sender_id) { + if (sender_id == 0 || sender_id > NUM_SENDERS) { + return 0; + } + return EXPECTED_SENDER_IDS[sender_id - 1]; +} + +static uint32_t kwh_to_wh_from_float(float value) { + if (isnan(value)) { + return 0; + } + double wh = static_cast(value) * 1000.0; + if (wh < 0.0) { + wh = 0.0; + } + if (wh > static_cast(UINT32_MAX)) { + wh = static_cast(UINT32_MAX); + } + return static_cast(llround(wh)); +} + +static bool float_to_i16_w(float value, int16_t &out) { + if (isnan(value)) { + out = 0; + return true; + } + long rounded = lroundf(value); + if (rounded < INT16_MIN || rounded > INT16_MAX) { + return false; + } + out = static_cast(rounded); + return true; +} + +static uint16_t battery_mv_from_voltage(float value) { + if (isnan(value) || value <= 0.0f) { + return 0; + } + long mv = lroundf(value * 1000.0f); + if (mv < 0) { + mv = 0; + } + if (mv > UINT16_MAX) { + mv = UINT16_MAX; + } + return static_cast(mv); +} + static uint32_t compute_batch_rx_timeout_ms(uint16_t total_len, uint8_t chunk_count) { if (total_len == 0 || chunk_count == 0) { return 10000; @@ -306,22 +450,6 @@ static uint32_t compute_batch_ack_timeout_ms(size_t payload_len) { return timeout_ms < 10000 ? 10000 : timeout_ms; } -static bool inject_batch_meta(String &json, int16_t rssi_dbm, float snr_db, uint32_t rx_ts_utc) { - DynamicJsonDocument doc(8192); - DeserializationError err = deserializeJson(doc, json); - if (err) { - return false; - } - - JsonObject meta = doc.createNestedObject("meta"); - meta["rssi"] = rssi_dbm; - meta["snr"] = snr_db; - meta["rx_ts"] = rx_ts_utc; - - json = ""; - return serializeJson(doc, json) > 0; -} - static bool send_batch_payload(const uint8_t *data, size_t len, uint32_t ts_for_display, uint16_t batch_id) { if (!data || len == 0 || len > BATCH_MAX_COMPRESSED) { return false; @@ -383,7 +511,18 @@ static void send_batch_ack(uint16_t batch_id, uint16_t sender_id) { write_u16_le(&ack.payload[0], batch_id); write_u16_le(&ack.payload[2], sender_id); write_u16_le(&ack.payload[4], g_short_id); - lora_send(ack); + uint8_t repeats = ACK_REPEAT_COUNT == 0 ? 1 : ACK_REPEAT_COUNT; + for (uint8_t i = 0; i < repeats; ++i) { + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("ack: tx repeat %u/%u batch_id=%u", static_cast(i + 1), + static_cast(repeats), batch_id); + } + lora_send(ack); + if (i + 1 < repeats && ACK_REPEAT_DELAY_MS > 0) { + delay(ACK_REPEAT_DELAY_MS); + } + } + lora_receive_continuous(); } static bool prepare_inflight_from_queue() { @@ -411,41 +550,53 @@ static bool send_inflight_batch(uint32_t ts_for_display) { if (!g_inflight_active || g_inflight_count == 0) { return false; } - uint32_t json_start = millis(); - String json; - if (!meterBatchToJson(g_inflight_samples, g_inflight_count, g_inflight_batch_id, json, &g_sender_faults, g_sender_last_error)) { - return false; - } - uint32_t json_ms = millis() - json_start; - if (SERIAL_DEBUG_MODE) { - serial_debug_printf("tx: batch_id=%u count=%u json_len=%u", g_inflight_batch_id, g_inflight_count, static_cast(json.length())); - if (json_ms > 200) { - serial_debug_printf("tx: json encode took %lums", static_cast(json_ms)); + BatchInput input = {}; + input.sender_id = sender_id_from_short_id(g_short_id); + input.batch_id = g_inflight_batch_id; + input.t_last = g_inflight_samples[g_inflight_count - 1].ts_utc; + uint32_t dt_s = METER_SAMPLE_INTERVAL_MS / 1000; + input.dt_s = dt_s > 0 ? static_cast(dt_s) : 1; + input.n = g_inflight_count; + input.battery_mV = battery_mv_from_voltage(g_inflight_samples[g_inflight_count - 1].battery_voltage_v); + input.err_m = g_sender_faults.meter_read_fail > 255 ? 255 : static_cast(g_sender_faults.meter_read_fail); + input.err_d = g_sender_faults.decode_fail > 255 ? 255 : static_cast(g_sender_faults.decode_fail); + input.err_tx = g_sender_faults.lora_tx_fail > 255 ? 255 : static_cast(g_sender_faults.lora_tx_fail); + input.err_last = static_cast(g_sender_last_error); + input.err_rx_reject = static_cast(g_sender_rx_reject_reason); + for (uint8_t i = 0; i < g_inflight_count; ++i) { + input.energy_wh[i] = kwh_to_wh_from_float(g_inflight_samples[i].energy_total_kwh); + if (!float_to_i16_w(g_inflight_samples[i].phase_power_w[0], input.p1_w[i]) || + !float_to_i16_w(g_inflight_samples[i].phase_power_w[1], input.p2_w[i]) || + !float_to_i16_w(g_inflight_samples[i].phase_power_w[2], input.p3_w[i])) { + return false; } - serial_debug_print_json(json); } - static uint8_t compressed[BATCH_MAX_COMPRESSED]; - size_t compressed_len = 0; - uint32_t compress_start = millis(); - if (!compressBuffer(reinterpret_cast(json.c_str()), json.length(), compressed, sizeof(compressed), compressed_len)) { + static uint8_t encoded[BATCH_MAX_COMPRESSED]; + size_t encoded_len = 0; + uint32_t encode_start = millis(); + if (!encode_batch(input, encoded, sizeof(encoded), &encoded_len)) { return false; } - uint32_t compress_ms = millis() - compress_start; - if (SERIAL_DEBUG_MODE && compress_ms > 200) { - serial_debug_printf("tx: compress took %lums", static_cast(compress_ms)); + uint32_t encode_ms = millis() - encode_start; + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("tx: batch_id=%u count=%u bin_len=%u", g_inflight_batch_id, g_inflight_count, + static_cast(encoded_len)); + if (encode_ms > 200) { + serial_debug_printf("tx: encode took %lums", static_cast(encode_ms)); + } } - g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(compressed_len); + g_batch_ack_timeout_ms = compute_batch_ack_timeout_ms(encoded_len); uint32_t send_start = millis(); - bool ok = send_batch_payload(compressed, compressed_len, ts_for_display, g_inflight_batch_id); + bool ok = send_batch_payload(encoded, encoded_len, ts_for_display, g_inflight_batch_id); uint32_t send_ms = millis() - send_start; if (SERIAL_DEBUG_MODE && send_ms > 1000) { serial_debug_printf("tx: send batch took %lums", static_cast(send_ms)); } if (ok) { g_last_batch_send_ms = millis(); - serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast(compressed_len)); + serial_debug_printf("tx: sent batch_id=%u len=%u", g_inflight_batch_id, static_cast(encoded_len)); } else { serial_debug_printf("tx: send failed batch_id=%u", g_inflight_batch_id); } @@ -498,7 +649,7 @@ static void reset_batch_rx() { g_batch_rx.timeout_ms = 0; } -static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool &decode_error, uint16_t &out_batch_id) { +static bool process_batch_packet(const LoraPacket &pkt, BatchInput &out_batch, bool &decode_error, uint16_t &out_batch_id) { decode_error = false; if (pkt.payload_len < BATCH_HEADER_SIZE) { return false; @@ -545,20 +696,11 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool & g_batch_rx.last_rx_ms = now_ms; if (g_batch_rx.next_index == g_batch_rx.expected_chunks && g_batch_rx.received_len == g_batch_rx.total_len) { - static uint8_t decompressed[BATCH_MAX_DECOMPRESSED]; - size_t decompressed_len = 0; - if (!decompressBuffer(g_batch_rx.buffer, g_batch_rx.received_len, decompressed, sizeof(decompressed) - 1, decompressed_len)) { + if (!decode_batch(g_batch_rx.buffer, g_batch_rx.received_len, &out_batch)) { decode_error = true; reset_batch_rx(); return false; } - if (decompressed_len >= sizeof(decompressed)) { - decode_error = true; - reset_batch_rx(); - return false; - } - decompressed[decompressed_len] = '\0'; - out_json = String(reinterpret_cast(decompressed)); out_batch_id = batch_id; reset_batch_rx(); return true; @@ -570,11 +712,15 @@ static bool process_batch_packet(const LoraPacket &pkt, String &out_json, bool & void setup() { Serial.begin(115200); delay(200); +#ifdef PAYLOAD_CODEC_TEST + payload_codec_self_test(); +#endif watchdog_init(); g_boot_ms = millis(); g_role = detect_role(); init_device_ids(g_short_id, g_device_id, sizeof(g_device_id)); + display_set_role(g_role); if (SERIAL_DEBUG_MODE) { #ifdef ARDUINO_ARCH_ESP32 serial_debug_printf("boot: reset_reason=%d", static_cast(esp_reset_reason())); @@ -587,17 +733,20 @@ void setup() { display_init(); time_rtc_init(); time_try_load_from_rtc(); - display_set_role(g_role); display_set_self_ids(g_short_id, g_device_id); if (g_role == DeviceRole::Sender) { power_sender_init(); + power_configure_unused_pins_sender(); meter_init(); g_last_sample_ms = millis() - METER_SAMPLE_INTERVAL_MS; g_last_send_ms = millis(); update_battery_cache(); } else { power_receiver_init(); + lora_receive_continuous(); + pinMode(PIN_ROLE, INPUT); // release pulldown before SD uses GPIO14 as SCK + sd_logger_init(); wifi_manager_init(); init_sender_statuses(); display_set_sender_statuses(g_sender_statuses, NUM_SENDERS); @@ -613,8 +762,8 @@ void setup() { } else { g_ap_mode = true; char ap_ssid[32]; - snprintf(ap_ssid, sizeof(ap_ssid), "DD3-Bridge-%04X", g_short_id); - wifi_start_ap(ap_ssid, "changeme123"); + snprintf(ap_ssid, sizeof(ap_ssid), "%s%04X", AP_SSID_PREFIX, g_short_id); + wifi_start_ap(ap_ssid, AP_PASSWORD); if (g_cfg.ntp_server_1.isEmpty()) { g_cfg.ntp_server_1 = "pool.ntp.org"; } @@ -645,22 +794,52 @@ static void sender_loop() { g_batch_retry_count); } + const char *frame = nullptr; + size_t frame_len = 0; + if (meter_poll_frame(frame, frame_len)) { + MeterData parsed = {}; + parsed.energy_total_kwh = NAN; + parsed.total_power_w = NAN; + parsed.phase_power_w[0] = NAN; + parsed.phase_power_w[1] = NAN; + parsed.phase_power_w[2] = NAN; + parsed.valid = false; + if (meter_parse_frame(frame, frame_len, parsed)) { + g_last_meter_data = parsed; + g_last_meter_valid = true; + g_last_meter_rx_ms = now_ms; + g_meter_stale_seconds = 0; + } + } + if (now_ms - g_last_sample_ms >= METER_SAMPLE_INTERVAL_MS) { g_last_sample_ms = now_ms; MeterData data = {}; data.short_id = g_short_id; strncpy(data.device_id, g_device_id, sizeof(data.device_id)); - bool meter_ok = meter_read(data); + bool meter_ok = g_last_meter_valid; + if (meter_ok) { + data.energy_total_kwh = g_last_meter_data.energy_total_kwh; + data.total_power_w = g_last_meter_data.total_power_w; + data.phase_power_w[0] = g_last_meter_data.phase_power_w[0]; + data.phase_power_w[1] = g_last_meter_data.phase_power_w[1]; + data.phase_power_w[2] = g_last_meter_data.phase_power_w[2]; + uint32_t age_ms = now_ms - g_last_meter_rx_ms; + g_meter_stale_seconds = age_ms >= 1000 ? (age_ms / 1000) : 0; + } else { + g_meter_stale_seconds++; + } if (!meter_ok) { note_fault(g_sender_faults, g_sender_last_error, g_sender_last_error_utc, g_sender_last_error_ms, FaultType::MeterRead); display_set_last_error(g_sender_last_error, g_sender_last_error_utc, g_sender_last_error_ms); } - if (g_build_count == 0) { + if (g_build_count == 0 && battery_sample_due(now_ms)) { update_battery_cache(); } data.battery_voltage_v = g_last_battery_voltage_v; data.battery_percent = g_last_battery_percent; + data.rx_reject_reason = static_cast(g_sender_rx_reject_reason); uint32_t now_utc = time_get_utc(); data.ts_utc = now_utc > 0 ? now_utc : millis() / 1000; @@ -682,44 +861,102 @@ static void sender_loop() { } if (g_batch_ack_pending) { - uint32_t end_ms = millis() + 400; - while (millis() < end_ms) { - LoraPacket ack_pkt = {}; - if (!lora_receive(ack_pkt, 0) || ack_pkt.protocol_version != PROTOCOL_VERSION) { - delay(5); - continue; - } - if (ack_pkt.payload_type == PayloadType::Ack && ack_pkt.payload_len >= 6 && ack_pkt.role == DeviceRole::Receiver) { - uint16_t ack_id = read_u16_le(ack_pkt.payload); - uint16_t ack_sender = read_u16_le(&ack_pkt.payload[2]); - uint16_t ack_receiver = read_u16_le(&ack_pkt.payload[4]); - if (ack_sender == g_short_id && ack_receiver == ack_pkt.device_id_short && - g_batch_ack_pending && ack_id == g_last_sent_batch_id) { - g_last_acked_batch_id = ack_id; - serial_debug_printf("ack: ok batch_id=%u", ack_id); - finish_inflight_batch(); - break; + LoraPacket ack_pkt = {}; + const uint32_t ack_len = 5 + 6 + 2; + uint32_t ack_air_ms = lora_airtime_ms(ack_len); + uint32_t ack_window_ms = ack_air_ms + 300; + if (ack_window_ms < 1200) { + ack_window_ms = 1200; + } + if (ack_window_ms > 4000) { + ack_window_ms = 4000; + } + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("ack: rx window=%lu airtime=%lu", static_cast(ack_window_ms), + static_cast(ack_air_ms)); + } + uint32_t rx_start = millis(); + bool got_ack = lora_receive_window(ack_pkt, ack_window_ms); + if (!got_ack) { + got_ack = lora_receive_window(ack_pkt, ack_window_ms / 2); + } + uint32_t rx_elapsed = millis() - rx_start; + if (SERIAL_DEBUG_MODE) { + g_sender_rx_window_ms += rx_elapsed; + } + if (!got_ack) { + sender_note_rx_reject(lora_get_last_rx_reject_reason(), "ack"); + } else if (ack_pkt.role != DeviceRole::Receiver) { + sender_note_rx_reject(RxRejectReason::WrongRole, "ack"); + } else if (ack_pkt.payload_type != PayloadType::Ack) { + sender_note_rx_reject(RxRejectReason::WrongPayloadType, "ack"); + } else if (ack_pkt.payload_len < 6) { + sender_note_rx_reject(RxRejectReason::LengthMismatch, "ack"); + } else { + uint16_t ack_id = read_u16_le(ack_pkt.payload); + uint16_t ack_sender = read_u16_le(&ack_pkt.payload[2]); + uint16_t ack_receiver = read_u16_le(&ack_pkt.payload[4]); + if (ack_sender == g_short_id && ack_receiver == ack_pkt.device_id_short && + g_batch_ack_pending && ack_id == g_last_sent_batch_id) { + g_last_acked_batch_id = ack_id; + serial_debug_printf("ack: rx ok batch_id=%u", ack_id); + finish_inflight_batch(); + } else { + if (ack_sender != g_short_id || ack_receiver != ack_pkt.device_id_short) { + sender_note_rx_reject(RxRejectReason::DeviceIdMismatch, "ack"); + } else if (ack_id != g_last_sent_batch_id) { + sender_note_rx_reject(RxRejectReason::BatchIdMismatch, "ack"); + } + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("ack: reject batch_id=%u sender=%u receiver=%u exp_batch=%u exp_sender=%u", + ack_id, ack_sender, ack_receiver, g_last_sent_batch_id, g_short_id); } } } } - LoraPacket rx = {}; - if (lora_receive(rx, 0) && rx.protocol_version == PROTOCOL_VERSION) { - if (rx.payload_type == PayloadType::TimeSync) { - time_handle_timesync_payload(rx.payload, rx.payload_len); - } else if (rx.payload_type == PayloadType::Ack && rx.payload_len >= 6 && rx.role == DeviceRole::Receiver) { - uint16_t ack_id = read_u16_le(rx.payload); - uint16_t ack_sender = read_u16_le(&rx.payload[2]); - uint16_t ack_receiver = read_u16_le(&rx.payload[4]); - if (ack_sender == g_short_id && ack_receiver == rx.device_id_short && - g_batch_ack_pending && ack_id == g_last_sent_batch_id) { - g_last_acked_batch_id = ack_id; - serial_debug_printf("ack: ok batch_id=%u", ack_id); - finish_inflight_batch(); + bool timesync_due = (!g_batch_ack_pending && sender_timesync_window_due()); + if (timesync_due) { + LoraPacket rx = {}; + uint32_t rx_start = millis(); + uint32_t window_ms = (g_sender_timesync_mode == 2) ? SENDER_TIMESYNC_ACQUIRE_WINDOW_MS : SENDER_TIMESYNC_WINDOW_MS; + bool got = lora_receive_window(rx, window_ms); + uint32_t rx_elapsed = millis() - rx_start; + if (SERIAL_DEBUG_MODE) { + g_sender_rx_window_ms += rx_elapsed; + } + if (!got) { + sender_note_rx_reject(lora_get_last_rx_reject_reason(), "timesync"); + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("timesync: rx miss window_ms=%lu", static_cast(window_ms)); + } + } else if (rx.role != DeviceRole::Receiver) { + sender_note_rx_reject(RxRejectReason::WrongRole, "timesync"); + } else if (rx.payload_type != PayloadType::TimeSync) { + sender_note_rx_reject(RxRejectReason::WrongPayloadType, "timesync"); + } else if (time_handle_timesync_payload(rx.payload, rx.payload_len)) { + g_sender_last_timesync_rx_ms = now_ms; + if (g_sender_timesync_error) { + g_sender_timesync_error = false; + display_set_last_error(FaultType::None, 0, 0); + } + serial_debug_printf("timesync: rx ok window_ms=%lu", static_cast(window_ms)); + } else { + sender_note_rx_reject(RxRejectReason::LengthMismatch, "timesync"); + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("timesync: rx miss window_ms=%lu", static_cast(window_ms)); } } } + uint32_t timesync_age_ms = (g_sender_last_timesync_rx_ms > 0) ? (now_ms - g_sender_last_timesync_rx_ms) + : (now_ms - g_boot_ms); + if (!g_sender_timesync_error && timesync_age_ms > TIME_SYNC_ERROR_TIMEOUT_MS) { + g_sender_timesync_error = true; + display_set_last_error(FaultType::TimeSync, time_get_utc(), now_ms); + } + if (!g_batch_ack_pending) { + lora_sleep(); + } if (g_batch_ack_pending && (now_ms - g_last_batch_send_ms >= g_batch_ack_timeout_ms)) { if (g_batch_retry_count < BATCH_MAX_RETRIES) { @@ -750,6 +987,15 @@ static void sender_loop() { uint32_t next_due = next_sample_due < next_send_due ? next_sample_due : next_send_due; if (!g_batch_ack_pending && next_due > now_ms) { watchdog_kick(); + if (SERIAL_DEBUG_MODE) { + g_sender_sleep_ms += (next_due - now_ms); + if (now_ms - g_sender_power_log_ms >= 10000) { + g_sender_power_log_ms = now_ms; + serial_debug_printf("power: rx_ms=%lu sleep_ms=%lu", static_cast(g_sender_rx_window_ms), + static_cast(g_sender_sleep_ms)); + } + } + lora_sleep(); light_sleep_ms(next_due - now_ms); } } @@ -779,6 +1025,7 @@ static void receiver_loop() { data.link_valid = true; data.link_rssi_dbm = pkt.rssi_dbm; data.link_snr_db = pkt.snr_db; + sd_logger_log_sample(data, data.last_error != FaultType::None); for (uint8_t i = 0; i < NUM_SENDERS; ++i) { if (pkt.device_id_short == EXPECTED_SENDER_IDS[i]) { data.short_id = pkt.device_id_short; @@ -787,6 +1034,7 @@ static void receiver_loop() { g_sender_statuses[i].has_data = true; g_sender_faults_remote[i].meter_read_fail = data.err_meter_read; g_sender_faults_remote[i].lora_tx_fail = data.err_lora_tx; + receiver_note_timesync_drift(i, data.ts_utc); g_sender_last_error_remote[i] = data.last_error; g_sender_last_error_remote_utc[i] = time_get_utc(); g_sender_last_error_remote_ms[i] = millis(); @@ -806,15 +1054,10 @@ static void receiver_loop() { } } } else if (pkt.payload_type == PayloadType::MeterBatch) { - String json; + BatchInput batch = {}; bool decode_error = false; uint16_t batch_id = 0; - if (process_batch_packet(pkt, json, decode_error, batch_id)) { - uint32_t rx_ts_utc = time_get_utc(); - if (rx_ts_utc == 0) { - rx_ts_utc = millis() / 1000; - } - inject_batch_meta(json, pkt.rssi_dbm, pkt.snr_db, rx_ts_utc); + if (process_batch_packet(pkt, batch, decode_error, batch_id)) { MeterData samples[METER_BATCH_MAX_SAMPLES]; size_t count = 0; int8_t sender_idx = -1; @@ -827,13 +1070,63 @@ static void receiver_loop() { bool duplicate = sender_idx >= 0 && g_last_batch_id_rx[sender_idx] == batch_id; if (duplicate) { send_batch_ack(batch_id, pkt.device_id_short); - } else if (jsonToMeterBatch(json, samples, METER_BATCH_MAX_SAMPLES, count)) { + } else { + g_last_batch_id_rx[sender_idx] = batch_id; + send_batch_ack(batch_id, pkt.device_id_short); + count = batch.n; + if (count == 0 || count > METER_BATCH_MAX_SAMPLES) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + return; + } + uint16_t short_id = pkt.device_id_short; + if (short_id == 0) { + short_id = short_id_from_sender_id(batch.sender_id); + } + uint64_t span = static_cast(batch.dt_s) * static_cast(count - 1); + if (batch.t_last < span) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + return; + } + uint32_t t_first = batch.t_last - static_cast(span); + float bat_v = batch.battery_mV > 0 ? static_cast(batch.battery_mV) / 1000.0f : NAN; + for (size_t s = 0; s < count; ++s) { + MeterData &data = samples[s]; + data = {}; + data.short_id = short_id; + if (short_id != 0) { + snprintf(data.device_id, sizeof(data.device_id), "dd3-%04X", short_id); + } else { + snprintf(data.device_id, sizeof(data.device_id), "dd3-0000"); + } + data.ts_utc = t_first + static_cast(s) * batch.dt_s; + data.energy_total_kwh = static_cast(batch.energy_wh[s]) / 1000.0f; + data.phase_power_w[0] = static_cast(batch.p1_w[s]); + data.phase_power_w[1] = static_cast(batch.p2_w[s]); + data.phase_power_w[2] = static_cast(batch.p3_w[s]); + data.total_power_w = data.phase_power_w[0] + data.phase_power_w[1] + data.phase_power_w[2]; + data.battery_voltage_v = bat_v; + if (!isnan(bat_v)) { + data.battery_percent = battery_percent_from_voltage(bat_v); + } else { + data.battery_percent = 0; + } + data.valid = true; + data.link_valid = true; + data.link_rssi_dbm = pkt.rssi_dbm; + data.link_snr_db = pkt.snr_db; + data.err_meter_read = batch.err_m; + data.err_decode = batch.err_d; + data.err_lora_tx = batch.err_tx; + data.last_error = static_cast(batch.err_last); + data.rx_reject_reason = batch.err_rx_reject; + sd_logger_log_sample(data, (s + 1 == count) && data.last_error != FaultType::None); + } + if (sender_idx >= 0) { web_server_set_last_batch(static_cast(sender_idx), samples, count); for (size_t s = 0; s < count; ++s) { - samples[s].link_valid = true; - samples[s].link_rssi_dbm = pkt.rssi_dbm; - samples[s].link_snr_db = pkt.snr_db; samples[s].short_id = pkt.device_id_short; mqtt_publish_state(samples[s]); } @@ -843,6 +1136,7 @@ static void receiver_loop() { g_sender_statuses[sender_idx].has_data = true; g_sender_faults_remote[sender_idx].meter_read_fail = samples[count - 1].err_meter_read; g_sender_faults_remote[sender_idx].lora_tx_fail = samples[count - 1].err_lora_tx; + receiver_note_timesync_drift(static_cast(sender_idx), samples[count - 1].ts_utc); g_sender_last_error_remote[sender_idx] = samples[count - 1].last_error; g_sender_last_error_remote_utc[sender_idx] = time_get_utc(); g_sender_last_error_remote_ms[sender_idx] = millis(); @@ -853,12 +1147,7 @@ static void receiver_loop() { g_sender_last_error_remote[sender_idx], g_sender_last_error_remote_published[sender_idx], g_sender_last_error_remote_utc[sender_idx], g_sender_last_error_remote_ms[sender_idx]); } - g_last_batch_id_rx[sender_idx] = batch_id; - send_batch_ack(batch_id, pkt.device_id_short); } - } else { - note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); - display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); } } else if (decode_error) { note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::Decode); @@ -868,14 +1157,38 @@ static void receiver_loop() { } uint32_t interval_sec = TIME_SYNC_INTERVAL_SEC; - if (time_rtc_present() && millis() - g_boot_ms >= TIME_SYNC_FAST_WINDOW_MS) { - interval_sec = TIME_SYNC_SLOW_INTERVAL_SEC; - } - if (!g_ap_mode && millis() - g_last_timesync_ms > interval_sec * 1000UL) { - g_last_timesync_ms = millis(); - if (!time_send_timesync(g_short_id)) { - note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::LoraTx); - display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + uint32_t now_ms = millis(); + if (!g_ap_mode) { + bool burst_sent = false; + for (uint8_t i = 0; i < NUM_SENDERS; ++i) { + TimeSyncBurstState &state = g_timesync_burst[i]; + if (state.active) { + if (now_ms - state.start_ms >= TIME_SYNC_BURST_DURATION_MS) { + state.active = false; + } else if (state.last_send_ms == 0 || now_ms - state.last_send_ms >= TIME_SYNC_BURST_INTERVAL_MS) { + state.last_send_ms = now_ms; + burst_sent = true; + } + } + } + if (burst_sent) { + if (!time_send_timesync(g_short_id)) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::LoraTx); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + } + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("timesync: tx burst"); + } + g_last_timesync_ms = now_ms; + } else if (now_ms - g_last_timesync_ms > interval_sec * 1000UL) { + g_last_timesync_ms = now_ms; + if (!time_send_timesync(g_short_id)) { + note_fault(g_receiver_faults, g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms, FaultType::LoraTx); + display_set_last_error(g_receiver_last_error, g_receiver_last_error_utc, g_receiver_last_error_ms); + } + if (SERIAL_DEBUG_MODE) { + serial_debug_printf("timesync: tx normal"); + } } } diff --git a/src/meter_driver.cpp b/src/meter_driver.cpp index 507f3bc..665c4a1 100644 --- a/src/meter_driver.cpp +++ b/src/meter_driver.cpp @@ -4,7 +4,24 @@ #include #include -static constexpr uint32_t METER_READ_TIMEOUT_MS = 2000; +static constexpr uint32_t METER_FRAME_TIMEOUT_MS = 1500; +static constexpr size_t METER_FRAME_MAX = 512; + +enum class MeterRxState : uint8_t { + WaitStart = 0, + InFrame = 1 +}; + +static MeterRxState g_rx_state = MeterRxState::WaitStart; +static char g_frame_buf[METER_FRAME_MAX + 1]; +static size_t g_frame_len = 0; +static uint32_t g_last_rx_ms = 0; +static uint32_t g_bytes_rx = 0; +static uint32_t g_frames_ok = 0; +static uint32_t g_frames_parse_fail = 0; +static uint32_t g_rx_overflow = 0; +static uint32_t g_rx_timeout = 0; +static uint32_t g_last_log_ms = 0; void meter_init() { Serial2.begin(9600, SERIAL_7E1, PIN_METER_RX, -1); @@ -77,11 +94,76 @@ static bool parse_obis_ascii_unit_scale(const char *line, const char *obis, floa return false; } -static bool meter_read_ascii(MeterData &data) { - const uint32_t start_ms = millis(); - bool in_telegram = false; - bool got_any = false; +static void meter_debug_log() { + if (!SERIAL_DEBUG_MODE) { + return; + } + uint32_t now_ms = millis(); + if (now_ms - g_last_log_ms < 60000) { + return; + } + g_last_log_ms = now_ms; + Serial.printf("meter: ok=%lu parse_fail=%lu overflow=%lu timeout=%lu bytes=%lu\n", + static_cast(g_frames_ok), + static_cast(g_frames_parse_fail), + static_cast(g_rx_overflow), + static_cast(g_rx_timeout), + static_cast(g_bytes_rx)); +} +bool meter_poll_frame(const char *&frame, size_t &len) { + frame = nullptr; + len = 0; + uint32_t now_ms = millis(); + + if (g_rx_state == MeterRxState::InFrame && (now_ms - g_last_rx_ms > METER_FRAME_TIMEOUT_MS)) { + g_rx_timeout++; + g_rx_state = MeterRxState::WaitStart; + g_frame_len = 0; + } + + while (Serial2.available()) { + char c = static_cast(Serial2.read()); + g_bytes_rx++; + g_last_rx_ms = now_ms; + + if (g_rx_state == MeterRxState::WaitStart) { + if (c == '/') { + g_rx_state = MeterRxState::InFrame; + g_frame_len = 0; + g_frame_buf[g_frame_len++] = c; + } + continue; + } + + if (g_frame_len + 1 >= sizeof(g_frame_buf)) { + g_rx_overflow++; + g_rx_state = MeterRxState::WaitStart; + g_frame_len = 0; + continue; + } + + g_frame_buf[g_frame_len++] = c; + if (c == '!') { + g_frame_buf[g_frame_len] = '\0'; + frame = g_frame_buf; + len = g_frame_len; + g_rx_state = MeterRxState::WaitStart; + g_frame_len = 0; + meter_debug_log(); + return true; + } + } + + meter_debug_log(); + return false; +} + +bool meter_parse_frame(const char *frame, size_t len, MeterData &data) { + if (!frame || len == 0) { + return false; + } + bool got_any = false; bool energy_ok = false; bool total_p_ok = false; bool p1_ok = false; @@ -90,66 +172,78 @@ static bool meter_read_ascii(MeterData &data) { char line[128]; size_t line_len = 0; - while (millis() - start_ms < METER_READ_TIMEOUT_MS) { - while (Serial2.available()) { - char c = static_cast(Serial2.read()); - if (!in_telegram) { - if (c == '/') { - in_telegram = true; - line_len = 0; - line[line_len++] = c; - } - continue; - } - - if (c == '\r') { - continue; - } - if (c == '\n') { - line[line_len] = '\0'; - if (line[0] == '!') { - return got_any; - } - - float value = NAN; - if (parse_obis_ascii_value(line, "1-0:1.8.0", value)) { - parse_obis_ascii_unit_scale(line, "1-0:1.8.0", value); - data.energy_total_kwh = value; - energy_ok = true; - got_any = true; - } - if (parse_obis_ascii_value(line, "1-0:16.7.0", value)) { - data.total_power_w = value; - total_p_ok = true; - got_any = true; - } - if (parse_obis_ascii_value(line, "1-0:36.7.0", value)) { - data.phase_power_w[0] = value; - p1_ok = true; - got_any = true; - } - if (parse_obis_ascii_value(line, "1-0:56.7.0", value)) { - data.phase_power_w[1] = value; - p2_ok = true; - got_any = true; - } - if (parse_obis_ascii_value(line, "1-0:76.7.0", value)) { - data.phase_power_w[2] = value; - p3_ok = true; - got_any = true; - } - - line_len = 0; - continue; - } + for (size_t i = 0; i < len; ++i) { + char c = frame[i]; + if (c == '\r') { + continue; + } + if (c == '!') { if (line_len + 1 < sizeof(line)) { line[line_len++] = c; } + line[line_len] = '\0'; + data.valid = energy_ok || total_p_ok || p1_ok || p2_ok || p3_ok; + if (data.valid) { + g_frames_ok++; + } else { + g_frames_parse_fail++; + } + return data.valid; + } + if (c == '\n') { + line[line_len] = '\0'; + if (line[0] == '!') { + data.valid = energy_ok || total_p_ok || p1_ok || p2_ok || p3_ok; + if (data.valid) { + g_frames_ok++; + } else { + g_frames_parse_fail++; + } + return data.valid; + } + + float value = NAN; + if (parse_obis_ascii_value(line, "1-0:1.8.0", value)) { + parse_obis_ascii_unit_scale(line, "1-0:1.8.0", value); + data.energy_total_kwh = value; + energy_ok = true; + got_any = true; + } + if (parse_obis_ascii_value(line, "1-0:16.7.0", value)) { + data.total_power_w = value; + total_p_ok = true; + got_any = true; + } + if (parse_obis_ascii_value(line, "1-0:36.7.0", value)) { + data.phase_power_w[0] = value; + p1_ok = true; + got_any = true; + } + if (parse_obis_ascii_value(line, "1-0:56.7.0", value)) { + data.phase_power_w[1] = value; + p2_ok = true; + got_any = true; + } + if (parse_obis_ascii_value(line, "1-0:76.7.0", value)) { + data.phase_power_w[2] = value; + p3_ok = true; + got_any = true; + } + + line_len = 0; + continue; + } + if (line_len + 1 < sizeof(line)) { + line[line_len++] = c; } - delay(5); } - data.valid = energy_ok || total_p_ok || p1_ok || p2_ok || p3_ok; + data.valid = got_any; + if (data.valid) { + g_frames_ok++; + } else { + g_frames_parse_fail++; + } return data.valid; } @@ -161,5 +255,10 @@ bool meter_read(MeterData &data) { data.phase_power_w[2] = NAN; data.valid = false; - return meter_read_ascii(data); + const char *frame = nullptr; + size_t len = 0; + if (!meter_poll_frame(frame, len)) { + return false; + } + return meter_parse_frame(frame, len, data); } diff --git a/src/mqtt_client.cpp b/src/mqtt_client.cpp index d08c616..2c6008f 100644 --- a/src/mqtt_client.cpp +++ b/src/mqtt_client.cpp @@ -10,6 +10,21 @@ static PubSubClient mqtt_client(wifi_client); static WifiMqttConfig g_cfg; static String g_client_id; +static const char *fault_text(FaultType fault) { + switch (fault) { + case FaultType::MeterRead: + return "meter"; + case FaultType::Decode: + return "decode"; + case FaultType::LoraTx: + return "loratx"; + case FaultType::TimeSync: + return "timesync"; + default: + return "none"; + } +} + void mqtt_init(const WifiMqttConfig &config, const char *device_id) { g_cfg = config; mqtt_client.setServer(config.mqtt_host.c_str(), config.mqtt_port); @@ -66,10 +81,9 @@ bool mqtt_publish_faults(const char *device_id, const FaultCounters &counters, F doc["err_m"] = counters.meter_read_fail; doc["err_d"] = counters.decode_fail; doc["err_tx"] = counters.lora_tx_fail; - if (last_error != FaultType::None) { - doc["err_last"] = static_cast(last_error); - doc["err_last_age"] = last_error_age_sec; - } + doc["err_last"] = static_cast(last_error); + doc["err_last_text"] = fault_text(last_error); + doc["err_last_age"] = last_error != FaultType::None ? last_error_age_sec : 0; String payload; size_t len = serializeJson(doc, payload); @@ -138,6 +152,8 @@ bool mqtt_publish_discovery(const char *device_id) { ok = ok && publish_discovery_sensor(device_id, "err_m", "Meter Read Errors", "count", "", faults_topic.c_str(), "{{ value_json.err_m }}"); ok = ok && publish_discovery_sensor(device_id, "err_d", "Decode Errors", "count", "", faults_topic.c_str(), "{{ value_json.err_d }}"); ok = ok && publish_discovery_sensor(device_id, "err_tx", "LoRa TX Errors", "count", "", faults_topic.c_str(), "{{ value_json.err_tx }}"); + ok = ok && publish_discovery_sensor(device_id, "err_last", "Last Error Code", "", "", faults_topic.c_str(), "{{ value_json.err_last }}"); + ok = ok && publish_discovery_sensor(device_id, "err_last_text", "Last Error", "", "", faults_topic.c_str(), "{{ value_json.err_last_text }}"); ok = ok && publish_discovery_sensor(device_id, "err_last_age", "Last Error Age", "s", "", faults_topic.c_str(), "{{ value_json.err_last_age }}"); return ok; } diff --git a/src/payload_codec.cpp b/src/payload_codec.cpp new file mode 100644 index 0000000..6403057 --- /dev/null +++ b/src/payload_codec.cpp @@ -0,0 +1,342 @@ +#include "payload_codec.h" +#include + +static constexpr uint16_t kMagic = 0xDDB3; +static constexpr uint8_t kSchema = 2; +static constexpr uint8_t kFlags = 0x01; +static constexpr size_t kMaxSamples = 30; + +static void write_u16_le(uint8_t *dst, uint16_t value) { + dst[0] = static_cast(value & 0xFF); + dst[1] = static_cast((value >> 8) & 0xFF); +} + +static void write_u32_le(uint8_t *dst, uint32_t value) { + dst[0] = static_cast(value & 0xFF); + dst[1] = static_cast((value >> 8) & 0xFF); + dst[2] = static_cast((value >> 16) & 0xFF); + dst[3] = static_cast((value >> 24) & 0xFF); +} + +static uint16_t read_u16_le(const uint8_t *src) { + return static_cast(src[0]) | (static_cast(src[1]) << 8); +} + +static uint32_t read_u32_le(const uint8_t *src) { + return static_cast(src[0]) | + (static_cast(src[1]) << 8) | + (static_cast(src[2]) << 16) | + (static_cast(src[3]) << 24); +} + +size_t uleb128_encode(uint32_t v, uint8_t *out, size_t cap) { + size_t i = 0; + do { + if (i >= cap) { + return 0; + } + uint8_t byte = static_cast(v & 0x7F); + v >>= 7; + if (v != 0) { + byte |= 0x80; + } + out[i++] = byte; + } while (v != 0); + return i; +} + +bool uleb128_decode(const uint8_t *in, size_t len, size_t *pos, uint32_t *v) { + if (!in || !pos || !v) { + return false; + } + uint32_t result = 0; + uint8_t shift = 0; + size_t p = *pos; + for (uint8_t i = 0; i < 5; ++i) { + if (p >= len) { + return false; + } + uint8_t byte = in[p++]; + if (i == 4 && (byte & 0xF0) != 0) { + return false; + } + result |= static_cast(byte & 0x7F) << shift; + if ((byte & 0x80) == 0) { + *pos = p; + *v = result; + return true; + } + shift = static_cast(shift + 7); + } + return false; +} + +uint32_t zigzag32(int32_t x) { + return (static_cast(x) << 1) ^ static_cast(x >> 31); +} + +int32_t unzigzag32(uint32_t u) { + return static_cast((u >> 1) ^ (static_cast(-static_cast(u & 1)))); +} + +size_t svarint_encode(int32_t x, uint8_t *out, size_t cap) { + uint32_t zz = zigzag32(x); + return uleb128_encode(zz, out, cap); +} + +bool svarint_decode(const uint8_t *in, size_t len, size_t *pos, int32_t *x) { + uint32_t u = 0; + if (!uleb128_decode(in, len, pos, &u)) { + return false; + } + *x = unzigzag32(u); + return true; +} + +static bool ensure_capacity(size_t needed, size_t cap, size_t pos) { + return pos + needed <= cap; +} + +bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *out_len) { + if (!out || !out_len) { + return false; + } + if (in.n == 0 || in.n > kMaxSamples) { + return false; + } + if (in.dt_s == 0) { + return false; + } + size_t pos = 0; + if (!ensure_capacity(21, out_cap, pos)) { + return false; + } + write_u16_le(&out[pos], kMagic); + pos += 2; + out[pos++] = kSchema; + out[pos++] = kFlags; + write_u16_le(&out[pos], in.sender_id); + pos += 2; + write_u16_le(&out[pos], in.batch_id); + pos += 2; + write_u32_le(&out[pos], in.t_last); + pos += 4; + out[pos++] = in.dt_s; + out[pos++] = in.n; + write_u16_le(&out[pos], in.battery_mV); + pos += 2; + out[pos++] = in.err_m; + out[pos++] = in.err_d; + out[pos++] = in.err_tx; + out[pos++] = in.err_last; + out[pos++] = in.err_rx_reject; + + if (!ensure_capacity(4, out_cap, pos)) { + return false; + } + write_u32_le(&out[pos], in.energy_wh[0]); + pos += 4; + for (uint8_t i = 1; i < in.n; ++i) { + if (in.energy_wh[i] < in.energy_wh[i - 1]) { + return false; + } + uint32_t delta = in.energy_wh[i] - in.energy_wh[i - 1]; + size_t wrote = uleb128_encode(delta, &out[pos], out_cap - pos); + if (wrote == 0) { + return false; + } + pos += wrote; + } + + auto encode_phase = [&](const int16_t *phase) -> bool { + if (!ensure_capacity(2, out_cap, pos)) { + return false; + } + write_u16_le(&out[pos], static_cast(phase[0])); + pos += 2; + for (uint8_t i = 1; i < in.n; ++i) { + int32_t delta = static_cast(phase[i]) - static_cast(phase[i - 1]); + size_t wrote = svarint_encode(delta, &out[pos], out_cap - pos); + if (wrote == 0) { + return false; + } + pos += wrote; + } + return true; + }; + + if (!encode_phase(in.p1_w)) { + return false; + } + if (!encode_phase(in.p2_w)) { + return false; + } + if (!encode_phase(in.p3_w)) { + return false; + } + + *out_len = pos; + return true; +} + +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out) { + if (!buf || !out) { + return false; + } + size_t pos = 0; + if (len < 21) { + return false; + } + uint16_t magic = read_u16_le(&buf[pos]); + pos += 2; + uint8_t schema = buf[pos++]; + uint8_t flags = buf[pos++]; + if (magic != kMagic || schema != kSchema || (flags & 0x01) == 0) { + return false; + } + out->sender_id = read_u16_le(&buf[pos]); + pos += 2; + out->batch_id = read_u16_le(&buf[pos]); + pos += 2; + out->t_last = read_u32_le(&buf[pos]); + pos += 4; + out->dt_s = buf[pos++]; + out->n = buf[pos++]; + out->battery_mV = read_u16_le(&buf[pos]); + pos += 2; + out->err_m = buf[pos++]; + out->err_d = buf[pos++]; + out->err_tx = buf[pos++]; + out->err_last = buf[pos++]; + out->err_rx_reject = buf[pos++]; + + if (out->n == 0 || out->n > kMaxSamples || out->dt_s == 0) { + return false; + } + if (pos + 4 > len) { + return false; + } + out->energy_wh[0] = read_u32_le(&buf[pos]); + pos += 4; + for (uint8_t i = 1; i < out->n; ++i) { + uint32_t delta = 0; + if (!uleb128_decode(buf, len, &pos, &delta)) { + return false; + } + uint64_t sum = static_cast(out->energy_wh[i - 1]) + static_cast(delta); + if (sum > UINT32_MAX) { + return false; + } + out->energy_wh[i] = static_cast(sum); + } + + auto decode_phase = [&](int16_t *phase) -> bool { + if (pos + 2 > len) { + return false; + } + phase[0] = static_cast(read_u16_le(&buf[pos])); + pos += 2; + int32_t prev = static_cast(phase[0]); + for (uint8_t i = 1; i < out->n; ++i) { + int32_t delta = 0; + if (!svarint_decode(buf, len, &pos, &delta)) { + return false; + } + int32_t value = prev + delta; + if (value < INT16_MIN || value > INT16_MAX) { + return false; + } + phase[i] = static_cast(value); + prev = value; + } + return true; + }; + + if (!decode_phase(out->p1_w)) { + return false; + } + if (!decode_phase(out->p2_w)) { + return false; + } + if (!decode_phase(out->p3_w)) { + return false; + } + + for (uint8_t i = out->n; i < kMaxSamples; ++i) { + out->energy_wh[i] = 0; + out->p1_w[i] = 0; + out->p2_w[i] = 0; + out->p3_w[i] = 0; + } + + return pos == len; +} + +#ifdef PAYLOAD_CODEC_TEST +bool payload_codec_self_test() { + BatchInput in = {}; + in.sender_id = 1; + in.batch_id = 42; + in.t_last = 1700000000; + in.dt_s = 1; + in.n = 5; + in.battery_mV = 3750; + in.err_m = 2; + in.err_d = 1; + in.err_tx = 3; + in.err_last = 2; + in.err_rx_reject = 1; + in.energy_wh[0] = 100000; + in.energy_wh[1] = 100001; + in.energy_wh[2] = 100050; + in.energy_wh[3] = 100050; + in.energy_wh[4] = 100200; + in.p1_w[0] = -120; + in.p1_w[1] = -90; + in.p1_w[2] = 1910; + in.p1_w[3] = -90; + in.p1_w[4] = 500; + in.p2_w[0] = 50; + in.p2_w[1] = -1950; + in.p2_w[2] = 60; + in.p2_w[3] = 2060; + in.p2_w[4] = -10; + in.p3_w[0] = 0; + in.p3_w[1] = 10; + in.p3_w[2] = -1990; + in.p3_w[3] = 10; + in.p3_w[4] = 20; + + uint8_t buf[256]; + size_t len = 0; + if (!encode_batch(in, buf, sizeof(buf), &len)) { + Serial.println("payload_codec_self_test: encode failed"); + return false; + } + + BatchInput out = {}; + if (!decode_batch(buf, len, &out)) { + Serial.println("payload_codec_self_test: decode failed"); + return false; + } + + if (out.sender_id != in.sender_id || out.batch_id != in.batch_id || out.t_last != in.t_last || + out.dt_s != in.dt_s || out.n != in.n || out.battery_mV != in.battery_mV || + out.err_m != in.err_m || out.err_d != in.err_d || out.err_tx != in.err_tx || out.err_last != in.err_last || + out.err_rx_reject != in.err_rx_reject) { + Serial.println("payload_codec_self_test: header mismatch"); + return false; + } + + for (uint8_t i = 0; i < in.n; ++i) { + if (out.energy_wh[i] != in.energy_wh[i] || out.p1_w[i] != in.p1_w[i] || out.p2_w[i] != in.p2_w[i] || + out.p3_w[i] != in.p3_w[i]) { + Serial.println("payload_codec_self_test: sample mismatch"); + return false; + } + } + + Serial.printf("payload_codec_self_test: ok len=%u\n", static_cast(len)); + return true; +} +#endif diff --git a/src/payload_codec.h b/src/payload_codec.h new file mode 100644 index 0000000..8887bd1 --- /dev/null +++ b/src/payload_codec.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +struct BatchInput { + uint16_t sender_id; + uint16_t batch_id; + uint32_t t_last; + uint8_t dt_s; + uint8_t n; + uint16_t battery_mV; + uint8_t err_m; + uint8_t err_d; + uint8_t err_tx; + uint8_t err_last; + uint8_t err_rx_reject; + uint32_t energy_wh[30]; + int16_t p1_w[30]; + int16_t p2_w[30]; + int16_t p3_w[30]; +}; + +bool encode_batch(const BatchInput &in, uint8_t *out, size_t out_cap, size_t *out_len); +bool decode_batch(const uint8_t *buf, size_t len, BatchInput *out); + +size_t uleb128_encode(uint32_t v, uint8_t *out, size_t cap); +bool uleb128_decode(const uint8_t *in, size_t len, size_t *pos, uint32_t *v); + +uint32_t zigzag32(int32_t x); +int32_t unzigzag32(uint32_t u); + +size_t svarint_encode(int32_t x, uint8_t *out, size_t cap); +bool svarint_decode(const uint8_t *in, size_t len, size_t *pos, int32_t *x); + +#ifdef PAYLOAD_CODEC_TEST +bool payload_codec_self_test(); +#endif diff --git a/src/power_manager.cpp b/src/power_manager.cpp index 50b74e0..c439bf6 100644 --- a/src/power_manager.cpp +++ b/src/power_manager.cpp @@ -6,11 +6,13 @@ #include static constexpr float BATTERY_DIVIDER = 2.0f; -static constexpr float BATTERY_CAL = 1.0f; static constexpr float ADC_REF_V = 3.3f; void power_sender_init() { + setCpuFrequencyMhz(80); + WiFi.mode(WIFI_OFF); esp_wifi_stop(); + esp_wifi_deinit(); btStop(); analogReadResolution(12); pinMode(PIN_BAT_ADC, INPUT); @@ -22,29 +24,89 @@ void power_receiver_init() { pinMode(PIN_BAT_ADC, INPUT); } -void read_battery(MeterData &data) { - const int samples = 8; - uint32_t sum = 0; - for (int i = 0; i < samples; ++i) { - sum += analogRead(PIN_BAT_ADC); - delay(5); +void power_configure_unused_pins_sender() { + // Board-specific: only touch pins that are known unused and safe on TTGO LoRa32 v1.6.1 + const uint8_t pins[] = {32, 33}; + for (uint8_t pin : pins) { + pinMode(pin, INPUT_PULLDOWN); } - float avg = static_cast(sum) / samples; +} + +void read_battery(MeterData &data) { + uint32_t sum = 0; + uint16_t samples[5] = {}; + for (uint8_t i = 0; i < 5; ++i) { + samples[i] = analogRead(PIN_BAT_ADC); + sum += samples[i]; + } + float avg = static_cast(sum) / 5.0f; float v = (avg / 4095.0f) * ADC_REF_V * BATTERY_DIVIDER * BATTERY_CAL; + if (SERIAL_DEBUG_MODE) { + Serial.printf("bat_adc: %u %u %u %u %u avg=%.1f v=%.3f\n", + samples[0], samples[1], samples[2], samples[3], samples[4], + static_cast(avg), static_cast(v)); + } data.battery_voltage_v = v; data.battery_percent = battery_percent_from_voltage(v); } uint8_t battery_percent_from_voltage(float voltage_v) { - float pct = (voltage_v - 3.0f) / (4.2f - 3.0f) * 100.0f; - if (pct < 0.0f) { - pct = 0.0f; + if (isnan(voltage_v)) { + return 0; } - if (pct > 100.0f) { - pct = 100.0f; + struct LutPoint { + float v; + uint8_t pct; + }; + static const LutPoint kCurve[] = { + {4.20f, 100}, + {4.15f, 95}, + {4.11f, 90}, + {4.08f, 85}, + {4.02f, 80}, + {3.98f, 75}, + {3.95f, 70}, + {3.91f, 60}, + {3.87f, 50}, + {3.85f, 45}, + {3.84f, 40}, + {3.82f, 35}, + {3.80f, 30}, + {3.77f, 25}, + {3.75f, 20}, + {3.73f, 15}, + {3.70f, 10}, + {3.65f, 5}, + {3.60f, 2}, + {2.90f, 0}, + }; + if (voltage_v >= kCurve[0].v) { + return kCurve[0].pct; } - return static_cast(pct + 0.5f); + if (voltage_v <= kCurve[sizeof(kCurve) / sizeof(kCurve[0]) - 1].v) { + return 0; + } + for (size_t i = 0; i + 1 < sizeof(kCurve) / sizeof(kCurve[0]); ++i) { + const LutPoint &hi = kCurve[i]; + const LutPoint &lo = kCurve[i + 1]; + if (voltage_v <= hi.v && voltage_v >= lo.v) { + float span = hi.v - lo.v; + if (span <= 0.0f) { + return lo.pct; + } + float t = (voltage_v - lo.v) / span; + float pct = lo.pct + t * (hi.pct - lo.pct); + if (pct < 0.0f) { + pct = 0.0f; + } + if (pct > 100.0f) { + pct = 100.0f; + } + return static_cast(pct + 0.5f); + } + } + return 0; } void light_sleep_ms(uint32_t ms) { diff --git a/src/rtc_ds3231.cpp b/src/rtc_ds3231.cpp index e68fcfc..f4103d7 100644 --- a/src/rtc_ds3231.cpp +++ b/src/rtc_ds3231.cpp @@ -1,6 +1,7 @@ #include "rtc_ds3231.h" #include "config.h" #include +#include #include static constexpr uint8_t DS3231_ADDR = 0x68; @@ -17,12 +18,14 @@ static time_t timegm_fallback(struct tm *tm_utc) { if (!tm_utc) { return static_cast(-1); } - char *old_tz = getenv("TZ"); + const char *old_tz = getenv("TZ"); + // getenv() may return a pointer into mutable storage that becomes invalid after setenv(). + std::string old_tz_copy = old_tz ? old_tz : ""; setenv("TZ", "UTC0", 1); tzset(); time_t t = mktime(tm_utc); - if (old_tz) { - setenv("TZ", old_tz, 1); + if (!old_tz_copy.empty()) { + setenv("TZ", old_tz_copy.c_str(), 1); } else { unsetenv("TZ"); } diff --git a/src/sd_logger.cpp b/src/sd_logger.cpp new file mode 100644 index 0000000..915744a --- /dev/null +++ b/src/sd_logger.cpp @@ -0,0 +1,130 @@ +#include "sd_logger.h" +#include "config.h" +#include +#include +#include + +static bool g_sd_ready = false; +static SPIClass *g_sd_spi = nullptr; + +static const char *fault_text(FaultType fault) { + switch (fault) { + case FaultType::MeterRead: + return "meter"; + case FaultType::Decode: + return "decode"; + case FaultType::LoraTx: + return "loratx"; + case FaultType::TimeSync: + return "timesync"; + default: + return ""; + } +} + +static bool ensure_dir(const String &path) { + if (SD.exists(path)) { + return true; + } + return SD.mkdir(path); +} + +static String format_date_utc(uint32_t ts_utc) { + time_t t = static_cast(ts_utc); + struct tm tm_utc; + gmtime_r(&t, &tm_utc); + char buf[16]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02d", + tm_utc.tm_year + 1900, + tm_utc.tm_mon + 1, + tm_utc.tm_mday); + return String(buf); +} + +void sd_logger_init() { + if (!ENABLE_SD_LOGGING) { + g_sd_ready = false; + return; + } + if (!g_sd_spi) { + g_sd_spi = new SPIClass(HSPI); + } + g_sd_spi->begin(PIN_SD_SCK, PIN_SD_MISO, PIN_SD_MOSI, PIN_SD_CS); + g_sd_ready = SD.begin(PIN_SD_CS, *g_sd_spi); + if (SERIAL_DEBUG_MODE) { + if (g_sd_ready) { + uint8_t type = SD.cardType(); + uint64_t size = SD.cardSize(); + Serial.printf("sd: ok type=%u size=%llu\n", static_cast(type), static_cast(size)); + } else { + Serial.println("sd: init failed"); + } + } +} + +bool sd_logger_is_ready() { + return g_sd_ready; +} + +void sd_logger_log_sample(const MeterData &data, bool include_error_text) { + if (!g_sd_ready || data.ts_utc == 0) { + return; + } + + String root_dir = "/dd3"; + if (!ensure_dir(root_dir)) { + return; + } + + String sender_dir = root_dir + "/" + String(data.device_id); + if (!ensure_dir(sender_dir)) { + return; + } + + String filename = sender_dir + "/" + format_date_utc(data.ts_utc) + ".csv"; + bool new_file = !SD.exists(filename); + File f = SD.open(filename, FILE_APPEND); + if (!f) { + return; + } + + if (new_file) { + f.println("ts_utc,p_w,p1_w,p2_w,p3_w,e_kwh,bat_v,bat_pct,rssi,snr,err_m,err_d,err_tx,err_last"); + } + + f.print(data.ts_utc); + f.print(','); + f.print(data.total_power_w, 1); + f.print(','); + f.print(data.phase_power_w[0], 1); + f.print(','); + f.print(data.phase_power_w[1], 1); + f.print(','); + f.print(data.phase_power_w[2], 1); + f.print(','); + f.print(data.energy_total_kwh, 3); + f.print(','); + f.print(data.battery_voltage_v, 2); + f.print(','); + f.print(data.battery_percent); + f.print(','); + f.print(data.link_rssi_dbm); + f.print(','); + if (isnan(data.link_snr_db)) { + f.print(""); + } else { + f.print(data.link_snr_db, 1); + } + f.print(','); + f.print(data.err_meter_read); + f.print(','); + f.print(data.err_decode); + f.print(','); + f.print(data.err_lora_tx); + f.print(','); + if (include_error_text && data.last_error != FaultType::None) { + f.print(fault_text(data.last_error)); + } + f.println(); + f.close(); +} diff --git a/src/sd_logger.h b/src/sd_logger.h new file mode 100644 index 0000000..60b1611 --- /dev/null +++ b/src/sd_logger.h @@ -0,0 +1,7 @@ +#pragma once + +#include "data_model.h" + +void sd_logger_init(); +bool sd_logger_is_ready(); +void sd_logger_log_sample(const MeterData &data, bool include_error_text); diff --git a/src/time_manager.cpp b/src/time_manager.cpp index 9d46f0b..91bcd6a 100644 --- a/src/time_manager.cpp +++ b/src/time_manager.cpp @@ -8,6 +8,8 @@ static bool g_time_synced = false; static bool g_tz_set = false; static bool g_rtc_present = false; static uint32_t g_last_sync_utc = 0; +static constexpr uint32_t kMinValidEpoch = 1672531200UL; // 2023-01-01 +static constexpr uint32_t kMaxValidEpoch = 4102444800UL; // 2100-01-01 static void note_last_sync(uint32_t epoch) { if (epoch == 0) { @@ -83,7 +85,11 @@ bool time_send_timesync(uint16_t device_id_short) { pkt.payload_type = PayloadType::TimeSync; pkt.payload_len = compressed_len; memcpy(pkt.payload, compressed, compressed_len); - return lora_send(pkt); + bool ok = lora_send(pkt); + if (ok) { + lora_receive_continuous(); + } + return ok; } bool time_handle_timesync_payload(const uint8_t *payload, size_t len) { @@ -138,6 +144,16 @@ bool time_try_load_from_rtc() { } uint32_t epoch = 0; if (!rtc_ds3231_read_epoch(epoch) || epoch == 0) { + if (SERIAL_DEBUG_MODE) { + Serial.println("rtc: read failed"); + } + return false; + } + bool valid = epoch >= kMinValidEpoch && epoch <= kMaxValidEpoch; + if (SERIAL_DEBUG_MODE) { + Serial.printf("rtc: epoch=%lu %s\n", static_cast(epoch), valid ? "accepted" : "rejected"); + } + if (!valid) { return false; } time_set_utc(epoch); diff --git a/src/web_server.cpp b/src/web_server.cpp index e56bd6b..ae40b03 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -2,21 +2,96 @@ #include #include "wifi_manager.h" #include "config.h" +#include "sd_logger.h" +#include "time_manager.h" +#include "html_util.h" +#include +#include +#include +#include +#include static WebServer server(80); static const SenderStatus *g_statuses = nullptr; static uint8_t g_status_count = 0; static WifiMqttConfig g_config; static bool g_is_ap = false; +static String g_web_user; +static String g_web_pass; static const FaultCounters *g_sender_faults = nullptr; static const FaultType *g_sender_last_errors = nullptr; static MeterData g_last_batch[NUM_SENDERS][METER_BATCH_MAX_SAMPLES]; static uint8_t g_last_batch_count[NUM_SENDERS] = {}; +struct HistoryBin { + uint32_t ts; + float value; + uint32_t count; +}; + +enum class HistoryMode : uint8_t { + Avg = 0, + Max = 1 +}; + +struct HistoryJob { + bool active; + bool done; + bool error; + String error_msg; + String device_id; + HistoryMode mode; + uint32_t start_ts; + uint32_t end_ts; + uint32_t res_sec; + uint32_t bins_count; + uint32_t bins_filled; + uint16_t day_index; + File file; + HistoryBin *bins; +}; + +static HistoryJob g_history = {}; +static constexpr size_t SD_LIST_MAX_FILES = 200; +static constexpr size_t SD_DOWNLOAD_MAX_PATH = 160; + +static bool auth_required() { + return g_is_ap ? WEB_AUTH_REQUIRE_AP : WEB_AUTH_REQUIRE_STA; +} + +static const char *fault_text(FaultType fault) { + switch (fault) { + case FaultType::MeterRead: + return "meter"; + case FaultType::Decode: + return "decode"; + case FaultType::LoraTx: + return "loratx"; + case FaultType::TimeSync: + return "timesync"; + default: + return "none"; + } +} + +static bool ensure_auth() { + if (!auth_required()) { + return true; + } + const char *user = g_web_user.c_str(); + const char *pass = g_web_pass.c_str(); + if (server.authenticate(user, pass)) { + return true; + } + server.requestAuthentication(BASIC_AUTH, "DD3", "Authentication required"); + return false; +} + static String html_header(const String &title) { + String safe_title = html_escape(title); String h = ""; - h += "" + title + ""; - h += "

" + title + "

"; + h += "" + safe_title + ""; + h += "

" + safe_title + "

"; return h; } @@ -37,9 +112,223 @@ static String format_faults(uint8_t idx) { s += String(g_sender_faults[idx].lora_tx_fail); s += " last:"; s += String(static_cast(g_sender_last_errors[idx])); + s += " (" + String(fault_text(g_sender_last_errors[idx])) + ")"; return s; } +static bool sanitize_sd_download_path(String &path, String &error) { + path.trim(); + if (path.length() == 0) { + error = "empty"; + return false; + } + if (path.startsWith("dd3/")) { + path = "/" + path; + } + if (path.length() > SD_DOWNLOAD_MAX_PATH) { + error = "too_long"; + return false; + } + if (!path.startsWith("/dd3/")) { + error = "prefix"; + return false; + } + if (path.indexOf("..") >= 0) { + error = "dotdot"; + return false; + } + if (path.indexOf('\\') >= 0) { + error = "backslash"; + return false; + } + if (path.indexOf("//") >= 0) { + error = "repeated_slash"; + return false; + } + return true; +} + +static bool checkbox_checked(const char *name) { + if (!server.hasArg(name)) { + return false; + } + String val = server.arg(name); + return val == "on" || val == "true" || val == "1"; +} + +static bool sanitize_history_device_id(const String &input, String &out_device_id) { + if (sanitize_device_id(input, out_device_id)) { + return true; + } + if (g_statuses) { + for (uint8_t i = 0; i < g_status_count; ++i) { + String known = g_statuses[i].last_data.device_id; + if (input.equalsIgnoreCase(known) && sanitize_device_id(known, out_device_id)) { + return true; + } + } + } + return false; +} + +static String sanitize_download_filename(const String &input, bool &clean) { + String out; + out.reserve(input.length()); + clean = true; + for (size_t i = 0; i < input.length(); ++i) { + unsigned char c = static_cast(input[i]); + if (c < 32 || c == 127 || c == '"' || c == '\\' || c == '/') { + out += '_'; + clean = false; + continue; + } + out += static_cast(c); + } + out.trim(); + if (out.length() == 0) { + out = "download.bin"; + clean = false; + } + return out; +} + +static void history_reset() { + if (g_history.file) { + g_history.file.close(); + } + if (g_history.bins) { + delete[] g_history.bins; + } + g_history = {}; +} + +static String history_date_from_epoch(uint32_t ts_utc) { + time_t t = static_cast(ts_utc); + struct tm tm_utc; + gmtime_r(&t, &tm_utc); + char buf[16]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02d", tm_utc.tm_year + 1900, tm_utc.tm_mon + 1, tm_utc.tm_mday); + return String(buf); +} + +static bool history_open_next_file() { + if (!g_history.active || g_history.done || g_history.error) { + return false; + } + if (g_history.file) { + g_history.file.close(); + } + uint32_t day_ts = g_history.start_ts + static_cast(g_history.day_index) * 86400UL; + if (day_ts > g_history.end_ts) { + g_history.done = true; + return false; + } + String path = String("/dd3/") + g_history.device_id + "/" + history_date_from_epoch(day_ts) + ".csv"; + g_history.file = SD.open(path.c_str(), FILE_READ); + g_history.day_index++; + return true; +} + +static bool history_parse_line(const char *line, uint32_t &ts_out, float &p_out) { + if (!line || line[0] < '0' || line[0] > '9') { + return false; + } + const char *comma = strchr(line, ','); + if (!comma) { + return false; + } + char ts_buf[16]; + size_t ts_len = static_cast(comma - line); + if (ts_len >= sizeof(ts_buf)) { + return false; + } + memcpy(ts_buf, line, ts_len); + ts_buf[ts_len] = '\0'; + char *end = nullptr; + uint32_t ts = static_cast(strtoul(ts_buf, &end, 10)); + if (end == ts_buf) { + return false; + } + const char *p_start = comma + 1; + const char *p_end = strchr(p_start, ','); + char p_buf[16]; + size_t p_len = p_end ? static_cast(p_end - p_start) : strlen(p_start); + if (p_len >= sizeof(p_buf)) { + return false; + } + memcpy(p_buf, p_start, p_len); + p_buf[p_len] = '\0'; + char *endp = nullptr; + float p = strtof(p_buf, &endp); + if (endp == p_buf) { + return false; + } + ts_out = ts; + p_out = p; + return true; +} + +static void history_tick() { + if (!g_history.active || g_history.done || g_history.error) { + return; + } + if (!sd_logger_is_ready()) { + g_history.error = true; + g_history.error_msg = "sd_not_ready"; + return; + } + + uint32_t start_ms = millis(); + while (millis() - start_ms < SD_HISTORY_TIME_BUDGET_MS) { + if (!g_history.file) { + if (!history_open_next_file()) { + if (g_history.done) { + g_history.active = false; + } + return; + } + } + if (!g_history.file.available()) { + g_history.file.close(); + continue; + } + + char line[160]; + size_t n = g_history.file.readBytesUntil('\n', line, sizeof(line) - 1); + line[n] = '\0'; + if (n == 0) { + continue; + } + uint32_t ts = 0; + float p = 0.0f; + if (!history_parse_line(line, ts, p)) { + continue; + } + if (ts < g_history.start_ts || ts > g_history.end_ts) { + continue; + } + uint32_t idx = (ts - g_history.start_ts) / g_history.res_sec; + if (idx >= g_history.bins_count) { + continue; + } + HistoryBin &bin = g_history.bins[idx]; + if (bin.count == 0) { + bin.ts = g_history.start_ts + idx * g_history.res_sec; + bin.value = p; + bin.count = 1; + g_history.bins_filled++; + } else if (g_history.mode == HistoryMode::Avg) { + bin.value += p; + bin.count++; + } else { + if (p > bin.value) { + bin.value = p; + } + bin.count++; + } + } +} + static String render_sender_block(const SenderStatus &status) { String s; s += "
"; @@ -52,9 +341,19 @@ static String render_sender_block(const SenderStatus &status) { } } } - s += "" + String(status.last_data.device_id) + ""; + String device_id = status.last_data.device_id; + String device_id_safe = html_escape(device_id); + String device_id_url = url_encode_component(device_id); + s += "" + device_id_safe + ""; if (status.has_data && status.last_data.link_valid) { - s += " R:" + String(status.last_data.link_rssi_dbm) + " S:" + String(status.last_data.link_snr_db, 1); + s += " RSSI:" + String(status.last_data.link_rssi_dbm) + " SNR:" + String(status.last_data.link_snr_db, 1); + } + if (status.has_data) { + s += " err_tx:" + String(status.last_data.err_lora_tx); + s += " err_last:" + String(static_cast(status.last_data.last_error)); + s += " (" + String(fault_text(status.last_data.last_error)) + ")"; + s += " rx_reject:" + String(status.last_data.rx_reject_reason); + s += " (" + String(rx_reject_reason_text(static_cast(status.last_data.rx_reject_reason))) + ")"; } s += format_faults(idx); s += "
"; @@ -63,13 +362,55 @@ static String render_sender_block(const SenderStatus &status) { } else { s += "Energy: " + String(status.last_data.energy_total_kwh, 3) + " kWh
"; s += "Power: " + String(status.last_data.total_power_w, 1) + " W
"; - s += "Battery: " + String(status.last_data.battery_voltage_v, 2) + " V (" + String(status.last_data.battery_percent) + ")"; + s += "P1/P2/P3: " + String(status.last_data.phase_power_w[0], 1) + " / " + String(status.last_data.phase_power_w[1], 1) + + " / " + String(status.last_data.phase_power_w[2], 1) + " W
"; + s += "Battery: " + String(status.last_data.battery_percent) + "% (" + String(status.last_data.battery_voltage_v, 2) + " V)"; } s += "
"; return s; } +static void append_sd_listing(String &html, const String &dir_path, uint8_t depth, size_t &count) { + if (count >= SD_LIST_MAX_FILES || depth > 4) { + return; + } + File dir = SD.open(dir_path.c_str()); + if (!dir || !dir.isDirectory()) { + return; + } + File entry = dir.openNextFile(); + while (entry && count < SD_LIST_MAX_FILES) { + String name = entry.name(); + String full_path = name; + if (!full_path.startsWith(dir_path)) { + if (!dir_path.endsWith("/")) { + full_path = dir_path + "/" + name; + } else { + full_path = dir_path + name; + } + } + if (entry.isDirectory()) { + html += "
  • " + html_escape(full_path) + "/
    • "; + append_sd_listing(html, full_path, depth + 1, count); + } else { + String href = full_path; + if (!href.startsWith("/")) { + href = "/" + href; + } + String href_enc = url_encode_component(href); + html += "
  • " + html_escape(full_path) + ""; + html += " (" + String(entry.size()) + " bytes)
    • "; + count++; + } + entry = dir.openNextFile(); + } + dir.close(); +} + static void handle_root() { + if (!ensure_auth()) { + return; + } String html = html_header("DD3 Bridge Status"); html += g_is_ap ? "

    Mode: AP

    " : "

    Mode: STA

    "; @@ -79,6 +420,18 @@ static void handle_root() { } } + if (sd_logger_is_ready()) { + html += "

    SD Files

      "; + size_t count = 0; + append_sd_listing(html, "/dd3", 0, count); + if (count >= SD_LIST_MAX_FILES) { + html += "
    • Listing truncated...
      • "; + } + html += "
    "; + } else { + html += "

    SD: not ready

    "; + } + html += "

    Configure WiFi/MQTT/NTP

    "; html += "

    Manual

    "; html += html_footer(); @@ -86,16 +439,26 @@ static void handle_root() { } static void handle_wifi_get() { + if (!ensure_auth()) { + return; + } String html = html_header("WiFi/MQTT Config"); html += "
    "; - html += "SSID:
    "; - html += "Password:
    "; - html += "MQTT Host:
    "; + html += "SSID:
    "; + html += "Password: "; + html += "
    "; + html += "MQTT Host:
    "; html += "MQTT Port:
    "; - html += "MQTT User:
    "; - html += "MQTT Pass:
    "; - html += "NTP Server 1:
    "; - html += "NTP Server 2:
    "; + html += "MQTT User:
    "; + html += "MQTT Pass: "; + html += "
    "; + html += "NTP Server 1:
    "; + html += "NTP Server 2:
    "; + html += "
    "; + html += "Web UI User:
    "; + html += "Web UI Pass: "; + html += "
    "; + html += "
    Leaving password blank keeps the existing one.
    "; html += ""; html += "
    "; html += html_footer(); @@ -103,15 +466,38 @@ static void handle_wifi_get() { } static void handle_wifi_post() { - WifiMqttConfig cfg; - cfg.ntp_server_1 = "pool.ntp.org"; - cfg.ntp_server_2 = "time.nist.gov"; + if (!ensure_auth()) { + return; + } + WifiMqttConfig cfg = g_config; + cfg.ntp_server_1 = g_config.ntp_server_1.length() > 0 ? g_config.ntp_server_1 : "pool.ntp.org"; + cfg.ntp_server_2 = g_config.ntp_server_2.length() > 0 ? g_config.ntp_server_2 : "time.nist.gov"; cfg.ssid = server.arg("ssid"); - cfg.password = server.arg("pass"); + String wifi_pass = server.arg("pass"); + if (checkbox_checked("clear_wifi_pass")) { + cfg.password = ""; + } else if (wifi_pass.length() > 0) { + cfg.password = wifi_pass; + } cfg.mqtt_host = server.arg("mqhost"); cfg.mqtt_port = static_cast(server.arg("mqport").toInt()); cfg.mqtt_user = server.arg("mquser"); - cfg.mqtt_pass = server.arg("mqpass"); + String mqtt_pass = server.arg("mqpass"); + if (checkbox_checked("clear_mqtt_pass")) { + cfg.mqtt_pass = ""; + } else if (mqtt_pass.length() > 0) { + cfg.mqtt_pass = mqtt_pass; + } + String web_user = server.arg("webuser"); + if (web_user.length() > 0) { + cfg.web_user = web_user; + } + String web_pass = server.arg("webpass"); + if (checkbox_checked("clear_web_pass")) { + cfg.web_pass = ""; + } else if (web_pass.length() > 0) { + cfg.web_pass = web_pass; + } if (server.arg("ntp1").length() > 0) { cfg.ntp_server_1 = server.arg("ntp1"); } @@ -119,6 +505,9 @@ static void handle_wifi_post() { cfg.ntp_server_2 = server.arg("ntp2"); } cfg.valid = true; + g_config = cfg; + g_web_user = cfg.web_user; + g_web_pass = cfg.web_pass; wifi_save_config(cfg); server.send(200, "text/html", "Saved. Rebooting..."); delay(1000); @@ -126,21 +515,83 @@ static void handle_wifi_post() { } static void handle_sender() { + if (!ensure_auth()) { + return; + } if (!g_statuses) { server.send(404, "text/plain", "No senders"); return; } String uri = server.uri(); String device_id = uri.substring(String("/sender/").length()); + String device_id_url = url_encode_component(device_id); for (uint8_t i = 0; i < g_status_count; ++i) { if (device_id.equalsIgnoreCase(g_statuses[i].last_data.device_id)) { String html = html_header("Sender " + device_id); html += render_sender_block(g_statuses[i]); + html += "

    History (Power)

    "; + html += "
    "; + html += "Days: "; + html += "Res(min): "; + html += " "; + html += ""; + html += "
    "; + html += ""; + html += "
    "; + html += ""; if (g_last_batch_count[i] > 0) { html += "

    Last batch (" + String(g_last_batch_count[i]) + " samples)

    "; html += ""; html += ""; - html += ""; + html += ""; for (uint8_t r = 0; r < g_last_batch_count[i]; ++r) { const MeterData &d = g_last_batch[i][r]; html += ""; @@ -156,7 +607,9 @@ static void handle_sender() { html += ""; html += ""; html += ""; - html += ""; + html += ""; + html += ""; html += ""; } html += "
    #tse_kwhp_wp1_wp2_wp3_wbat_vbat_pctrssisnrerr_txerr_last
    bat_vbat_pctrssisnrerr_txerr_lastrx_reject
    " + String(d.link_rssi_dbm) + "" + String(d.link_snr_db, 1) + "" + String(d.err_lora_tx) + "" + String(static_cast(d.last_error)) + "" + String(static_cast(d.last_error)) + " (" + String(fault_text(d.last_error)) + ")" + String(d.rx_reject_reason) + " (" + + String(rx_reject_reason_text(static_cast(d.rx_reject_reason))) + ")
    "; @@ -170,22 +623,197 @@ static void handle_sender() { } static void handle_manual() { + if (!ensure_auth()) { + return; + } String html = html_header("DD3 Manual"); html += "
      "; html += "
    • Energy: total kWh since meter start.
      • "; html += "
    • Power: total active power in W.
      • "; html += "
    • P1/P2/P3: phase power in W.
      • "; - html += "
    • bat_v: battery voltage (V), bat_pct: estimated percent.
      • "; + html += "
    • Battery: percent with voltage in V.
      • "; html += "
    • RSSI/SNR: LoRa link quality from last packet.
      • "; - html += "
    • err_tx: LoRa TX error count; err_last: last error code.
      • "; - html += "
    • faults m/d/tx: meter read/decode/tx counters.
      • "; + html += "
    • err_tx: sender-side LoRa TX error counter.
      • "; + html += "
    • err_last: last error code (0=None, 1=MeterRead, 2=Decode, 3=LoraTx, 4=TimeSync).
      • "; + html += "
    • rx_reject: last RX reject reason (0=None, 1=crc_fail, 2=bad_protocol_version, 3=wrong_role, 4=wrong_payload_type, 5=length_mismatch, 6=device_id_mismatch, 7=batch_id_mismatch).
      • "; + html += "
    • faults m/d/tx: receiver-side counters (meter read fails, decode fails, LoRa TX fails).
      • "; + html += "
    • faults last: last receiver-side error code (same mapping as err_last).
      • "; html += "
    "; html += html_footer(); server.send(200, "text/html", html); } +static void handle_history_start() { + if (!ensure_auth()) { + return; + } + if (!sd_logger_is_ready()) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"sd_not_ready\"}"); + return; + } + if (!time_is_synced()) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"time_not_synced\"}"); + return; + } + String device_id_arg = server.arg("device_id"); + String device_id; + if (!sanitize_history_device_id(device_id_arg, device_id)) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"bad_device_id\"}"); + return; + } + uint16_t days = static_cast(server.arg("days").toInt()); + uint16_t res_min = static_cast(server.arg("res").toInt()); + String mode_str = server.arg("mode"); + if (device_id.length() == 0 || days == 0 || res_min == 0) { + server.send(200, "application/json", "{\"ok\":false,\"error\":\"bad_params\"}"); + return; + } + if (days > SD_HISTORY_MAX_DAYS) { + days = SD_HISTORY_MAX_DAYS; + } + if (res_min < SD_HISTORY_MIN_RES_MIN) { + res_min = SD_HISTORY_MIN_RES_MIN; + } + uint32_t bins = (static_cast(days) * 24UL * 60UL) / res_min; + if (bins == 0 || bins > SD_HISTORY_MAX_BINS) { + String resp = String("{\"ok\":false,\"error\":\"too_many_bins\",\"max_bins\":") + SD_HISTORY_MAX_BINS + "}"; + server.send(200, "application/json", resp); + return; + } + + history_reset(); + g_history.active = true; + g_history.done = false; + g_history.error = false; + g_history.device_id = device_id; + g_history.mode = (mode_str == "max") ? HistoryMode::Max : HistoryMode::Avg; + g_history.res_sec = static_cast(res_min) * 60UL; + g_history.bins_count = bins; + g_history.day_index = 0; + g_history.bins = new (std::nothrow) HistoryBin[bins]; + if (!g_history.bins) { + g_history.error = true; + g_history.error_msg = "oom"; + server.send(200, "application/json", "{\"ok\":false,\"error\":\"oom\"}"); + return; + } + for (uint32_t i = 0; i < bins; ++i) { + g_history.bins[i] = {}; + } + g_history.end_ts = time_get_utc(); + uint32_t span = static_cast(days) * 86400UL; + g_history.start_ts = g_history.end_ts > span ? (g_history.end_ts - span) : 0; + if (g_history.res_sec > 0) { + g_history.start_ts = (g_history.start_ts / g_history.res_sec) * g_history.res_sec; + } + + String resp = String("{\"ok\":true,\"bins\":") + bins + "}"; + server.send(200, "application/json", resp); +} + +static void handle_history_data() { + if (!ensure_auth()) { + return; + } + String device_id_arg = server.arg("device_id"); + String device_id; + if (!sanitize_history_device_id(device_id_arg, device_id)) { + server.send(200, "application/json", "{\"ready\":false,\"error\":\"bad_device_id\"}"); + return; + } + if (!g_history.bins || device_id.length() == 0 || device_id != g_history.device_id) { + server.send(200, "application/json", "{\"ready\":false,\"error\":\"no_job\"}"); + return; + } + if (g_history.error) { + String resp = String("{\"ready\":false,\"error\":\"") + g_history.error_msg + "\"}"; + server.send(200, "application/json", resp); + return; + } + if (g_history.active && !g_history.done) { + uint32_t progress = g_history.bins_count == 0 ? 0 : (g_history.bins_filled * 100UL / g_history.bins_count); + String resp = String("{\"ready\":false,\"progress\":") + progress + "}"; + server.send(200, "application/json", resp); + return; + } + + server.setContentLength(CONTENT_LENGTH_UNKNOWN); + server.send(200, "application/json", ""); + server.sendContent("{\"ready\":true,\"series\":["); + bool first = true; + for (uint32_t i = 0; i < g_history.bins_count; ++i) { + const HistoryBin &bin = g_history.bins[i]; + if (!first) { + server.sendContent(","); + } + first = false; + float value = NAN; + if (bin.count > 0) { + value = (g_history.mode == HistoryMode::Avg) ? (bin.value / static_cast(bin.count)) : bin.value; + } + if (bin.count == 0) { + server.sendContent(String("[") + bin.ts + ",null]"); + } else { + server.sendContent(String("[") + bin.ts + "," + String(value, 2) + "]"); + } + } + server.sendContent("]}"); +} + +static void handle_sd_download() { + if (!ensure_auth()) { + return; + } + if (!sd_logger_is_ready()) { + server.send(404, "text/plain", "SD not ready"); + return; + } + String path = server.arg("path"); + String error; + if (!sanitize_sd_download_path(path, error)) { + if (SERIAL_DEBUG_MODE) { + Serial.printf("sd: reject path '%s' reason=%s\n", path.c_str(), error.c_str()); + } + server.send(400, "text/plain", "Invalid path"); + return; + } + File f = SD.open(path.c_str(), FILE_READ); + if (!f) { + server.send(404, "text/plain", "Not found"); + return; + } + size_t size = f.size(); + String filename = path.substring(path.lastIndexOf('/') + 1); + bool name_clean = true; + (void)name_clean; + String safe_name = sanitize_download_filename(filename, name_clean); + String cd = "attachment; filename=\"" + safe_name + "\"; filename*=UTF-8''" + url_encode_component(safe_name); + server.sendHeader("Content-Disposition", cd); + server.setContentLength(size); + const char *content_type = "application/octet-stream"; + if (filename.endsWith(".csv")) { + content_type = "text/csv"; + } else if (filename.endsWith(".txt")) { + content_type = "text/plain"; + } + server.send(200, content_type, ""); + WiFiClient client = server.client(); + uint8_t buf[512]; + while (f.available()) { + size_t n = f.read(buf, sizeof(buf)); + if (n == 0) { + break; + } + client.write(buf, n); + delay(0); + } + f.close(); +} + void web_server_set_config(const WifiMqttConfig &config) { g_config = config; + g_web_user = config.web_user; + g_web_pass = config.web_pass; } void web_server_set_sender_faults(const FaultCounters *faults, const FaultType *last_errors) { @@ -213,6 +841,9 @@ void web_server_begin_ap(const SenderStatus *statuses, uint8_t count) { server.on("/", handle_root); server.on("/manual", handle_manual); + server.on("/history/start", handle_history_start); + server.on("/history/data", handle_history_data); + server.on("/sd/download", handle_sd_download); server.on("/wifi", HTTP_GET, handle_wifi_get); server.on("/wifi", HTTP_POST, handle_wifi_post); server.on("/sender/", handle_sender); @@ -234,6 +865,9 @@ void web_server_begin_sta(const SenderStatus *statuses, uint8_t count) { server.on("/", handle_root); server.on("/manual", handle_manual); server.on("/sender/", handle_sender); + server.on("/history/start", handle_history_start); + server.on("/history/data", handle_history_data); + server.on("/sd/download", handle_sd_download); server.on("/wifi", HTTP_GET, handle_wifi_get); server.on("/wifi", HTTP_POST, handle_wifi_post); server.onNotFound([]() { @@ -247,5 +881,6 @@ void web_server_begin_sta(const SenderStatus *statuses, uint8_t count) { } void web_server_loop() { + history_tick(); server.handleClient(); } diff --git a/src/wifi_manager.cpp b/src/wifi_manager.cpp index 5d5494a..9ea7195 100644 --- a/src/wifi_manager.cpp +++ b/src/wifi_manager.cpp @@ -1,4 +1,5 @@ #include "wifi_manager.h" +#include "config.h" #include #include @@ -10,9 +11,6 @@ void wifi_manager_init() { bool wifi_load_config(WifiMqttConfig &config) { config.valid = prefs.getBool("valid", false); - if (!config.valid) { - return false; - } config.ssid = prefs.getString("ssid", ""); config.password = prefs.getString("pass", ""); config.mqtt_host = prefs.getString("mqhost", ""); @@ -21,6 +19,11 @@ bool wifi_load_config(WifiMqttConfig &config) { config.mqtt_pass = prefs.getString("mqpass", ""); config.ntp_server_1 = prefs.getString("ntp1", "pool.ntp.org"); config.ntp_server_2 = prefs.getString("ntp2", "time.nist.gov"); + config.web_user = prefs.getString("webuser", WEB_AUTH_DEFAULT_USER); + config.web_pass = prefs.getString("webpass", WEB_AUTH_DEFAULT_PASS); + if (!config.valid) { + return false; + } return config.ssid.length() > 0 && config.mqtt_host.length() > 0; } @@ -34,6 +37,8 @@ bool wifi_save_config(const WifiMqttConfig &config) { prefs.putString("mqpass", config.mqtt_pass); prefs.putString("ntp1", config.ntp_server_1); prefs.putString("ntp2", config.ntp_server_2); + prefs.putString("webuser", config.web_user); + prefs.putString("webpass", config.web_pass); return true; } diff --git a/test/test_html_escape/test_html_escape.cpp b/test/test_html_escape/test_html_escape.cpp new file mode 100644 index 0000000..2f782cc --- /dev/null +++ b/test/test_html_escape/test_html_escape.cpp @@ -0,0 +1,37 @@ +#include +#include +#include "html_util.h" + +static void test_html_escape_basic() { + TEST_ASSERT_EQUAL_STRING("", html_escape("").c_str()); + TEST_ASSERT_EQUAL_STRING("plain", html_escape("plain").c_str()); + TEST_ASSERT_EQUAL_STRING("a&b", html_escape("a&b").c_str()); + TEST_ASSERT_EQUAL_STRING("<tag>", html_escape("").c_str()); + TEST_ASSERT_EQUAL_STRING(""hi"", html_escape("\"hi\"").c_str()); + TEST_ASSERT_EQUAL_STRING("it's", html_escape("it's").c_str()); + TEST_ASSERT_EQUAL_STRING("&<>"'", html_escape("&<>\"'").c_str()); +} + +static void test_sanitize_device_id() { + String out; + TEST_ASSERT_TRUE(sanitize_device_id("F19C", out)); + TEST_ASSERT_EQUAL_STRING("dd3-F19C", out.c_str()); + TEST_ASSERT_TRUE(sanitize_device_id("dd3-f19c", out)); + TEST_ASSERT_EQUAL_STRING("dd3-F19C", out.c_str()); + TEST_ASSERT_FALSE(sanitize_device_id("F19G", out)); + TEST_ASSERT_FALSE(sanitize_device_id("dd3-12", out)); + TEST_ASSERT_FALSE(sanitize_device_id("dd3-12345", out)); + TEST_ASSERT_FALSE(sanitize_device_id("../F19C", out)); + TEST_ASSERT_FALSE(sanitize_device_id("dd3-%2f", out)); + TEST_ASSERT_FALSE(sanitize_device_id("dd3-12/3", out)); + TEST_ASSERT_FALSE(sanitize_device_id("dd3-12\\3", out)); +} + +void setup() { + UNITY_BEGIN(); + RUN_TEST(test_html_escape_basic); + RUN_TEST(test_sanitize_device_id); + UNITY_END(); +} + +void loop() {}