From aa25aa2c128e3eb4ee9e7f152da822917c55606b Mon Sep 17 00:00:00 2001 From: acidburns Date: Mon, 17 Aug 2026 22:33:04 +0200 Subject: [PATCH] feat: add HIL diagnostics and meter health handling --- .gitignore | 5 + docs/engineering/hil-test-matrix.md | 20 ++ docs/engineering/local-hil-setup.md | 94 ++++++++ hil_tests/README.md | 15 ++ hil_tests/meter_parser_harness.cpp | 104 +++++++++ hil_tests/test_hil_tools.py | 48 ++++ hil_tests/test_live_evidence.py | 54 +++++ hil_tests/test_meter_parser.py | 33 +++ include/hil_trace.h | 41 ++++ include/meter_driver.h | 15 ++ include/meter_health.h | 26 +++ include/meter_parser.h | 54 +++++ platformio.ini | 7 + pytest.ini | 4 + requirements-hil.txt | 2 + src/hil_trace.cpp | 156 +++++++++++++ src/lora_transport.cpp | 9 + src/main.cpp | 258 ++++++++++++++++++--- src/meter_driver.cpp | 298 +++++++++---------------- src/meter_health.cpp | 25 +++ src/meter_parser.cpp | 334 ++++++++++++++++++++++++++++ tools/hil/build.py | 21 ++ tools/hil/capture_pair.py | 38 ++++ tools/hil/discover.py | 21 ++ tools/hil/flash_pair.py | 33 +++ tools/hil/hil_common.py | 261 ++++++++++++++++++++++ tools/hil/meter_simulator.py | 68 ++++++ tools/hil/report.py | 157 +++++++++++++ tools/hil/run_baseline.py | 39 ++++ tools/hil/run_fault_suite.py | 104 +++++++++ 30 files changed, 2108 insertions(+), 236 deletions(-) create mode 100644 docs/engineering/hil-test-matrix.md create mode 100644 docs/engineering/local-hil-setup.md create mode 100644 hil_tests/README.md create mode 100644 hil_tests/meter_parser_harness.cpp create mode 100644 hil_tests/test_hil_tools.py create mode 100644 hil_tests/test_live_evidence.py create mode 100644 hil_tests/test_meter_parser.py create mode 100644 include/hil_trace.h create mode 100644 include/meter_health.h create mode 100644 include/meter_parser.h create mode 100644 pytest.ini create mode 100644 requirements-hil.txt create mode 100644 src/hil_trace.cpp create mode 100644 src/meter_health.cpp create mode 100644 src/meter_parser.cpp create mode 100644 tools/hil/build.py create mode 100644 tools/hil/capture_pair.py create mode 100644 tools/hil/discover.py create mode 100644 tools/hil/flash_pair.py create mode 100644 tools/hil/hil_common.py create mode 100644 tools/hil/meter_simulator.py create mode 100644 tools/hil/report.py create mode 100644 tools/hil/run_baseline.py create mode 100644 tools/hil/run_fault_suite.py diff --git a/.gitignore b/.gitignore index 89cc49c..66a60a2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ .vscode/c_cpp_properties.json .vscode/launch.json .vscode/ipch +.hil/local.toml +artifacts/hil/ +__pycache__/ +.pytest_cache/ +.venv-hil/ diff --git a/docs/engineering/hil-test-matrix.md b/docs/engineering/hil-test-matrix.md new file mode 100644 index 0000000..6deb04d --- /dev/null +++ b/docs/engineering/hil-test-matrix.md @@ -0,0 +1,20 @@ +# DD3 HIL test matrix + +| Area | Stimulus | Required evidence | Expected result | +|---|---|---|---| +| Boot | Flash/start both boards | `boot`, `role`, periodic `health` from each | Exactly one sender and receiver; no reset loop; bounded heap/stack | +| Time | Sender receive window and receiver time-sync | sender request/completion, receiver transmit | Valid UTC completion; misses remain visible | +| Meter baseline | Real optical meter | `meter_frame`, `meter_sample` | Frame classification and seconds index when present; no secrets/values logged | +| Batch | Normal 30 s batching | create, encode, every TX/RX chunk, complete, decode, ACK TX/RX | One batch ID correlates end to end | +| Meter timeout | Partial start then >1.5 s pause | `classification=timeout`, health | No reset; next valid fixture recovers | +| Meter malformed | Each parser fixture then valid | failure/partial classification, then valid | No reset; failure is not treated as valid evidence | +| Meter time | rollback/jump seconds fixtures | `meter_time` rollback/jump | Diagnostic classification; subsequent valid frame recovers | +| Oversize/slow | >512-byte or slow fixture | oversized or valid event, health | Bounded buffers and continued operation | +| Drop chunk | `drop_chunk` once | incomplete transfer, retry, later complete/ACK | Incomplete batch is not decoded; retry recovers | +| Duplicate chunk | `duplicate_chunk` once | duplicate/incomplete diagnostic, retry, later ACK | No double publication; transfer recovers | +| Corrupt chunk | post-CRC byte flip once | `lora_reject=crc_fail`, retry, later ACK | CRC rejects frame; incomplete batch not decoded | +| Suppress ACK | suppress one ACK response | receiver suppression, sender timeout/retry, later ACK | Retry count increments and recovers | +| Delay ACK | delay one ACK response | delay event and ACK/retry timing | No crash/deadlock; transfer recovers | +| Wrong ACK ID | altered batch ID once | sender rejected `ack_rx`, retry, correct ACK | Incorrect ACK cannot complete inflight batch | + +For every fault case, compare `health` events before and after injection, require a later successful ACK, and inspect raw logs for resets. MQTT duplicate-publication evidence may require a separately approved broker observer; without it, report that assertion as blocked rather than inferring success solely from radio logs. diff --git a/docs/engineering/local-hil-setup.md b/docs/engineering/local-hil-setup.md new file mode 100644 index 0000000..3efec60 --- /dev/null +++ b/docs/engineering/local-hil-setup.md @@ -0,0 +1,94 @@ +# Safe local DD3 HIL setup + +## Safety boundary + +This harness is for two USB-connected LilyGO boards and, optionally, an electrically isolated USB-UART or optical meter simulator. It does not authorize direct connection to mains-referenced meter conductors. Do not change wiring, GPIO14 role selection, straps, eFuses, secure boot, flash encryption, NVS, credentials, or udev/OS permissions as part of this workflow. + +The firmware remains role-neutral: GPIO14 is sampled at boot exactly as before. The `hil` environment inherits `lilygo-t3-v1-6-1` (the current 433 MHz debug build) and adds only `ENABLE_HIL_TRACE=1`. Production environments are unchanged. + +## Dependencies and Linux serial access + +PlatformIO is accepted from `PATH` or `~/.platformio/penv/bin/pio`. Install Python dependencies into a repository-local environment: + +```bash +python3 -m venv .venv-hil +.venv-hil/bin/pip install -r requirements-hil.txt +``` + +Inspect access with `id -nG` and `stat -c '%A %U %G %n' /dev/ttyACM0`. On this machine the user is already in `dialout`, and the discovered ports are `root:dialout` with group read/write. The tools never run `sudo`, modify group membership, change permissions, or install udev rules. If access is absent, stop and ask the system administrator to apply the site's normal policy, then start a new login session. + +## Discover and approve stable ports + +Discovery lists metadata and symlinks but does not open a device: + +```bash +.venv-hil/bin/python tools/hil/discover.py +``` + +Prefer `/dev/serial/by-id` names. Copy `.hil/local.example.toml` to ignored `.hil/local.toml` only after visually confirming the two targets. Set the sender and receiver path, then set `approved_access = true` only after approving the first serial open. Set `approved_flash = true` only after separately approving flashing. The tools refuse missing, duplicate, unstable, or unapproved ports. Never commit local `/dev` paths. + +Current discovery on 2026-07-13 found two uniquely identified board candidates and one ambiguous device: + +```text +usb-1a86_USB_Single_Serial_5644005006-if00 -> ttyACM0 +usb-1a86_USB_Single_Serial_5644020877-if00 -> ttyACM1 +usb-1a86_USB_Serial-if00-port0 -> ttyUSB0 (ambiguous; do not access) +``` + +Serial role evidence, not the filename, is authoritative. A valid mapping requires a `HIL:` role event from both devices and must agree with the configured names. + +## Build, flash, and capture + +Building never accesses serial devices and does not require local port configuration: + +```bash +.venv-hil/bin/python tools/hil/build.py +``` + +After explicit access and flash approval is recorded: + +```bash +.venv-hil/bin/python tools/hil/flash_pair.py --yes-flash +.venv-hil/bin/python tools/hil/capture_pair.py --reset --seconds 180 +.venv-hil/bin/python tools/hil/run_baseline.py --yes-flash --seconds 180 +``` + +`flash_pair.py` builds the HIL environment once, then uploads the same environment to each approved stable path. It does not issue a full-chip or NVS erase; normal uploading necessarily rewrites the bootloader, partition-table, boot-app, and application address ranges shown in the flash log. `--reset` pulses the boards' normal USB auto-reset circuit only after capture readers are active. Ctrl+C closes both serial handles and retains all bytes captured so far. + +Each run writes beneath `artifacts/hil/-