feat: add HIL diagnostics and meter health handling

2026-08-17 22:33:04 +02:00
parent bc2ac6e8f6
commit aa25aa2c12
30 changed files with 2108 additions and 236 deletions
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import time
+
+from hil_common import PairCapture, load_config, main_guard, new_artifact_dir, pair_ports
+from report import generate
+
+
+FAULTS = [
+    "drop_chunk",
+    "duplicate_chunk",
+    "corrupt_chunk",
+    "suppress_ack",
+    "delay_ack",
+    "wrong_ack_batch",
+]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run all one-shot LoRa HIL faults in one approved capture")
+    parser.add_argument("--fault", action="append", choices=FAULTS,
+                        help="run only this fault (repeatable); defaults to all")
+    parser.add_argument("--seconds", type=float, default=600)
+    parser.add_argument("--arm-after", type=float, default=10)
+    parser.add_argument("--case-timeout", type=float, default=100)
+    args = parser.parse_args()
+    faults = args.fault or FAULTS
+    config = load_config()
+    ports = pair_ports(config, "access")
+    baud = int(config.get("hil", {}).get("baud", 115200))
+    artifact = new_artifact_dir("fault-suite")
+    state = {"index": 0, "armed_at": None, "last_command_at": None, "armed_confirmed": False,
+             "injected_ns": None, "recovered_at": None, "cases": []}
+
+    def arm(capture: PairCapture, now: float) -> None:
+        fault = faults[state["index"]]
+        target = "sender" if fault.endswith("chunk") else "receiver"
+        command = "delay_ack:1000" if fault == "delay_ack" else fault
+        capture.send_fault(target, command)
+        state["armed_at"] = now
+        state["last_command_at"] = now
+        state["armed_confirmed"] = False
+        state["injected_ns"] = None
+        state["recovered_at"] = None
+        print(f"Armed {fault} on {target}", flush=True)
+
+    def finish_case(capture: PairCapture, now: float, recovered: bool) -> None:
+        fault = faults[state["index"]]
+        state["cases"].append({"fault": fault, "recovered": recovered})
+        state["index"] += 1
+        if state["index"] >= len(faults):
+            capture.stop.set()
+            return
+        arm(capture, now)
+
+    def tick(elapsed: float, capture: PairCapture) -> None:
+        now = time.monotonic()
+        if state["index"] >= len(faults):
+            capture.stop.set()
+            return
+        if state["armed_at"] is None:
+            if elapsed >= args.arm_after:
+                arm(capture, now)
+            return
+        fault = faults[state["index"]]
+        snapshot = list(capture.events)
+        armed = [event for event in snapshot if event.get("event") == "fault_armed" and
+                 event.get("fault") == fault]
+        if armed:
+            state["armed_confirmed"] = True
+        if not state["armed_confirmed"] and now - state["last_command_at"] >= 5:
+            target = "sender" if fault.endswith("chunk") else "receiver"
+            command = "delay_ack:1000" if fault == "delay_ack" else fault
+            capture.send_fault(target, command)
+            state["last_command_at"] = now
+            print(f"Re-sent {fault}; awaiting device arm acknowledgement", flush=True)
+        injections = [event for event in snapshot if event.get("event") == "fault_injected" and
+                      event.get("fault") == fault]
+        if injections and state["injected_ns"] is None:
+            state["injected_ns"] = min(event.get("host_time_ns", 0) for event in injections)
+        if state["injected_ns"]:
+            recovered = any(event.get("event") == "ack_rx" and event.get("ok") is True and
+                            event.get("host_time_ns", 0) > state["injected_ns"] for event in snapshot)
+            if recovered:
+                if state["recovered_at"] is None:
+                    state["recovered_at"] = now
+                elif now - state["recovered_at"] >= 3:
+                    finish_case(capture, now, True)
+                    return
+        if now - state["armed_at"] >= args.case_timeout:
+            finish_case(capture, now, False)
+
+    PairCapture(ports, baud, artifact).run(args.seconds, tick, reset_on_start=True)
+    (artifact / "fault-suite.json").write_text(json.dumps(state["cases"], indent=2) + "\n", encoding="utf-8")
+    checks = generate(artifact, faults)
+    print(f"Fault-suite report: {artifact / 'report.md'}")
+    return 1 if any(check.status == "fail" for check in checks) else 0
+
+
+if __name__ == "__main__":
+    main_guard(main)