feat: add HIL diagnostics and meter health handling

2026-08-17 22:33:04 +02:00
parent bc2ac6e8f6
commit aa25aa2c12
30 changed files with 2108 additions and 236 deletions
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from hil_common import HilError, main_guard, read_events
+
+
+@dataclass
+class Check:
+    name: str
+    status: str
+    detail: str
+
+
+def _events(events, name, source=None):
+    return [event for event in events if event.get("event") == name and (source is None or event.get("source") == source)]
+
+
+def evaluate(events: list[dict], requested_fault: str | list[str] | None = None) -> list[Check]:
+    checks: list[Check] = []
+    roles = {event.get("source"): event.get("role") for event in _events(events, "role")}
+    role_ok = roles.get("sender") == "sender" and roles.get("receiver") == "receiver"
+    checks.append(Check("role mapping", "pass" if role_ok else "fail", json.dumps(roles, sort_keys=True)))
+
+    boot_counts = {source: len(_events(events, "boot", source)) for source in ("sender", "receiver")}
+    reset_ok = all(1 <= count <= 2 for count in boot_counts.values())
+    checks.append(Check("no reset loop", "pass" if reset_ok else "fail", json.dumps(boot_counts, sort_keys=True)))
+
+    time_ok = any(event.get("stage") == "complete" and event.get("ok") is True
+                  for event in _events(events, "time_bootstrap", "sender"))
+    checks.append(Check("time bootstrap", "pass" if time_ok else "fail", "sender completion event captured" if time_ok else "no sender completion event"))
+
+    meter = _events(events, "meter_frame", "sender")
+    meter_ok = any(event.get("ok") is True for event in meter)
+    meter_status = "pass" if meter_ok else "blocked"
+    checks.append(Check("meter parser", meter_status,
+                        "valid or partial meter frame captured" if meter_ok else "no valid meter evidence; meter/safe simulator unavailable or silent"))
+
+    health = _events(events, "health")
+    checks.append(Check("runtime diagnostics", "pass" if len(health) >= 2 else "fail", f"{len(health)} health events"))
+
+    created = {event.get("batch_id") for event in _events(events, "batch_created", "sender")}
+    complete = {event.get("batch_id") for event in _events(events, "reassembly_complete", "receiver")}
+    decoded = {event.get("batch_id") for event in _events(events, "payload_decode", "receiver") if event.get("ok") is True}
+    ack_tx = {event.get("batch_id") for event in _events(events, "ack_tx", "receiver") if event.get("ok") is True}
+    ack_rx = {event.get("batch_id") for event in _events(events, "ack_rx", "sender") if event.get("ok") is True}
+    correlated = sorted((created & complete & decoded & ack_tx & ack_rx) - {None})
+    if correlated:
+        batch_status = "pass"
+        batch_detail = f"correlated batch IDs: {correlated}"
+    elif not created and not meter_ok:
+        batch_status = "blocked"
+        batch_detail = "no valid meter input, so publication was correctly suppressed"
+    else:
+        batch_status = "fail"
+        batch_detail = f"correlated batch IDs: {correlated}"
+    checks.append(Check("batch and ACK correlation", batch_status, batch_detail))
+
+    requested_faults = ([requested_fault] if isinstance(requested_fault, str) else requested_fault) or []
+    for fault in requested_faults:
+        injected = [event for event in _events(events, "fault_injected") if event.get("fault") == fault]
+        injection_ns = min((event.get("host_time_ns", 0) for event in injected), default=0)
+        later_injections = [event.get("host_time_ns", 0) for event in _events(events, "fault_injected")
+                            if event.get("host_time_ns", 0) > injection_ns]
+        window_end = min(later_injections) if later_injections else 2**63 - 1
+        in_window = lambda event: injection_ns < event.get("host_time_ns", 0) < window_end
+        recovered = any(event.get("event") == "ack_rx" and event.get("ok") is True and in_window(event)
+                        for event in events)
+        details = [f"injected={bool(injected)}", f"later successful ACK={recovered}"]
+        expected = True
+        if fault in {"drop_chunk", "corrupt_chunk", "suppress_ack", "wrong_ack_batch"}:
+            retry = any(event.get("event") == "retry" and in_window(event) for event in events)
+            details.append(f"retry={retry}")
+            expected = retry
+            if fault in {"drop_chunk", "corrupt_chunk"} and retry:
+                first_retry_ns = min(event.get("host_time_ns", 0) for event in events
+                                     if event.get("event") == "retry" and in_window(event))
+                premature_decode = any(event.get("event") == "payload_decode" and event.get("ok") is True and
+                                       injection_ns < event.get("host_time_ns", 0) < first_retry_ns
+                                       for event in events)
+                details.append(f"decoded before retry={premature_decode}")
+                expected = expected and not premature_decode
+        if fault == "duplicate_chunk":
+            duplicate_detected = any(event.get("event") == "duplicate" and in_window(event) for event in events)
+            details.append(f"receiver duplicate detection={duplicate_detected}")
+            expected = duplicate_detected
+        if fault == "corrupt_chunk":
+            crc_reject = any(event.get("event") == "lora_reject" and event.get("reason") == "crc_fail" and
+                             in_window(event) for event in events)
+            details.append(f"CRC rejection={crc_reject}")
+            expected = expected and crc_reject
+        if fault == "wrong_ack_batch":
+            wrong_reject = any(event.get("event") == "ack_rx" and event.get("ok") is False and
+                               in_window(event) for event in events)
+            details.append(f"wrong ACK rejected={wrong_reject}")
+            expected = expected and wrong_reject
+        passed = bool(injected) and recovered and expected
+        checks.append(Check(f"fault {fault}", "pass" if passed else "fail", ", ".join(details)))
+    return checks
+
+
+def write_report(artifact: Path, checks: list[Check], events: list[dict]) -> None:
+    lines = ["# DD3 local HIL report", "", f"Artifact directory: `{artifact}`", "", "## Results", ""]
+    for check in checks:
+        lines.append(f"- **{check.status.upper()}** — {check.name}: {check.detail}")
+    lines.extend(["", "## Evidence summary", ""])
+    resets = [{"source": e.get("source"), "reset_reason": e.get("reset_reason")} for e in _events(events, "boot")]
+    lines.append(f"- Reset evidence: `{json.dumps(resets, sort_keys=True)}`")
+    lines.append(f"- Meter classifications: `{json.dumps([e.get('classification') for e in _events(events, 'meter_frame')])}`")
+    lines.append(f"- Retry events: `{len(_events(events, 'retry'))}`")
+    queue_drops = [event for event in _events(events, "queue_drop") if event.get("inflight") is not True]
+    lines.append(f"- Queue drops: `{len(queue_drops)}`")
+    heaps = [e.get("heap_free") for e in _events(events, "health") if isinstance(e.get("heap_free"), int)]
+    lines.append(f"- Minimum observed free heap: `{min(heaps) if heaps else 'not captured'}`")
+    lines.extend(["", "Raw serial bytes and timestamped text logs are retained beside this report.", ""])
+    (artifact / "report.md").write_text("\n".join(lines), encoding="utf-8")
+
+    suite = ET.Element("testsuite", name="dd3-hil", tests=str(len(checks)),
+                       failures=str(sum(c.status == "fail" for c in checks)),
+                       skipped=str(sum(c.status == "blocked" for c in checks)))
+    for check in checks:
+        case = ET.SubElement(suite, "testcase", name=check.name, classname="hil")
+        if check.status == "fail":
+            ET.SubElement(case, "failure", message=check.detail)
+        elif check.status == "blocked":
+            ET.SubElement(case, "skipped", message=check.detail)
+        ET.SubElement(case, "system-out").text = check.detail
+    ET.ElementTree(suite).write(artifact / "junit.xml", encoding="utf-8", xml_declaration=True)
+
+
+def generate(artifact: Path, requested_fault: str | list[str] | None = None) -> list[Check]:
+    events = read_events(artifact)
+    if not events:
+        raise HilError(f"no parsed HIL events in {artifact / 'events.jsonl'}")
+    checks = evaluate(events, requested_fault)
+    write_report(artifact, checks, events)
+    return checks
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Generate Markdown and JUnit reports from captured HIL evidence")
+    parser.add_argument("artifact_dir", type=Path)
+    parser.add_argument("--fault", action="append")
+    args = parser.parse_args()
+    checks = generate(args.artifact_dir, args.fault)
+    for check in checks:
+        print(f"{check.status.upper():7} {check.name}: {check.detail}")
+    return 1 if any(check.status == "fail" for check in checks) else 0
+
+
+if __name__ == "__main__":
+    main_guard(main)