105 lines
4.3 KiB
Python
105 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
|
|
from hil_common import PairCapture, load_config, main_guard, new_artifact_dir, pair_ports
|
|
from report import generate
|
|
|
|
|
|
FAULTS = [
|
|
"drop_chunk",
|
|
"duplicate_chunk",
|
|
"corrupt_chunk",
|
|
"suppress_ack",
|
|
"delay_ack",
|
|
"wrong_ack_batch",
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Run all one-shot LoRa HIL faults in one approved capture")
|
|
parser.add_argument("--fault", action="append", choices=FAULTS,
|
|
help="run only this fault (repeatable); defaults to all")
|
|
parser.add_argument("--seconds", type=float, default=600)
|
|
parser.add_argument("--arm-after", type=float, default=10)
|
|
parser.add_argument("--case-timeout", type=float, default=100)
|
|
args = parser.parse_args()
|
|
faults = args.fault or FAULTS
|
|
config = load_config()
|
|
ports = pair_ports(config, "access")
|
|
baud = int(config.get("hil", {}).get("baud", 115200))
|
|
artifact = new_artifact_dir("fault-suite")
|
|
state = {"index": 0, "armed_at": None, "last_command_at": None, "armed_confirmed": False,
|
|
"injected_ns": None, "recovered_at": None, "cases": []}
|
|
|
|
def arm(capture: PairCapture, now: float) -> None:
|
|
fault = faults[state["index"]]
|
|
target = "sender" if fault.endswith("chunk") else "receiver"
|
|
command = "delay_ack:1000" if fault == "delay_ack" else fault
|
|
capture.send_fault(target, command)
|
|
state["armed_at"] = now
|
|
state["last_command_at"] = now
|
|
state["armed_confirmed"] = False
|
|
state["injected_ns"] = None
|
|
state["recovered_at"] = None
|
|
print(f"Armed {fault} on {target}", flush=True)
|
|
|
|
def finish_case(capture: PairCapture, now: float, recovered: bool) -> None:
|
|
fault = faults[state["index"]]
|
|
state["cases"].append({"fault": fault, "recovered": recovered})
|
|
state["index"] += 1
|
|
if state["index"] >= len(faults):
|
|
capture.stop.set()
|
|
return
|
|
arm(capture, now)
|
|
|
|
def tick(elapsed: float, capture: PairCapture) -> None:
|
|
now = time.monotonic()
|
|
if state["index"] >= len(faults):
|
|
capture.stop.set()
|
|
return
|
|
if state["armed_at"] is None:
|
|
if elapsed >= args.arm_after:
|
|
arm(capture, now)
|
|
return
|
|
fault = faults[state["index"]]
|
|
snapshot = list(capture.events)
|
|
armed = [event for event in snapshot if event.get("event") == "fault_armed" and
|
|
event.get("fault") == fault]
|
|
if armed:
|
|
state["armed_confirmed"] = True
|
|
if not state["armed_confirmed"] and now - state["last_command_at"] >= 5:
|
|
target = "sender" if fault.endswith("chunk") else "receiver"
|
|
command = "delay_ack:1000" if fault == "delay_ack" else fault
|
|
capture.send_fault(target, command)
|
|
state["last_command_at"] = now
|
|
print(f"Re-sent {fault}; awaiting device arm acknowledgement", flush=True)
|
|
injections = [event for event in snapshot if event.get("event") == "fault_injected" and
|
|
event.get("fault") == fault]
|
|
if injections and state["injected_ns"] is None:
|
|
state["injected_ns"] = min(event.get("host_time_ns", 0) for event in injections)
|
|
if state["injected_ns"]:
|
|
recovered = any(event.get("event") == "ack_rx" and event.get("ok") is True and
|
|
event.get("host_time_ns", 0) > state["injected_ns"] for event in snapshot)
|
|
if recovered:
|
|
if state["recovered_at"] is None:
|
|
state["recovered_at"] = now
|
|
elif now - state["recovered_at"] >= 3:
|
|
finish_case(capture, now, True)
|
|
return
|
|
if now - state["armed_at"] >= args.case_timeout:
|
|
finish_case(capture, now, False)
|
|
|
|
PairCapture(ports, baud, artifact).run(args.seconds, tick, reset_on_start=True)
|
|
(artifact / "fault-suite.json").write_text(json.dumps(state["cases"], indent=2) + "\n", encoding="utf-8")
|
|
checks = generate(artifact, faults)
|
|
print(f"Fault-suite report: {artifact / 'report.md'}")
|
|
return 1 if any(check.status == "fail" for check in checks) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main_guard(main)
|