feat: add HIL diagnostics and meter health handling
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
|
||||
from hil_common import load_config_if_present, main_guard, new_artifact_dir, pio_command, run_checked
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build the configured HIL firmware without accessing serial ports")
|
||||
parser.add_argument("--artifact-dir")
|
||||
args = parser.parse_args()
|
||||
config = load_config_if_present()
|
||||
environment = config.get("hil", {}).get("environment", "hil")
|
||||
artifact = new_artifact_dir("build") if not args.artifact_dir else __import__("pathlib").Path(args.artifact_dir)
|
||||
artifact.mkdir(parents=True, exist_ok=True)
|
||||
run_checked([pio_command(), "run", "-e", environment], log=artifact / "build.log")
|
||||
print(f"Build log: {artifact / 'build.log'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from hil_common import PairCapture, load_config, main_guard, new_artifact_dir, pair_ports
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Capture both approved serial streams concurrently")
|
||||
parser.add_argument("--seconds", type=float)
|
||||
parser.add_argument("--artifact-dir")
|
||||
parser.add_argument("--reset", action="store_true", help="normally reboot both boards after capture starts")
|
||||
parser.add_argument("--fault", choices=["drop_chunk", "duplicate_chunk", "corrupt_chunk", "suppress_ack",
|
||||
"delay_ack", "wrong_ack_batch"])
|
||||
parser.add_argument("--fault-after", type=float, default=10.0)
|
||||
args = parser.parse_args()
|
||||
config = load_config()
|
||||
ports = pair_ports(config, "access")
|
||||
baud = int(config.get("hil", {}).get("baud", 115200))
|
||||
seconds = args.seconds or float(config.get("hil", {}).get("capture_seconds", 180))
|
||||
artifact = Path(args.artifact_dir) if args.artifact_dir else new_artifact_dir("capture")
|
||||
sent = False
|
||||
|
||||
def tick(elapsed: float, capture: PairCapture) -> None:
|
||||
nonlocal sent
|
||||
if args.fault and not sent and elapsed >= args.fault_after:
|
||||
target = "sender" if args.fault.endswith("chunk") else "receiver"
|
||||
command = "delay_ack:1000" if args.fault == "delay_ack" else args.fault
|
||||
capture.send_fault(target, command)
|
||||
sent = True
|
||||
|
||||
PairCapture(ports, baud, artifact).run(seconds, tick, reset_on_start=args.reset)
|
||||
print(f"Capture artifacts: {artifact}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
|
||||
from hil_common import main_guard, pio_command, run_checked
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("Stable serial candidates (listing only; no device is opened):")
|
||||
by_id = Path("/dev/serial/by-id")
|
||||
candidates = sorted(by_id.iterdir()) if by_id.is_dir() else []
|
||||
if not candidates:
|
||||
print(" none")
|
||||
for candidate in candidates:
|
||||
print(f" {candidate} -> {candidate.resolve()}")
|
||||
print("\nPlatformIO discovery (metadata only):")
|
||||
run_checked([pio_command(), "device", "list"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from hil_common import HilError, load_config, main_guard, new_artifact_dir, pair_ports, pio_command, run_checked
|
||||
|
||||
|
||||
def flash_pair(artifact: Path, yes_flash: bool) -> None:
|
||||
if not yes_flash:
|
||||
raise HilError("refusing to flash without --yes-flash")
|
||||
config = load_config()
|
||||
ports = pair_ports(config, "flash")
|
||||
environment = config.get("hil", {}).get("environment", "hil")
|
||||
artifact.mkdir(parents=True, exist_ok=True)
|
||||
run_checked([pio_command(), "run", "-e", environment], log=artifact / "build.log")
|
||||
for port in ports:
|
||||
run_checked([pio_command(), "run", "-e", environment, "-t", "upload", "--upload-port", str(port.device)],
|
||||
log=artifact / f"flash-{port.name}.log")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build once and flash the explicitly approved board pair")
|
||||
parser.add_argument("--yes-flash", action="store_true", help="confirm the approved_flash entries for this invocation")
|
||||
parser.add_argument("--artifact-dir")
|
||||
args = parser.parse_args()
|
||||
artifact = Path(args.artifact_dir) if args.artifact_dir else new_artifact_dir("flash")
|
||||
flash_pair(artifact, args.yes_flash)
|
||||
print(f"Flash logs: {artifact}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - Python >=3.11 is documented
|
||||
tomllib = None
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_CONFIG = ROOT / ".hil" / "local.toml"
|
||||
ARTIFACT_ROOT = ROOT / "artifacts" / "hil"
|
||||
HIL_PREFIX = b"HIL:"
|
||||
|
||||
|
||||
class HilError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PortConfig:
|
||||
name: str
|
||||
device: Path
|
||||
approved_access: bool
|
||||
approved_flash: bool = False
|
||||
|
||||
|
||||
def utc_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
def new_artifact_dir(label: str = "run") -> Path:
|
||||
path = ARTIFACT_ROOT / f"{utc_stamp()}-{label}"
|
||||
suffix = 1
|
||||
while path.exists():
|
||||
path = ARTIFACT_ROOT / f"{utc_stamp()}-{label}-{suffix}"
|
||||
suffix += 1
|
||||
path.mkdir(parents=True)
|
||||
return path
|
||||
|
||||
|
||||
def load_config(path: Path = DEFAULT_CONFIG) -> dict[str, Any]:
|
||||
if tomllib is None:
|
||||
raise HilError("Python 3.11 or newer is required (tomllib is unavailable)")
|
||||
if not path.is_file():
|
||||
raise HilError(f"missing {path}; copy .hil/local.example.toml and enter only approved ports")
|
||||
with path.open("rb") as handle:
|
||||
config = tomllib.load(handle)
|
||||
return config
|
||||
|
||||
|
||||
def load_config_if_present(path: Path = DEFAULT_CONFIG) -> dict[str, Any]:
|
||||
return load_config(path) if path.is_file() else {}
|
||||
|
||||
|
||||
def configured_port(config: dict[str, Any], name: str, operation: str) -> PortConfig:
|
||||
raw = config.get("ports", {}).get(name, {})
|
||||
device_text = str(raw.get("device", "")).strip()
|
||||
if not device_text:
|
||||
raise HilError(f"ports.{name}.device is not configured")
|
||||
device = Path(device_text)
|
||||
if not str(device).startswith("/dev/serial/by-id/"):
|
||||
raise HilError(f"{name} must use a stable /dev/serial/by-id path, got {device}")
|
||||
if not device.exists():
|
||||
raise HilError(f"configured {name} device does not exist: {device}")
|
||||
access = raw.get("approved_access") is True
|
||||
flash = raw.get("approved_flash") is True
|
||||
if operation in {"access", "flash"} and not access:
|
||||
raise HilError(f"first access to {device} is not approved in {DEFAULT_CONFIG}")
|
||||
if operation == "flash" and not flash:
|
||||
raise HilError(f"flashing {device} is not approved in {DEFAULT_CONFIG}")
|
||||
return PortConfig(name, device, access, flash)
|
||||
|
||||
|
||||
def pair_ports(config: dict[str, Any], operation: str) -> tuple[PortConfig, PortConfig]:
|
||||
sender = configured_port(config, "sender", operation)
|
||||
receiver = configured_port(config, "receiver", operation)
|
||||
if sender.device.resolve() == receiver.device.resolve():
|
||||
raise HilError("sender and receiver resolve to the same device")
|
||||
return sender, receiver
|
||||
|
||||
|
||||
def pio_command() -> str:
|
||||
for candidate in (shutil.which("pio"), shutil.which("platformio")):
|
||||
if candidate:
|
||||
return candidate
|
||||
local = Path.home() / ".platformio" / "penv" / "bin" / "pio"
|
||||
if local.is_file() and os.access(local, os.X_OK):
|
||||
return str(local)
|
||||
raise HilError("PlatformIO CLI was not found on PATH or in ~/.platformio/penv/bin/pio")
|
||||
|
||||
|
||||
def run_checked(command: list[str], *, cwd: Path = ROOT, log: Path | None = None) -> str:
|
||||
process = subprocess.run(command, cwd=cwd, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, check=False)
|
||||
if log:
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
log.write_text(process.stdout, encoding="utf-8")
|
||||
sys.stdout.write(process.stdout)
|
||||
if process.returncode:
|
||||
raise HilError(f"command failed ({process.returncode}): {' '.join(command)}")
|
||||
return process.stdout
|
||||
|
||||
|
||||
def parse_hil_line(line: bytes) -> dict[str, Any] | None:
|
||||
marker = line.find(HIL_PREFIX)
|
||||
if marker < 0:
|
||||
return None
|
||||
try:
|
||||
value = json.loads(line[marker + len(HIL_PREFIX):].decode("utf-8", "strict").strip())
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None
|
||||
return value if isinstance(value, dict) and isinstance(value.get("event"), str) else None
|
||||
|
||||
|
||||
def serial_module():
|
||||
try:
|
||||
import serial
|
||||
except ModuleNotFoundError as exc:
|
||||
raise HilError("pyserial is required; install requirements-hil.txt in a project-local virtualenv") from exc
|
||||
return serial
|
||||
|
||||
|
||||
class PairCapture:
|
||||
def __init__(self, ports: tuple[PortConfig, PortConfig], baud: int, output: Path):
|
||||
self.ports = ports
|
||||
self.baud = baud
|
||||
self.output = output
|
||||
self.stop = threading.Event()
|
||||
self.events: list[dict[str, Any]] = []
|
||||
self._lock = threading.Lock()
|
||||
self._handles: dict[str, Any] = {}
|
||||
self._threads: list[threading.Thread] = []
|
||||
|
||||
def _reader(self, port: PortConfig) -> None:
|
||||
raw_path = self.output / f"{port.name}.raw"
|
||||
log_path = self.output / f"{port.name}.log"
|
||||
handle = self._handles[port.name]
|
||||
with raw_path.open("ab", buffering=0) as raw, log_path.open("a", encoding="utf-8", buffering=1) as log:
|
||||
while not self.stop.is_set():
|
||||
line = handle.readline()
|
||||
if not line:
|
||||
continue
|
||||
host_ns = time.time_ns()
|
||||
raw.write(line)
|
||||
rendered = line.decode("utf-8", "replace").rstrip("\r\n")
|
||||
log.write(f"{datetime.fromtimestamp(host_ns / 1e9, timezone.utc).isoformat()} {rendered}\n")
|
||||
event = parse_hil_line(line)
|
||||
if event is not None:
|
||||
event["host_time_ns"] = host_ns
|
||||
event["source"] = port.name
|
||||
with self._lock:
|
||||
self.events.append(event)
|
||||
|
||||
def start(self) -> None:
|
||||
serial = serial_module()
|
||||
self.output.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
for port in self.ports:
|
||||
# Opening is deliberately delayed until approval checks have completed.
|
||||
self._handles[port.name] = serial.Serial(str(port.device), self.baud, timeout=0.25,
|
||||
write_timeout=1, exclusive=True)
|
||||
for port in self.ports:
|
||||
thread = threading.Thread(target=self._reader, args=(port,), daemon=True,
|
||||
name=f"hil-{port.name}")
|
||||
thread.start()
|
||||
self._threads.append(thread)
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def send_fault(self, target: str, fault: str) -> None:
|
||||
allowed = {"drop_chunk", "duplicate_chunk", "corrupt_chunk", "suppress_ack",
|
||||
"wrong_ack_batch", "clear"}
|
||||
if fault not in allowed and not fault.startswith("delay_ack:"):
|
||||
raise HilError(f"unsupported HIL fault: {fault}")
|
||||
handle = self._handles.get(target)
|
||||
if handle is None:
|
||||
raise HilError(f"serial target is not open: {target}")
|
||||
handle.write(f"HILCMD:{fault}\n".encode("ascii"))
|
||||
handle.flush()
|
||||
|
||||
def reset_pair(self) -> None:
|
||||
"""Pulse ESP32 EN through the normal USB auto-reset circuit.
|
||||
|
||||
DTR remains deasserted so GPIO0 is not held low and the boards boot the
|
||||
flashed application rather than the ROM download mode.
|
||||
"""
|
||||
for handle in self._handles.values():
|
||||
handle.dtr = False
|
||||
handle.rts = True
|
||||
time.sleep(0.15)
|
||||
for handle in self._handles.values():
|
||||
handle.rts = False
|
||||
time.sleep(0.25)
|
||||
|
||||
def close(self) -> None:
|
||||
self.stop.set()
|
||||
for thread in self._threads:
|
||||
thread.join(timeout=2)
|
||||
for handle in self._handles.values():
|
||||
try:
|
||||
handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._handles.clear()
|
||||
ordered = sorted(self.events, key=lambda item: item["host_time_ns"])
|
||||
with (self.output / "events.jsonl").open("w", encoding="utf-8") as stream:
|
||||
for event in ordered:
|
||||
stream.write(json.dumps(event, separators=(",", ":"), sort_keys=True) + "\n")
|
||||
|
||||
def run(self, seconds: float, on_tick: Callable[[float, "PairCapture"], None] | None = None,
|
||||
reset_on_start: bool = False) -> None:
|
||||
self.start()
|
||||
if reset_on_start:
|
||||
self.reset_pair()
|
||||
started = time.monotonic()
|
||||
try:
|
||||
while time.monotonic() - started < seconds and not self.stop.is_set():
|
||||
elapsed = time.monotonic() - started
|
||||
if on_tick:
|
||||
on_tick(elapsed, self)
|
||||
time.sleep(0.2)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
|
||||
def read_events(artifact: Path) -> list[dict[str, Any]]:
|
||||
path = artifact / "events.jsonl"
|
||||
if not path.is_file():
|
||||
return []
|
||||
events = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
events.append(item)
|
||||
return events
|
||||
|
||||
|
||||
def main_guard(function: Callable[[], int]) -> None:
|
||||
try:
|
||||
raise SystemExit(function())
|
||||
except HilError as exc:
|
||||
print(f"HIL error: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from hil_common import configured_port, load_config, main_guard, serial_module
|
||||
|
||||
|
||||
def frame(seconds: str = "00000064", include_energy: bool = True, terminator: bool = True) -> bytes:
|
||||
lines = ["/DD3HIL", f"0-0:96.8.0*255({seconds})"]
|
||||
if include_energy:
|
||||
lines.append("1-0:1.8.0*255(001234.567*kWh)")
|
||||
lines.extend([
|
||||
"1-0:16.7.0*255(000950*W)",
|
||||
"1-0:36.7.0*255(000500*W)",
|
||||
"1-0:56.7.0*255(000450*W)",
|
||||
"1-0:76.7.0*255(000000*W)",
|
||||
])
|
||||
text = "\r\n".join(lines) + "\r\n"
|
||||
if terminator:
|
||||
text += "!\r\n"
|
||||
return text.encode("ascii")
|
||||
|
||||
|
||||
def fixture(name: str) -> list[tuple[bytes, float]]:
|
||||
valid = frame()
|
||||
fixtures = {
|
||||
"valid": [(valid, 0)],
|
||||
"timeout": [(b"/DD3HIL\r\n1-0:1.8.0*255(1*kWh)", 2.0)],
|
||||
"truncated": [(valid[:40], 2.0)],
|
||||
"missing_terminator": [(frame(terminator=False), 2.0)],
|
||||
"malformed_obis": [(valid.replace(b"1-0:1.8.0", b"1-0:X.Y.Z"), 0)],
|
||||
"missing_required": [(frame(include_energy=False), 0)],
|
||||
"invalid_meter_seconds": [(frame(seconds="GGGGGGGG"), 0)],
|
||||
"timestamp_rollback": [(frame(seconds="00000100"), 0.2), (frame(seconds="00000080"), 0)],
|
||||
"timestamp_jump": [(frame(seconds="00000100"), 0.2), (frame(seconds="00001000"), 0)],
|
||||
"oversized": [(b"/" + b"A" * 600 + b"!\r\n", 0)],
|
||||
"malformed_then_valid": [(valid.replace(b"1-0:1.8.0", b"1-0:X.Y.Z"), 0.2), (valid, 0)],
|
||||
}
|
||||
if name == "slow_inter_byte":
|
||||
return [(bytes([byte]), 0.02) for byte in valid]
|
||||
return fixtures[name]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
choices = ["valid", "timeout", "truncated", "missing_terminator", "malformed_obis",
|
||||
"missing_required", "invalid_meter_seconds", "timestamp_rollback", "timestamp_jump",
|
||||
"oversized", "slow_inter_byte", "malformed_then_valid"]
|
||||
parser = argparse.ArgumentParser(description="Drive an explicitly approved isolated 9600 7E1 meter simulator")
|
||||
parser.add_argument("fixture", choices=choices)
|
||||
args = parser.parse_args()
|
||||
config = load_config()
|
||||
port = configured_port(config, "meter_simulator", "access")
|
||||
serial = serial_module()
|
||||
with serial.Serial(str(port.device), 9600, bytesize=serial.SEVENBITS, parity=serial.PARITY_EVEN,
|
||||
stopbits=serial.STOPBITS_ONE, timeout=1, write_timeout=2, exclusive=True) as handle:
|
||||
for payload, pause in fixture(args.fixture):
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
if pause:
|
||||
time.sleep(pause)
|
||||
print(f"Sent {args.fixture} through approved isolated simulator {port.device}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from hil_common import HilError, main_guard, read_events
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
name: str
|
||||
status: str
|
||||
detail: str
|
||||
|
||||
|
||||
def _events(events, name, source=None):
|
||||
return [event for event in events if event.get("event") == name and (source is None or event.get("source") == source)]
|
||||
|
||||
|
||||
def evaluate(events: list[dict], requested_fault: str | list[str] | None = None) -> list[Check]:
|
||||
checks: list[Check] = []
|
||||
roles = {event.get("source"): event.get("role") for event in _events(events, "role")}
|
||||
role_ok = roles.get("sender") == "sender" and roles.get("receiver") == "receiver"
|
||||
checks.append(Check("role mapping", "pass" if role_ok else "fail", json.dumps(roles, sort_keys=True)))
|
||||
|
||||
boot_counts = {source: len(_events(events, "boot", source)) for source in ("sender", "receiver")}
|
||||
reset_ok = all(1 <= count <= 2 for count in boot_counts.values())
|
||||
checks.append(Check("no reset loop", "pass" if reset_ok else "fail", json.dumps(boot_counts, sort_keys=True)))
|
||||
|
||||
time_ok = any(event.get("stage") == "complete" and event.get("ok") is True
|
||||
for event in _events(events, "time_bootstrap", "sender"))
|
||||
checks.append(Check("time bootstrap", "pass" if time_ok else "fail", "sender completion event captured" if time_ok else "no sender completion event"))
|
||||
|
||||
meter = _events(events, "meter_frame", "sender")
|
||||
meter_ok = any(event.get("ok") is True for event in meter)
|
||||
meter_status = "pass" if meter_ok else "blocked"
|
||||
checks.append(Check("meter parser", meter_status,
|
||||
"valid or partial meter frame captured" if meter_ok else "no valid meter evidence; meter/safe simulator unavailable or silent"))
|
||||
|
||||
health = _events(events, "health")
|
||||
checks.append(Check("runtime diagnostics", "pass" if len(health) >= 2 else "fail", f"{len(health)} health events"))
|
||||
|
||||
created = {event.get("batch_id") for event in _events(events, "batch_created", "sender")}
|
||||
complete = {event.get("batch_id") for event in _events(events, "reassembly_complete", "receiver")}
|
||||
decoded = {event.get("batch_id") for event in _events(events, "payload_decode", "receiver") if event.get("ok") is True}
|
||||
ack_tx = {event.get("batch_id") for event in _events(events, "ack_tx", "receiver") if event.get("ok") is True}
|
||||
ack_rx = {event.get("batch_id") for event in _events(events, "ack_rx", "sender") if event.get("ok") is True}
|
||||
correlated = sorted((created & complete & decoded & ack_tx & ack_rx) - {None})
|
||||
if correlated:
|
||||
batch_status = "pass"
|
||||
batch_detail = f"correlated batch IDs: {correlated}"
|
||||
elif not created and not meter_ok:
|
||||
batch_status = "blocked"
|
||||
batch_detail = "no valid meter input, so publication was correctly suppressed"
|
||||
else:
|
||||
batch_status = "fail"
|
||||
batch_detail = f"correlated batch IDs: {correlated}"
|
||||
checks.append(Check("batch and ACK correlation", batch_status, batch_detail))
|
||||
|
||||
requested_faults = ([requested_fault] if isinstance(requested_fault, str) else requested_fault) or []
|
||||
for fault in requested_faults:
|
||||
injected = [event for event in _events(events, "fault_injected") if event.get("fault") == fault]
|
||||
injection_ns = min((event.get("host_time_ns", 0) for event in injected), default=0)
|
||||
later_injections = [event.get("host_time_ns", 0) for event in _events(events, "fault_injected")
|
||||
if event.get("host_time_ns", 0) > injection_ns]
|
||||
window_end = min(later_injections) if later_injections else 2**63 - 1
|
||||
in_window = lambda event: injection_ns < event.get("host_time_ns", 0) < window_end
|
||||
recovered = any(event.get("event") == "ack_rx" and event.get("ok") is True and in_window(event)
|
||||
for event in events)
|
||||
details = [f"injected={bool(injected)}", f"later successful ACK={recovered}"]
|
||||
expected = True
|
||||
if fault in {"drop_chunk", "corrupt_chunk", "suppress_ack", "wrong_ack_batch"}:
|
||||
retry = any(event.get("event") == "retry" and in_window(event) for event in events)
|
||||
details.append(f"retry={retry}")
|
||||
expected = retry
|
||||
if fault in {"drop_chunk", "corrupt_chunk"} and retry:
|
||||
first_retry_ns = min(event.get("host_time_ns", 0) for event in events
|
||||
if event.get("event") == "retry" and in_window(event))
|
||||
premature_decode = any(event.get("event") == "payload_decode" and event.get("ok") is True and
|
||||
injection_ns < event.get("host_time_ns", 0) < first_retry_ns
|
||||
for event in events)
|
||||
details.append(f"decoded before retry={premature_decode}")
|
||||
expected = expected and not premature_decode
|
||||
if fault == "duplicate_chunk":
|
||||
duplicate_detected = any(event.get("event") == "duplicate" and in_window(event) for event in events)
|
||||
details.append(f"receiver duplicate detection={duplicate_detected}")
|
||||
expected = duplicate_detected
|
||||
if fault == "corrupt_chunk":
|
||||
crc_reject = any(event.get("event") == "lora_reject" and event.get("reason") == "crc_fail" and
|
||||
in_window(event) for event in events)
|
||||
details.append(f"CRC rejection={crc_reject}")
|
||||
expected = expected and crc_reject
|
||||
if fault == "wrong_ack_batch":
|
||||
wrong_reject = any(event.get("event") == "ack_rx" and event.get("ok") is False and
|
||||
in_window(event) for event in events)
|
||||
details.append(f"wrong ACK rejected={wrong_reject}")
|
||||
expected = expected and wrong_reject
|
||||
passed = bool(injected) and recovered and expected
|
||||
checks.append(Check(f"fault {fault}", "pass" if passed else "fail", ", ".join(details)))
|
||||
return checks
|
||||
|
||||
|
||||
def write_report(artifact: Path, checks: list[Check], events: list[dict]) -> None:
|
||||
lines = ["# DD3 local HIL report", "", f"Artifact directory: `{artifact}`", "", "## Results", ""]
|
||||
for check in checks:
|
||||
lines.append(f"- **{check.status.upper()}** — {check.name}: {check.detail}")
|
||||
lines.extend(["", "## Evidence summary", ""])
|
||||
resets = [{"source": e.get("source"), "reset_reason": e.get("reset_reason")} for e in _events(events, "boot")]
|
||||
lines.append(f"- Reset evidence: `{json.dumps(resets, sort_keys=True)}`")
|
||||
lines.append(f"- Meter classifications: `{json.dumps([e.get('classification') for e in _events(events, 'meter_frame')])}`")
|
||||
lines.append(f"- Retry events: `{len(_events(events, 'retry'))}`")
|
||||
queue_drops = [event for event in _events(events, "queue_drop") if event.get("inflight") is not True]
|
||||
lines.append(f"- Queue drops: `{len(queue_drops)}`")
|
||||
heaps = [e.get("heap_free") for e in _events(events, "health") if isinstance(e.get("heap_free"), int)]
|
||||
lines.append(f"- Minimum observed free heap: `{min(heaps) if heaps else 'not captured'}`")
|
||||
lines.extend(["", "Raw serial bytes and timestamped text logs are retained beside this report.", ""])
|
||||
(artifact / "report.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
suite = ET.Element("testsuite", name="dd3-hil", tests=str(len(checks)),
|
||||
failures=str(sum(c.status == "fail" for c in checks)),
|
||||
skipped=str(sum(c.status == "blocked" for c in checks)))
|
||||
for check in checks:
|
||||
case = ET.SubElement(suite, "testcase", name=check.name, classname="hil")
|
||||
if check.status == "fail":
|
||||
ET.SubElement(case, "failure", message=check.detail)
|
||||
elif check.status == "blocked":
|
||||
ET.SubElement(case, "skipped", message=check.detail)
|
||||
ET.SubElement(case, "system-out").text = check.detail
|
||||
ET.ElementTree(suite).write(artifact / "junit.xml", encoding="utf-8", xml_declaration=True)
|
||||
|
||||
|
||||
def generate(artifact: Path, requested_fault: str | list[str] | None = None) -> list[Check]:
|
||||
events = read_events(artifact)
|
||||
if not events:
|
||||
raise HilError(f"no parsed HIL events in {artifact / 'events.jsonl'}")
|
||||
checks = evaluate(events, requested_fault)
|
||||
write_report(artifact, checks, events)
|
||||
return checks
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate Markdown and JUnit reports from captured HIL evidence")
|
||||
parser.add_argument("artifact_dir", type=Path)
|
||||
parser.add_argument("--fault", action="append")
|
||||
args = parser.parse_args()
|
||||
checks = generate(args.artifact_dir, args.fault)
|
||||
for check in checks:
|
||||
print(f"{check.status.upper():7} {check.name}: {check.detail}")
|
||||
return 1 if any(check.status == "fail" for check in checks) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from flash_pair import flash_pair
|
||||
from hil_common import PairCapture, load_config, main_guard, new_artifact_dir, pair_ports
|
||||
from report import generate
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build, flash, capture, and verify one DD3 HIL run")
|
||||
parser.add_argument("--yes-flash", action="store_true")
|
||||
parser.add_argument("--seconds", type=float)
|
||||
parser.add_argument("--fault", choices=["drop_chunk", "duplicate_chunk", "corrupt_chunk", "suppress_ack",
|
||||
"delay_ack", "wrong_ack_batch"])
|
||||
parser.add_argument("--fault-after", type=float, default=40.0)
|
||||
args = parser.parse_args()
|
||||
config = load_config()
|
||||
ports = pair_ports(config, "flash")
|
||||
artifact = new_artifact_dir("baseline" if not args.fault else f"fault-{args.fault}")
|
||||
flash_pair(artifact, args.yes_flash)
|
||||
baud = int(config.get("hil", {}).get("baud", 115200))
|
||||
seconds = args.seconds or float(config.get("hil", {}).get("capture_seconds", 180))
|
||||
sent = False
|
||||
|
||||
def tick(elapsed, capture):
|
||||
nonlocal sent
|
||||
if args.fault and not sent and elapsed >= args.fault_after:
|
||||
target = "sender" if args.fault.endswith("chunk") else "receiver"
|
||||
command = "delay_ack:1000" if args.fault == "delay_ack" else args.fault
|
||||
capture.send_fault(target, command)
|
||||
sent = True
|
||||
|
||||
PairCapture(ports, baud, artifact).run(seconds, tick, reset_on_start=True)
|
||||
checks = generate(artifact, args.fault)
|
||||
print(f"HIL report: {artifact / 'report.md'}")
|
||||
return 1 if any(check.status == "fail" for check in checks) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
|
||||
from hil_common import PairCapture, load_config, main_guard, new_artifact_dir, pair_ports
|
||||
from report import generate
|
||||
|
||||
|
||||
FAULTS = [
|
||||
"drop_chunk",
|
||||
"duplicate_chunk",
|
||||
"corrupt_chunk",
|
||||
"suppress_ack",
|
||||
"delay_ack",
|
||||
"wrong_ack_batch",
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run all one-shot LoRa HIL faults in one approved capture")
|
||||
parser.add_argument("--fault", action="append", choices=FAULTS,
|
||||
help="run only this fault (repeatable); defaults to all")
|
||||
parser.add_argument("--seconds", type=float, default=600)
|
||||
parser.add_argument("--arm-after", type=float, default=10)
|
||||
parser.add_argument("--case-timeout", type=float, default=100)
|
||||
args = parser.parse_args()
|
||||
faults = args.fault or FAULTS
|
||||
config = load_config()
|
||||
ports = pair_ports(config, "access")
|
||||
baud = int(config.get("hil", {}).get("baud", 115200))
|
||||
artifact = new_artifact_dir("fault-suite")
|
||||
state = {"index": 0, "armed_at": None, "last_command_at": None, "armed_confirmed": False,
|
||||
"injected_ns": None, "recovered_at": None, "cases": []}
|
||||
|
||||
def arm(capture: PairCapture, now: float) -> None:
|
||||
fault = faults[state["index"]]
|
||||
target = "sender" if fault.endswith("chunk") else "receiver"
|
||||
command = "delay_ack:1000" if fault == "delay_ack" else fault
|
||||
capture.send_fault(target, command)
|
||||
state["armed_at"] = now
|
||||
state["last_command_at"] = now
|
||||
state["armed_confirmed"] = False
|
||||
state["injected_ns"] = None
|
||||
state["recovered_at"] = None
|
||||
print(f"Armed {fault} on {target}", flush=True)
|
||||
|
||||
def finish_case(capture: PairCapture, now: float, recovered: bool) -> None:
|
||||
fault = faults[state["index"]]
|
||||
state["cases"].append({"fault": fault, "recovered": recovered})
|
||||
state["index"] += 1
|
||||
if state["index"] >= len(faults):
|
||||
capture.stop.set()
|
||||
return
|
||||
arm(capture, now)
|
||||
|
||||
def tick(elapsed: float, capture: PairCapture) -> None:
|
||||
now = time.monotonic()
|
||||
if state["index"] >= len(faults):
|
||||
capture.stop.set()
|
||||
return
|
||||
if state["armed_at"] is None:
|
||||
if elapsed >= args.arm_after:
|
||||
arm(capture, now)
|
||||
return
|
||||
fault = faults[state["index"]]
|
||||
snapshot = list(capture.events)
|
||||
armed = [event for event in snapshot if event.get("event") == "fault_armed" and
|
||||
event.get("fault") == fault]
|
||||
if armed:
|
||||
state["armed_confirmed"] = True
|
||||
if not state["armed_confirmed"] and now - state["last_command_at"] >= 5:
|
||||
target = "sender" if fault.endswith("chunk") else "receiver"
|
||||
command = "delay_ack:1000" if fault == "delay_ack" else fault
|
||||
capture.send_fault(target, command)
|
||||
state["last_command_at"] = now
|
||||
print(f"Re-sent {fault}; awaiting device arm acknowledgement", flush=True)
|
||||
injections = [event for event in snapshot if event.get("event") == "fault_injected" and
|
||||
event.get("fault") == fault]
|
||||
if injections and state["injected_ns"] is None:
|
||||
state["injected_ns"] = min(event.get("host_time_ns", 0) for event in injections)
|
||||
if state["injected_ns"]:
|
||||
recovered = any(event.get("event") == "ack_rx" and event.get("ok") is True and
|
||||
event.get("host_time_ns", 0) > state["injected_ns"] for event in snapshot)
|
||||
if recovered:
|
||||
if state["recovered_at"] is None:
|
||||
state["recovered_at"] = now
|
||||
elif now - state["recovered_at"] >= 3:
|
||||
finish_case(capture, now, True)
|
||||
return
|
||||
if now - state["armed_at"] >= args.case_timeout:
|
||||
finish_case(capture, now, False)
|
||||
|
||||
PairCapture(ports, baud, artifact).run(args.seconds, tick, reset_on_start=True)
|
||||
(artifact / "fault-suite.json").write_text(json.dumps(state["cases"], indent=2) + "\n", encoding="utf-8")
|
||||
checks = generate(artifact, faults)
|
||||
print(f"Fault-suite report: {artifact / 'report.md'}")
|
||||
return 1 if any(check.status == "fail" for check in checks) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_guard(main)
|
||||
Reference in New Issue
Block a user