2 Commits

Author SHA1 Message Date
b26206eb96 Introduce watchdog and serialization improvements
- Added watchdog timer for improved system stability and responsiveness.
- Switched save data serialization to Bincode for better efficiency.
- Enhanced compatibility by supporting fallback to older JSON format.
- Improved logging during flash operations for easier debugging.
- Simplified SavegameManager by managing storage directly.
2026-04-12 20:38:52 +02:00
95f7488fa3 Add save timestamp support and log interceptor for enhanced debugging
- Introduced `created_at` metadata for saves, enabling timestamp tracking.
- Added `InterceptorLogger` to capture logs, aiding in error diagnostics.
- Updated web UI to display save creation timestamps.
- Improved save/load functionality to maintain compatibility with older formats.
2026-04-11 22:40:25 +02:00
12 changed files with 247 additions and 68 deletions

View File

@@ -2,5 +2,6 @@
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/../../.." vcs="Git" />
<mapping directory="$PROJECT_DIR$/../../../website/themes/blowfish" vcs="Git" />
</component>
</project>

View File

@@ -140,7 +140,8 @@ macro_rules! mk_static {
impl Esp<'_> {
pub fn get_time(&self) -> DateTime<Utc> {
DateTime::from_timestamp_micros(self.rtc.current_time_us() as i64).unwrap_or(DateTime::UNIX_EPOCH)
DateTime::from_timestamp_micros(self.rtc.current_time_us() as i64)
.unwrap_or(DateTime::UNIX_EPOCH)
}
pub fn set_time(&mut self, time: DateTime<Utc>) {
@@ -517,10 +518,7 @@ impl Esp<'_> {
Ok(*stack)
}
pub fn deep_sleep(
&mut self,
duration_in_ms: u64,
) -> ! {
pub fn deep_sleep(&mut self, duration_in_ms: u64) -> ! {
// Mark the current OTA image as valid if we reached here while in pending verify.
if let Ok(cur) = self.ota.current_ota_state() {
if cur == OtaImageState::PendingVerify {
@@ -556,22 +554,24 @@ impl Esp<'_> {
}
/// Load a config from a specific save slot.
pub(crate) async fn load_config_slot(
&mut self,
idx: usize,
) -> FatResult<String> {
pub(crate) async fn load_config_slot(&mut self, idx: usize) -> FatResult<String> {
match self.savegame.load_slot(idx)? {
None => bail!("Slot {idx} is empty or invalid"),
Some(data) => {
Ok(String::from_utf8_lossy(&data).to_string())
}
Some(data) => Ok(String::from_utf8_lossy(&data).to_string()),
}
}
/// Persist a JSON config blob to the next wear-leveling slot.
pub(crate) async fn save_config(&mut self, mut config: Vec<u8>) -> FatResult<()> {
self.savegame.save(config.as_mut_slice())?;
Ok(())
/// Retries once on flash error.
pub(crate) async fn save_config(&mut self, config: Vec<u8>) -> FatResult<()> {
let timestamp = self.get_time().to_rfc3339();
match self.savegame.save(config.as_slice(), &timestamp) {
Ok(()) => Ok(()),
Err(e) => {
warn!("First save attempt failed: {e:?}. Retrying...");
self.savegame.save(config.as_slice(), &timestamp)
}
}
}
/// Delete a specific save slot by erasing it on flash.
@@ -607,7 +607,14 @@ impl Esp<'_> {
if to_config_mode {
RESTART_TO_CONF = 1;
}
log(LogMessage::RestartToConfig, RESTART_TO_CONF as u32, 0, "", "").await;
log(
LogMessage::RestartToConfig,
RESTART_TO_CONF as u32,
0,
"",
"",
)
.await;
log(
LogMessage::LowVoltage,
LOW_VOLTAGE_DETECTED as u32,
@@ -701,6 +708,7 @@ impl Esp<'_> {
let mqtt_timeout = 15000;
let res = async {
while !MQTT_CONNECTED_EVENT_RECEIVED.load(Ordering::Relaxed) {
crate::hal::PlantHal::feed_watchdog();
Timer::after(Duration::from_millis(100)).await;
}
Ok::<(), FatError>(())
@@ -719,6 +727,7 @@ impl Esp<'_> {
let res = async {
while !MQTT_ROUND_TRIP_RECEIVED.load(Ordering::Relaxed) {
crate::hal::PlantHal::feed_watchdog();
Timer::after(Duration::from_millis(100)).await;
}
Ok::<(), FatError>(())

View File

@@ -3,8 +3,8 @@ use lib_bms_protocol::BmsReadable;
pub(crate) mod battery;
// mod can_api; // replaced by external canapi crate
pub mod esp;
pub(crate) mod savegame_manager;
pub(crate) mod rtc;
pub(crate) mod savegame_manager;
mod shared_flash;
mod v4_hal;
mod water;
@@ -85,7 +85,7 @@ use esp_alloc as _;
use esp_backtrace as _;
use esp_bootloader_esp_idf::ota::{Ota, OtaImageState};
use esp_hal::delay::Delay;
use esp_hal::i2c::master::{BusTimeout, Config, I2c};
use esp_hal::i2c::master::{BusTimeout, Config, FsmTimeout, I2c, SoftwareTimeout};
use esp_hal::interrupt::software::SoftwareInterruptControl;
use esp_hal::pcnt::unit::Unit;
use esp_hal::pcnt::Pcnt;
@@ -93,7 +93,7 @@ use esp_hal::rng::Rng;
use esp_hal::rtc_cntl::{Rtc, SocResetReason};
use esp_hal::system::reset_reason;
use esp_hal::time::Rate;
use esp_hal::timer::timg::TimerGroup;
use esp_hal::timer::timg::{MwdtStage, TimerGroup, Wdt};
use esp_hal::uart::Uart;
use esp_hal::Blocking;
use esp_radio::{init, Controller};
@@ -108,6 +108,13 @@ pub const PLANT_COUNT: usize = 8;
pub static PROGRESS_ACTIVE: AtomicBool = AtomicBool::new(false);
pub static WATCHDOG: OnceLock<
embassy_sync::blocking_mutex::Mutex<
CriticalSectionRawMutex,
RefCell<Wdt<esp_hal::peripherals::TIMG0>>,
>,
> = OnceLock::new();
const TANK_MULTI_SAMPLE: usize = 11;
pub static I2C_DRIVER: OnceLock<
embassy_sync::blocking_mutex::Mutex<CriticalSectionRawMutex, RefCell<I2c<Blocking>>>,
@@ -174,6 +181,9 @@ pub trait BoardInteraction<'a> {
// Indicate progress is active to suppress default wait_infinity blinking
PROGRESS_ACTIVE.store(true, core::sync::atomic::Ordering::Relaxed);
// Feed watchdog during long-running webserver operations
PlantHal::feed_watchdog();
let current = counter % PLANT_COUNT as u32;
for led in 0..PLANT_COUNT {
if let Err(err) = self.fault(led, current == led as u32).await {
@@ -251,6 +261,16 @@ impl PlantHal {
let sw_int = SoftwareInterruptControl::new(peripherals.SW_INTERRUPT);
esp_rtos::start(timg0.timer0, sw_int.software_interrupt0);
// Initialize and enable the watchdog with 30 second timeout
let mut wdt = timg0.wdt;
wdt.set_timeout(MwdtStage::Stage0, esp_hal::time::Duration::from_secs(30));
wdt.enable();
WATCHDOG
.init(embassy_sync::blocking_mutex::Mutex::new(RefCell::new(wdt)))
.map_err(|_| FatError::String {
error: "Watchdog already initialized".to_string(),
})?;
let boot_button = Input::new(
peripherals.GPIO9,
InputConfig::default().with_pull(Pull::None),
@@ -380,7 +400,11 @@ impl PlantHal {
);
let savegame = SavegameManager::new(data);
info!("Savegame storage initialized ({} slots × {} KB)", savegame_manager::SAVEGAME_SLOT_COUNT, savegame_manager::SAVEGAME_SLOT_SIZE / 1024);
info!(
"Savegame storage initialized ({} slots × {} KB)",
savegame_manager::SAVEGAME_SLOT_COUNT,
savegame_manager::SAVEGAME_SLOT_SIZE / 1024
);
let uart0 =
Uart::new(peripherals.UART0, UartConfig::default()).map_err(|_| FatError::String {
@@ -458,11 +482,16 @@ impl PlantHal {
let sda = peripherals.GPIO20;
let scl = peripherals.GPIO19;
// Configure I2C with 1-second timeout
// At 100 Hz I2C clock, one bus cycle = 10ms
// For 1 second timeout: 100 bus cycles
let i2c = I2c::new(
peripherals.I2C0,
Config::default()
.with_frequency(Rate::from_hz(100))
.with_timeout(BusTimeout::Maximum),
//.with_frequency(Rate::from_hz(100))
//1s at 100khz
.with_timeout(BusTimeout::BusCycles(100_000))
.with_scl_main_st_timeout(FsmTimeout::new(21)?),
)?
.with_scl(scl)
.with_sda(sda);
@@ -563,6 +592,15 @@ impl PlantHal {
Ok(Mutex::new(hal))
}
/// Feed the watchdog timer to prevent system reset
pub fn feed_watchdog() {
if let Some(wdt_mutex) = WATCHDOG.try_get() {
wdt_mutex.lock(|cell| {
cell.borrow_mut().feed();
});
}
}
}
fn ota_state(

View File

@@ -1,8 +1,10 @@
use alloc::vec::Vec;
use bincode::{Decode, Encode};
use embedded_savegame::storage::{Flash, Storage};
use embedded_storage::nor_flash::{NorFlash, ReadNorFlash};
use esp_bootloader_esp_idf::partitions::{FlashRegion, Error as PartitionError};
use serde::Serialize;
use esp_bootloader_esp_idf::partitions::{Error as PartitionError, Error, FlashRegion};
use log::{error, info};
use serde::{Deserialize, Serialize};
use crate::fat_error::{FatError, FatResult};
use crate::hal::shared_flash::MutexFlashStorage;
@@ -19,6 +21,17 @@ pub const SAVEGAME_SLOT_COUNT: usize = (8 * 1024 * 1024) / SAVEGAME_SLOT_SIZE -
pub struct SaveInfo {
pub idx: usize,
pub len: u32,
/// UTC timestamp in RFC3339 format when the save was created
pub created_at: Option<alloc::string::String>,
}
/// Wrapper that includes both the config data and metadata like creation timestamp.
#[derive(Serialize, Deserialize, Debug, Encode, Decode)]
struct SaveWrapper {
/// UTC timestamp in RFC3339 format
created_at: alloc::string::String,
/// Raw config JSON data
data: Vec<u8>,
}
// ── Flash adapter ──────────────────────────────────────────────────────────────
@@ -60,7 +73,21 @@ impl Flash for SavegameFlashAdapter<'_> {
// Align end address up to erase boundary
let end = addr + SAVEGAME_SLOT_SIZE as u32;
let aligned_end = ((end + ERASE_SIZE - 1) / ERASE_SIZE) * ERASE_SIZE;
NorFlash::erase(self.region, aligned_start, aligned_end).map_err(SavegameFlashError)
if aligned_start != addr || aligned_end != end {
log::warn!("Flash erase address not aligned: addr=0x{:x}, slot_size=0x{:x}. Aligned to 0x{:x}-0x{:x}", addr, SAVEGAME_SLOT_SIZE, aligned_start, aligned_end);
}
match NorFlash::erase(self.region, aligned_start, aligned_end) {
Ok(_) => Ok(()),
Err(err) => {
error!(
"Flash erase failed: {:?}. 0x{:x}-0x{:x}",
err, aligned_start, aligned_end
);
Err(SavegameFlashError(err))
}
}
}
}
@@ -77,46 +104,59 @@ impl From<SavegameFlashError> for FatError {
/// High-level save-game manager that stores JSON config blobs on the storage
/// partition using [`embedded_savegame`] for wear leveling and power-fail safety.
pub struct SavegameManager {
region: &'static mut FlashRegion<'static, MutexFlashStorage>,
storage: Storage<SavegameFlashAdapter<'static>, SAVEGAME_SLOT_SIZE, SAVEGAME_SLOT_COUNT>,
}
impl SavegameManager {
pub fn new(region: &'static mut FlashRegion<'static, MutexFlashStorage>) -> Self {
Self { region }
Self {
storage: Storage::new(SavegameFlashAdapter { region }),
}
}
/// Build a short-lived [`Storage`] that borrows our flash region.
fn storage(
&mut self,
) -> Storage<SavegameFlashAdapter<'_>, SAVEGAME_SLOT_SIZE, SAVEGAME_SLOT_COUNT> {
Storage::new(SavegameFlashAdapter {
region: &mut *self.region,
})
}
/// Persist `data` (JSON bytes) to the next available slot.
/// Persist `data` (JSON bytes) to the next available slot with a UTC timestamp.
///
/// `scan()` advances the internal wear-leveling pointer to the latest valid
/// slot before `append()` writes to the next free one.
/// Both operations are performed atomically on the same Storage instance.
pub fn save(&mut self, data: &mut [u8]) -> FatResult<()> {
let mut st = self.storage();
let _slot = st.scan()?;
st.append(data)?;
pub fn save(&mut self, data: &[u8], timestamp: &str) -> FatResult<()> {
let wrapper = SaveWrapper {
created_at: alloc::string::String::from(timestamp),
data: data.to_vec(),
};
let mut serialized = bincode::encode_to_vec(&wrapper, bincode::config::standard())?;
info!("Serialized config with size {}", serialized.len());
(&mut self.storage).append(&mut serialized)?;
Ok(())
}
/// Load the most recently saved data. Returns `None` if no valid save exists.
/// Unwraps the SaveWrapper and returns only the config data.
pub fn load_latest(&mut self) -> FatResult<Option<Vec<u8>>> {
let mut st = self.storage();
let slot = st.scan()?;
let slot = (&mut self.storage).scan()?;
match slot {
None => Ok(None),
Some(slot) => {
let mut buf = alloc::vec![0u8; SAVEGAME_SLOT_SIZE];
match st.read(slot.idx, &mut buf)? {
match (&mut self.storage).read(slot.idx, &mut buf)? {
None => Ok(None),
Some(data) => Ok(Some(data.to_vec())),
Some(data) => {
// Try to deserialize as SaveWrapper (new Bincode format)
match bincode::decode_from_slice::<SaveWrapper, _>(
data,
bincode::config::standard(),
) {
Ok((wrapper, _)) => Ok(Some(wrapper.data)),
Err(_) => {
// Fallback to JSON SaveWrapper (intermediate format)
match serde_json::from_slice::<SaveWrapper>(data) {
Ok(wrapper) => Ok(Some(wrapper.data)),
// Fallback to raw data for backwards compatibility
Err(_) => Ok(Some(data.to_vec())),
}
}
}
}
}
}
}
@@ -124,32 +164,64 @@ impl SavegameManager {
/// Load a specific save by slot index. Returns `None` if the slot is
/// empty or contains an invalid checksum.
/// Unwraps the SaveWrapper and returns only the config data.
pub fn load_slot(&mut self, idx: usize) -> FatResult<Option<Vec<u8>>> {
let mut st = self.storage();
let mut buf = alloc::vec![0u8; SAVEGAME_SLOT_SIZE];
match st.read(idx, &mut buf)? {
match (&mut self.storage).read(idx, &mut buf)? {
None => Ok(None),
Some(data) => Ok(Some(data.to_vec())),
Some(data) => {
// Try to deserialize as SaveWrapper (new Bincode format)
match bincode::decode_from_slice::<SaveWrapper, _>(
data,
bincode::config::standard(),
) {
Ok((wrapper, _)) => Ok(Some(wrapper.data)),
Err(_) => {
// Fallback to JSON SaveWrapper (intermediate format)
match serde_json::from_slice::<SaveWrapper>(data) {
Ok(wrapper) => Ok(Some(wrapper.data)),
// Fallback to raw data for backwards compatibility
Err(_) => Ok(Some(data.to_vec())),
}
}
}
}
}
}
/// Erase a specific slot by index, effectively deleting it.
pub fn delete_slot(&mut self, idx: usize) -> FatResult<()> {
let mut st = self.storage();
st.erase(idx).map_err(Into::into)
(&mut self.storage).erase(idx).map_err(Into::into)
}
/// Iterate all slots and return metadata for every slot that contains a
/// valid save, using the Storage read API to avoid assuming internal slot structure.
/// Extracts timestamps from SaveWrapper if available.
pub fn list_saves(&mut self) -> FatResult<Vec<SaveInfo>> {
let mut saves = Vec::new();
let mut st = self.storage();
let mut buf = alloc::vec![0u8; SAVEGAME_SLOT_SIZE];
for idx in 0..SAVEGAME_SLOT_COUNT {
if let Some(data) = st.read(idx, &mut buf)? {
if let Some(data) = (&mut self.storage).read(idx, &mut buf)? {
// Try to deserialize as SaveWrapper (new Bincode format)
let (len, created_at) = match bincode::decode_from_slice::<SaveWrapper, _>(
data,
bincode::config::standard(),
) {
Ok((wrapper, _)) => (wrapper.data.len() as u32, Some(wrapper.created_at)),
Err(_) => {
// Fallback to JSON SaveWrapper (intermediate format)
match serde_json::from_slice::<SaveWrapper>(data) {
Ok(wrapper) => (wrapper.data.len() as u32, Some(wrapper.created_at)),
// Old format without timestamp
Err(_) => (data.len() as u32, None),
}
}
};
saves.push(SaveInfo {
idx,
len: data.len() as u32,
len,
created_at,
});
}
}

View File

@@ -1,5 +1,5 @@
use crate::BOARD_ACCESS;
use crate::vec;
use crate::BOARD_ACCESS;
use alloc::string::ToString;
use alloc::vec::Vec;
use bytemuck::{AnyBitPattern, Pod, Zeroable};
@@ -33,6 +33,12 @@ static mut LOG_ARRAY: LogArray = LogArray {
pub static LOG_ACCESS: Mutex<CriticalSectionRawMutex, &'static mut LogArray> =
unsafe { Mutex::new(&mut LOG_ARRAY) };
mod interceptor;
pub use interceptor::InterceptorLogger;
pub static INTERCEPTOR: InterceptorLogger = InterceptorLogger::new();
pub struct LogRequest {
pub message_key: LogMessage,
pub number_a: u32,

View File

@@ -167,7 +167,11 @@ async fn safe_main(spawner: Spawner) -> FatResult<()> {
let cur = match board.board_hal.get_rtc_module().get_rtc_time().await {
Ok(value) => {
{
board.board_hal.get_esp().rtc.set_current_time_us(value.timestamp_micros() as u64);
board
.board_hal
.get_esp()
.rtc
.set_current_time_us(value.timestamp_micros() as u64);
}
value
}
@@ -332,7 +336,14 @@ async fn safe_main(spawner: Spawner) -> FatResult<()> {
match err {
TankError::SensorDisabled => { /* unreachable */ }
TankError::SensorMissing(raw_value_mv) => {
log(LogMessage::TankSensorMissing, raw_value_mv as u32, 0, "", "").await
log(
LogMessage::TankSensorMissing,
raw_value_mv as u32,
0,
"",
"",
)
.await
}
TankError::SensorValueError { value, min, max } => {
log(
@@ -455,7 +466,7 @@ async fn safe_main(spawner: Spawner) -> FatResult<()> {
pump_ineffective,
result.median_current_ma,
result.max_current_ma,
result.min_current_ma
result.min_current_ma,
)
.await;
} else if !state.pump_in_timeout(plant_config, &timezone_time) {
@@ -738,6 +749,7 @@ pub async fn do_secure_pump(
}
None => Duration::from_millis(1),
};
hal::PlantHal::feed_watchdog();
Timer::after(sleep_time).await;
pump_time_ms += 50;
}
@@ -907,7 +919,7 @@ async fn pump_info(
pump_ineffective: bool,
median_current_ma: u16,
max_current_ma: u16,
min_current_ma: u16
min_current_ma: u16,
) {
let pump_info = PumpInfo {
enabled: pump_active,
@@ -920,7 +932,11 @@ async fn pump_info(
match serde_json::to_string(&pump_info) {
Ok(state) => {
board.board_hal.get_esp().mqtt_publish(&pump_topic, &state).await;
board
.board_hal
.get_esp()
.mqtt_publish(&pump_topic, &state)
.await;
}
Err(err) => {
warn!("Error publishing pump state {err}");
@@ -1104,6 +1120,8 @@ async fn wait_infinity(
Timer::after_millis(delay).await;
hal::PlantHal::feed_watchdog();
if wait_type == WaitType::MqttConfig && !MQTT_STAY_ALIVE.load(Ordering::Relaxed) {
reboot_now.store(true, Ordering::Relaxed);
}
@@ -1167,7 +1185,7 @@ use embassy_time::WithTimeout;
#[esp_rtos::main]
async fn main(spawner: Spawner) -> ! {
// intialize embassy
logger::init_logger_from_env();
crate::log::INTERCEPTOR.init();
spawner.must_spawn(crate::log::log_task());
//force init here!
match BOARD_ACCESS.init(

View File

@@ -10,8 +10,8 @@ mod post_json;
use crate::fat_error::{FatError, FatResult};
use crate::webserver::backup_manager::{backup_config, backup_info, get_backup_config};
use crate::webserver::get_json::{
get_battery_state, get_config, get_live_moisture, get_log_localization_config, get_solar_state,
delete_save, get_time, get_timezones, get_version_web, list_saves, tank_info,
delete_save, get_battery_state, get_config, get_live_moisture, get_log_localization_config,
get_solar_state, get_time, get_timezones, get_version_web, list_saves, tank_info,
};
use crate::webserver::get_log::get_log;
use crate::webserver::get_static::{serve_bundle, serve_favicon, serve_index};
@@ -64,6 +64,7 @@ impl Handler for HTTPRequestRouter {
e
})?
} else {
crate::log::INTERCEPTOR.start_capture().await;
match method {
Method::Get => match path {
"/favicon.ico" => serve_favicon(conn).await?,
@@ -138,7 +139,9 @@ impl Handler for HTTPRequestRouter {
.and_then(|s| s.parse().ok());
match idx {
Some(idx) => Some(delete_save(conn, idx).await),
None => Some(Err(FatError::String { error: "missing idx parameter".into() })),
None => Some(Err(FatError::String {
error: "missing idx parameter".into(),
})),
}
} else {
None
@@ -164,6 +167,7 @@ impl Handler for HTTPRequestRouter {
let response_time = Instant::now().duration_since(start).as_millis();
info!("\"{method} {path}\" {code} {response_time}ms");
crate::log::INTERCEPTOR.stop_capture().await;
Ok(())
}
}
@@ -261,8 +265,17 @@ where
}
},
Err(err) => {
let error_text = err.to_string();
let mut error_text = err.to_string();
info!("error handling process {error_text}");
if let Some(logs) = crate::log::INTERCEPTOR.stop_capture().await {
error_text.push_str("\n\nCaptured Logs:\n");
for log in logs {
error_text.push_str(&log);
error_text.push('\n');
}
}
conn.initiate_response(
500,
Some("OK"),

View File

@@ -37,6 +37,7 @@ export interface NetworkConfig {
export interface SaveInfo {
idx: number,
len: number,
created_at: string | null,
}
export interface FileList {

View File

@@ -33,6 +33,7 @@ class SaveEntry {
this.view.innerHTML = template.replaceAll("${fileid}", String(fileid));
let name = document.getElementById("file_" + fileid + "_name") as HTMLElement;
let created = document.getElementById("file_" + fileid + "_created") as HTMLElement;
let size = document.getElementById("file_" + fileid + "_size") as HTMLElement;
let deleteBtn = document.getElementById("file_" + fileid + "_delete") as HTMLButtonElement;
deleteBtn.onclick = () => {
@@ -45,5 +46,17 @@ class SaveEntry {
name.innerText = "Slot " + saveinfo.idx;
size.innerText = saveinfo.len + " bytes";
// Format timestamp in browser's local timezone
if (saveinfo.created_at) {
try {
const date = new Date(saveinfo.created_at);
created.innerText = date.toLocaleString();
} catch (e) {
created.innerText = "Invalid date";
}
} else {
created.innerText = "Unknown";
}
}
}

View File

@@ -2,6 +2,11 @@
<div id="file_${fileid}_name" class="filetitle">Slot</div>
</div>
<div class="flexcontainer">
<div class="filekey">Created</div>
<div id="file_${fileid}_created" class="filevalue"></div>
</div>
<div class="flexcontainer">
<div class="filekey">Size</div>
<div id="file_${fileid}_size" class="filevalue"></div>

View File

@@ -240,7 +240,10 @@ export class Controller {
.then(_ => {
controller.progressview.removeProgress("set_config");
setTimeout(() => {
controller.downloadConfig()
controller.downloadConfig().then(r => {
controller.updateSaveList().then(r => {
});
});
}, 250)
})
}