#!/usr/bin/env python3 """ DD3 LoRa Bridge - MQTT Data Republisher GUI Visual interface for recovering lost meter data from SD card """ import tkinter as tk from tkinter import ttk, filedialog, messagebox, scrolledtext import threading import json import csv import os import sys import time from datetime import datetime, timedelta from pathlib import Path from typing import Optional, Tuple import paho.mqtt.client as mqtt # Optional: for auto-detection try: from influxdb_client import InfluxDBClient HAS_INFLUXDB = True except ImportError: HAS_INFLUXDB = False class MQTTRepublisherGUI: def __init__(self, root): self.root = root self.root.title("DD3 MQTT Data Republisher") self.root.geometry("900x750") self.root.resizable(True, True) # Style style = ttk.Style() style.theme_use('clam') self.publishing = False self.mqtt_client = None self.published_count = 0 self.skipped_count = 0 self.create_widgets() def create_widgets(self): """Create GUI widgets""" # Main notebook (tabs) self.notebook = ttk.Notebook(self.root) self.notebook.pack(fill='both', expand=True, padx=5, pady=5) # Tab 1: Settings settings_frame = ttk.Frame(self.notebook) self.notebook.add(settings_frame, text='Settings') self.create_settings_tab(settings_frame) # Tab 2: Time Range time_frame = ttk.Frame(self.notebook) self.notebook.add(time_frame, text='Time Range') self.create_time_tab(time_frame) # Tab 3: Progress progress_frame = ttk.Frame(self.notebook) self.notebook.add(progress_frame, text='Progress') self.create_progress_tab(progress_frame) # Button bar at bottom button_frame = ttk.Frame(self.root) button_frame.pack(fill='x', padx=5, pady=5) ttk.Button(button_frame, text='Start Publishing', command=self.start_publishing).pack(side='left', padx=2) ttk.Button(button_frame, text='Stop', command=self.stop_publishing).pack(side='left', padx=2) ttk.Button(button_frame, text='Exit', command=self.root.quit).pack(side='right', padx=2) self.status_label = ttk.Label(button_frame, text='Ready', relief='sunken') self.status_label.pack(side='right', fill='x', expand=True, padx=2) def create_settings_tab(self, parent): """Create settings tab""" main_frame = ttk.Frame(parent, padding=10) main_frame.pack(fill='both', expand=True) # CSV File Selection ttk.Label(main_frame, text='CSV File:', font=('TkDefaultFont', 10, 'bold')).grid(row=0, column=0, sticky='w', pady=10) frame = ttk.Frame(main_frame) frame.grid(row=1, column=0, columnspan=2, sticky='ew', pady=(0, 20)) self.csv_file_var = tk.StringVar() ttk.Entry(frame, textvariable=self.csv_file_var, width=50).pack(side='left', fill='x', expand=True) ttk.Button(frame, text='Browse...', command=self.select_csv_file).pack(side='right', padx=5) # Device ID ttk.Label(main_frame, text='Device ID:', font=('TkDefaultFont', 10, 'bold')).grid(row=2, column=0, sticky='w', pady=5) self.device_id_var = tk.StringVar(value='dd3-F19C') ttk.Entry(main_frame, textvariable=self.device_id_var, width=30).grid(row=2, column=1, sticky='w', pady=5) # MQTT Settings ttk.Label(main_frame, text='MQTT Broker:', font=('TkDefaultFont', 10, 'bold')).grid(row=3, column=0, sticky='w', pady=5) self.mqtt_broker_var = tk.StringVar(value='localhost') ttk.Entry(main_frame, textvariable=self.mqtt_broker_var, width=30).grid(row=3, column=1, sticky='w', pady=5) ttk.Label(main_frame, text='Port:', font=('TkDefaultFont', 10)).grid(row=4, column=0, sticky='w', pady=5) self.mqtt_port_var = tk.StringVar(value='1883') ttk.Entry(main_frame, textvariable=self.mqtt_port_var, width=30).grid(row=4, column=1, sticky='w', pady=5) ttk.Label(main_frame, text='Username:', font=('TkDefaultFont', 10)).grid(row=5, column=0, sticky='w', pady=5) self.mqtt_user_var = tk.StringVar() ttk.Entry(main_frame, textvariable=self.mqtt_user_var, width=30).grid(row=5, column=1, sticky='w', pady=5) ttk.Label(main_frame, text='Password:', font=('TkDefaultFont', 10)).grid(row=6, column=0, sticky='w', pady=5) self.mqtt_pass_var = tk.StringVar() ttk.Entry(main_frame, textvariable=self.mqtt_pass_var, width=30, show='*').grid(row=6, column=1, sticky='w', pady=5) # Publish Rate ttk.Label(main_frame, text='Publish Rate (msg/sec):', font=('TkDefaultFont', 10)).grid(row=7, column=0, sticky='w', pady=5) self.rate_var = tk.StringVar(value='5') rate_spin = ttk.Spinbox(main_frame, from_=1, to=100, textvariable=self.rate_var, width=10) rate_spin.grid(row=7, column=1, sticky='w', pady=5) # Test Connection Button ttk.Button(main_frame, text='Test MQTT Connection', command=self.test_connection).grid(row=8, column=0, columnspan=2, sticky='ew', pady=20) # Configure grid weights main_frame.columnconfigure(1, weight=1) def create_time_tab(self, parent): """Create time range selection tab""" main_frame = ttk.Frame(parent, padding=10) main_frame.pack(fill='both', expand=True) # Mode selection ttk.Label(main_frame, text='Time Range Mode:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=10) self.time_mode_var = tk.StringVar(value='manual') ttk.Radiobutton(main_frame, text='Manual Selection', variable=self.time_mode_var, value='manual', command=self.update_time_mode).pack(anchor='w', padx=20, pady=5) if HAS_INFLUXDB: ttk.Radiobutton(main_frame, text='Auto-Detect from InfluxDB', variable=self.time_mode_var, value='influxdb', command=self.update_time_mode).pack(anchor='w', padx=20, pady=5) # Manual time selection frame self.manual_frame = ttk.LabelFrame(main_frame, text='Manual Time Range', padding=10) self.manual_frame.pack(fill='x', padx=20, pady=10) ttk.Label(self.manual_frame, text='Start Date (YYYY-MM-DD):').grid(row=0, column=0, sticky='w', pady=5) self.from_date_var = tk.StringVar(value=(datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')) ttk.Entry(self.manual_frame, textvariable=self.from_date_var, width=30).grid(row=0, column=1, sticky='w', pady=5) ttk.Label(self.manual_frame, text='Start Time (HH:MM:SS):').grid(row=1, column=0, sticky='w', pady=5) self.from_time_var = tk.StringVar(value='00:00:00') ttk.Entry(self.manual_frame, textvariable=self.from_time_var, width=30).grid(row=1, column=1, sticky='w', pady=5) ttk.Label(self.manual_frame, text='End Date (YYYY-MM-DD):').grid(row=2, column=0, sticky='w', pady=5) self.to_date_var = tk.StringVar(value=datetime.now().strftime('%Y-%m-%d')) ttk.Entry(self.manual_frame, textvariable=self.to_date_var, width=30).grid(row=2, column=1, sticky='w', pady=5) ttk.Label(self.manual_frame, text='End Time (HH:MM:SS):').grid(row=3, column=0, sticky='w', pady=5) self.to_time_var = tk.StringVar(value='23:59:59') ttk.Entry(self.manual_frame, textvariable=self.to_time_var, width=30).grid(row=3, column=1, sticky='w', pady=5) # InfluxDB frame self.influxdb_frame = ttk.LabelFrame(main_frame, text='InfluxDB Settings', padding=10) if self.time_mode_var.get() == 'influxdb': self.influxdb_frame.pack(fill='x', padx=20, pady=10) else: self.influxdb_frame.pack_forget() ttk.Label(self.influxdb_frame, text='InfluxDB URL:').grid(row=0, column=0, sticky='w', pady=5) self.influx_url_var = tk.StringVar(value='http://localhost:8086') ttk.Entry(self.influxdb_frame, textvariable=self.influx_url_var, width=30).grid(row=0, column=1, sticky='w', pady=5) ttk.Label(self.influxdb_frame, text='API Token:').grid(row=1, column=0, sticky='w', pady=5) self.influx_token_var = tk.StringVar() ttk.Entry(self.influxdb_frame, textvariable=self.influx_token_var, width=30, show='*').grid(row=1, column=1, sticky='w', pady=5) ttk.Label(self.influxdb_frame, text='Organization:').grid(row=2, column=0, sticky='w', pady=5) self.influx_org_var = tk.StringVar() ttk.Entry(self.influxdb_frame, textvariable=self.influx_org_var, width=30).grid(row=2, column=1, sticky='w', pady=5) ttk.Label(self.influxdb_frame, text='Bucket:').grid(row=3, column=0, sticky='w', pady=5) self.influx_bucket_var = tk.StringVar(value='smartmeter') ttk.Entry(self.influxdb_frame, textvariable=self.influx_bucket_var, width=30).grid(row=3, column=1, sticky='w', pady=5) # Info frame info_frame = ttk.LabelFrame(main_frame, text='Info', padding=10) info_frame.pack(fill='both', expand=True, padx=20, pady=10) info_text = """Time format examples: • 2026-03-01 (start of day) • 2026-03-10 14:30:00 (specific time) Manual mode: Select date range to republish Auto-detect: Find gaps in InfluxDB automatically""" ttk.Label(info_frame, text=info_text, justify='left').pack(anchor='w') def create_progress_tab(self, parent): """Create progress tab""" main_frame = ttk.Frame(parent, padding=10) main_frame.pack(fill='both', expand=True) # Progress bar ttk.Label(main_frame, text='Publishing Progress:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=5) self.progress_var = tk.DoubleVar() self.progress_bar = ttk.Progressbar(main_frame, variable=self.progress_var, maximum=100) self.progress_bar.pack(fill='x', pady=5) # Stats frame stats_frame = ttk.LabelFrame(main_frame, text='Statistics', padding=10) stats_frame.pack(fill='x', pady=10) self.stats_text = tk.StringVar(value='Published: 0\nSkipped: 0\nRate: 0 msg/sec') ttk.Label(stats_frame, textvariable=self.stats_text, font=('TkDefaultFont', 10)).pack(anchor='w') # Log output ttk.Label(main_frame, text='Log Output:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=(10, 5)) self.log_text = scrolledtext.ScrolledText(main_frame, height=20, width=100, state='disabled') self.log_text.pack(fill='both', expand=True) def update_time_mode(self): """Update visibility of time selection frames""" if self.time_mode_var.get() == 'manual': self.manual_frame.pack(fill='x', padx=20, pady=10) self.influxdb_frame.pack_forget() else: self.manual_frame.pack_forget() self.influxdb_frame.pack(fill='x', padx=20, pady=10) def select_csv_file(self): """Open file browser for CSV selection""" filename = filedialog.askopenfilename( title='Select CSV File', filetypes=[('CSV files', '*.csv'), ('All files', '*.*')] ) if filename: self.csv_file_var.set(filename) def log(self, message: str): """Add message to log""" self.log_text.config(state='normal') self.log_text.insert('end', message + '\n') self.log_text.see('end') self.log_text.config(state='disabled') self.root.update() def test_connection(self): """Test MQTT connection""" broker = self.mqtt_broker_var.get() port = int(self.mqtt_port_var.get()) user = self.mqtt_user_var.get() or None password = self.mqtt_pass_var.get() or None def test_thread(): self.status_label.config(text='Testing...') self.log('Testing MQTT connection...') client = mqtt.Client() if user and password: client.username_pw_set(user, password) try: client.connect(broker, port, keepalive=10) client.loop_start() time.sleep(2) client.loop_stop() client.disconnect() self.log('✓ MQTT connection successful!') self.status_label.config(text='Connection OK') messagebox.showinfo('Success', 'MQTT connection test passed!') except Exception as e: self.log(f'✗ Connection failed: {e}') self.status_label.config(text='Connection failed') messagebox.showerror('Error', f'Connection failed:\n{e}') thread = threading.Thread(target=test_thread, daemon=True) thread.start() def parse_time_input(self, date_str: str, time_str: str = '00:00:00') -> int: """Parse date/time input and return Unix timestamp""" try: dt_str = f"{date_str} {time_str}" dt = datetime.strptime(dt_str, '%Y-%m-%d %H:%M:%S') return int(dt.timestamp()) except ValueError as e: raise ValueError(f'Invalid date/time format: {e}') def get_time_range(self) -> Tuple[int, int]: """Get time range based on selected mode""" if self.time_mode_var.get() == 'manual': from_time = self.parse_time_input(self.from_date_var.get(), self.from_time_var.get()) to_time = self.parse_time_input(self.to_date_var.get(), self.to_time_var.get()) return from_time, to_time else: # InfluxDB mode if not HAS_INFLUXDB: raise RuntimeError('InfluxDB mode requires influxdb-client') self.log('Connecting to InfluxDB...') try: client = InfluxDBClient( url=self.influx_url_var.get(), token=self.influx_token_var.get(), org=self.influx_org_var.get() ) query_api = client.query_api() device_id = self.device_id_var.get() bucket = self.influx_bucket_var.get() # Query last 7 days now = int(time.time()) from_time = now - (7 * 24 * 3600) to_time = now self.log(f'Searching for missing data from {datetime.fromtimestamp(from_time)} to {datetime.fromtimestamp(to_time)}') query = f''' from(bucket: "{bucket}") |> range(start: {from_time}s, stop: {to_time}s) |> filter(fn: (r) => r._measurement == "smartmeter" and r.device_id == "{device_id}") |> keep(columns: ["_time"]) |> sort(columns: ["_time"]) ''' tables = query_api.query(query) existing_times = [] for table in tables: for record in table.records: ts = int(record.values["_time"].timestamp()) existing_times.append(ts) client.close() if not existing_times: self.log('No data in InfluxDB, will republish entire range') return from_time, to_time # Find first gap existing_times = sorted(set(existing_times)) for i, ts in enumerate(existing_times): if i > 0 and existing_times[i] - existing_times[i-1] > 60: # 60s gap gap_start = existing_times[i-1] gap_end = existing_times[i] self.log(f'Found gap: {datetime.fromtimestamp(gap_start)} to {datetime.fromtimestamp(gap_end)}') return gap_start, gap_end self.log('No gaps found in InfluxDB') return from_time, to_time except Exception as e: raise RuntimeError(f'InfluxDB error: {e}') def republish_csv(self, csv_file: str, device_id: str, from_time: int, to_time: int): """Republish CSV data to MQTT""" if not os.path.isfile(csv_file): self.log(f'✗ File not found: {csv_file}') return count = 0 skipped = 0 start_time = time.time() try: with open(csv_file, 'r') as f: reader = csv.DictReader(f) if not reader.fieldnames: self.log('✗ Invalid CSV: no header row') return for row in reader: if not self.publishing: self.log('Stopped by user') break try: ts_utc = int(row['ts_utc']) if ts_utc < from_time or ts_utc > to_time: skipped += 1 continue # Build payload short_id = device_id[-4:].upper() if len(device_id) >= 4 else device_id.upper() data = {'id': short_id, 'ts': ts_utc} for key in ['e_kwh', 'p_w', 'p1_w', 'p2_w', 'p3_w', 'bat_v', 'bat_pct', 'rssi', 'snr']: if key in row and row[key].strip(): try: val = float(row[key]) if '.' in row[key] else int(row[key]) data[key] = val except: pass # Publish topic = f"smartmeter/{device_id}/state" payload = json.dumps(data) self.mqtt_client.publish(topic, payload) count += 1 self.published_count = count # Update UI if count % 10 == 0: elapsed = time.time() - start_time rate = count / elapsed if elapsed > 0 else 0 self.stats_text.set(f'Published: {count}\nSkipped: {skipped}\nRate: {rate:.1f} msg/sec') self.log(f'[{count:4d}] {ts_utc} {data.get("p_w", "?")}W') # Rate limiting time.sleep(1.0 / int(self.rate_var.get())) except (ValueError, KeyError): skipped += 1 continue except Exception as e: self.log(f'✗ Error: {e}') elapsed = time.time() - start_time self.log(f'✓ Completed! Published {count} samples in {elapsed:.1f}s') self.published_count = count def start_publishing(self): """Start republishing data""" if not self.csv_file_var.get(): messagebox.showerror('Error', 'Please select a CSV file') return if not self.device_id_var.get(): messagebox.showerror('Error', 'Please enter device ID') return try: port = int(self.mqtt_port_var.get()) except ValueError: messagebox.showerror('Error', 'Invalid MQTT port') return try: rate = int(self.rate_var.get()) if rate < 1 or rate > 100: raise ValueError('Rate must be 1-100') except ValueError: messagebox.showerror('Error', 'Invalid publish rate') return self.publishing = True self.log_text.config(state='normal') self.log_text.delete('1.0', 'end') self.log_text.config(state='disabled') self.published_count = 0 def pub_thread(): try: # Get time range from_time, to_time = self.get_time_range() self.log(f'Time range: {datetime.fromtimestamp(from_time)} to {datetime.fromtimestamp(to_time)}') # Connect to MQTT self.status_label.config(text='Connecting to MQTT...') broker = self.mqtt_broker_var.get() port = int(self.mqtt_port_var.get()) user = self.mqtt_user_var.get() or None password = self.mqtt_pass_var.get() or None self.mqtt_client = mqtt.Client() if user and password: self.mqtt_client.username_pw_set(user, password) self.mqtt_client.connect(broker, port, keepalive=60) self.mqtt_client.loop_start() time.sleep(1) self.log('✓ Connected to MQTT broker') self.status_label.config(text='Publishing...') # Republish self.republish_csv(self.csv_file_var.get(), self.device_id_var.get(), from_time, to_time) self.mqtt_client.loop_stop() self.mqtt_client.disconnect() self.status_label.config(text='Done') messagebox.showinfo('Success', f'Published {self.published_count} samples') except Exception as e: self.log(f'✗ Error: {e}') self.status_label.config(text='Error') messagebox.showerror('Error', str(e)) finally: self.publishing = False thread = threading.Thread(target=pub_thread, daemon=True) thread.start() def stop_publishing(self): """Stop publishing""" self.publishing = False self.status_label.config(text='Stopping...') def main(): root = tk.Tk() app = MQTTRepublisherGUI(root) root.mainloop() if __name__ == '__main__': main()