DD3-LoRa-Bridge-MultiSender/republish_mqtt_gui.py

#!/usr/bin/env python3
"""
DD3 LoRa Bridge - MQTT Data Republisher GUI
Visual interface for recovering lost meter data from SD card
"""

import tkinter as tk
from tkinter import ttk, filedialog, messagebox, scrolledtext
import threading
import json
import csv
import os
import sys
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import Optional, Tuple

import paho.mqtt.client as mqtt

# Optional: for auto-detection
try:
    from influxdb_client import InfluxDBClient
    HAS_INFLUXDB = True
except ImportError:
    HAS_INFLUXDB = False


class MQTTRepublisherGUI:
    def __init__(self, root):
        self.root = root
        self.root.title("DD3 MQTT Data Republisher")
        self.root.geometry("900x750")
        self.root.resizable(True, True)

        # Style
        style = ttk.Style()
        style.theme_use('clam')

        self.publishing = False
        self.mqtt_client = None
        self.published_count = 0
        self.skipped_count = 0

        self.create_widgets()

    def create_widgets(self):
        """Create GUI widgets"""
        # Main notebook (tabs)
        self.notebook = ttk.Notebook(self.root)
        self.notebook.pack(fill='both', expand=True, padx=5, pady=5)

        # Tab 1: Settings
        settings_frame = ttk.Frame(self.notebook)
        self.notebook.add(settings_frame, text='Settings')
        self.create_settings_tab(settings_frame)

        # Tab 2: Time Range
        time_frame = ttk.Frame(self.notebook)
        self.notebook.add(time_frame, text='Time Range')
        self.create_time_tab(time_frame)

        # Tab 3: Progress
        progress_frame = ttk.Frame(self.notebook)
        self.notebook.add(progress_frame, text='Progress')
        self.create_progress_tab(progress_frame)

        # Button bar at bottom
        button_frame = ttk.Frame(self.root)
        button_frame.pack(fill='x', padx=5, pady=5)

        ttk.Button(button_frame, text='Start Publishing', command=self.start_publishing).pack(side='left', padx=2)
        ttk.Button(button_frame, text='Stop', command=self.stop_publishing).pack(side='left', padx=2)
        ttk.Button(button_frame, text='Exit', command=self.root.quit).pack(side='right', padx=2)

        self.status_label = ttk.Label(button_frame, text='Ready', relief='sunken')
        self.status_label.pack(side='right', fill='x', expand=True, padx=2)

    def create_settings_tab(self, parent):
        """Create settings tab"""
        main_frame = ttk.Frame(parent, padding=10)
        main_frame.pack(fill='both', expand=True)

        # CSV File Selection
        ttk.Label(main_frame, text='CSV File:', font=('TkDefaultFont', 10, 'bold')).grid(row=0, column=0, sticky='w', pady=10)
        frame = ttk.Frame(main_frame)
        frame.grid(row=1, column=0, columnspan=2, sticky='ew', pady=(0, 20))
        
        self.csv_file_var = tk.StringVar()
        ttk.Entry(frame, textvariable=self.csv_file_var, width=50).pack(side='left', fill='x', expand=True)
        ttk.Button(frame, text='Browse...', command=self.select_csv_file).pack(side='right', padx=5)

        # Device ID
        ttk.Label(main_frame, text='Device ID:', font=('TkDefaultFont', 10, 'bold')).grid(row=2, column=0, sticky='w', pady=5)
        self.device_id_var = tk.StringVar(value='dd3-F19C')
        ttk.Entry(main_frame, textvariable=self.device_id_var, width=30).grid(row=2, column=1, sticky='w', pady=5)

        # MQTT Settings
        ttk.Label(main_frame, text='MQTT Broker:', font=('TkDefaultFont', 10, 'bold')).grid(row=3, column=0, sticky='w', pady=5)
        self.mqtt_broker_var = tk.StringVar(value='localhost')
        ttk.Entry(main_frame, textvariable=self.mqtt_broker_var, width=30).grid(row=3, column=1, sticky='w', pady=5)

        ttk.Label(main_frame, text='Port:', font=('TkDefaultFont', 10)).grid(row=4, column=0, sticky='w', pady=5)
        self.mqtt_port_var = tk.StringVar(value='1883')
        ttk.Entry(main_frame, textvariable=self.mqtt_port_var, width=30).grid(row=4, column=1, sticky='w', pady=5)

        ttk.Label(main_frame, text='Username:', font=('TkDefaultFont', 10)).grid(row=5, column=0, sticky='w', pady=5)
        self.mqtt_user_var = tk.StringVar()
        ttk.Entry(main_frame, textvariable=self.mqtt_user_var, width=30).grid(row=5, column=1, sticky='w', pady=5)

        ttk.Label(main_frame, text='Password:', font=('TkDefaultFont', 10)).grid(row=6, column=0, sticky='w', pady=5)
        self.mqtt_pass_var = tk.StringVar()
        ttk.Entry(main_frame, textvariable=self.mqtt_pass_var, width=30, show='*').grid(row=6, column=1, sticky='w', pady=5)

        # Publish Rate
        ttk.Label(main_frame, text='Publish Rate (msg/sec):', font=('TkDefaultFont', 10)).grid(row=7, column=0, sticky='w', pady=5)
        self.rate_var = tk.StringVar(value='5')
        rate_spin = ttk.Spinbox(main_frame, from_=1, to=100, textvariable=self.rate_var, width=10)
        rate_spin.grid(row=7, column=1, sticky='w', pady=5)

        # Test Connection Button
        ttk.Button(main_frame, text='Test MQTT Connection', command=self.test_connection).grid(row=8, column=0, columnspan=2, sticky='ew', pady=20)

        # Configure grid weights
        main_frame.columnconfigure(1, weight=1)

    def create_time_tab(self, parent):
        """Create time range selection tab"""
        main_frame = ttk.Frame(parent, padding=10)
        main_frame.pack(fill='both', expand=True)

        # Mode selection
        ttk.Label(main_frame, text='Time Range Mode:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=10)

        self.time_mode_var = tk.StringVar(value='manual')

        ttk.Radiobutton(main_frame, text='Manual Selection', variable=self.time_mode_var, 
                       value='manual', command=self.update_time_mode).pack(anchor='w', padx=20, pady=5)
        
        if HAS_INFLUXDB:
            ttk.Radiobutton(main_frame, text='Auto-Detect from InfluxDB', variable=self.time_mode_var,
                           value='influxdb', command=self.update_time_mode).pack(anchor='w', padx=20, pady=5)

        # Manual time selection frame
        self.manual_frame = ttk.LabelFrame(main_frame, text='Manual Time Range', padding=10)
        self.manual_frame.pack(fill='x', padx=20, pady=10)

        ttk.Label(self.manual_frame, text='Start Date (YYYY-MM-DD):').grid(row=0, column=0, sticky='w', pady=5)
        self.from_date_var = tk.StringVar(value=(datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d'))
        ttk.Entry(self.manual_frame, textvariable=self.from_date_var, width=30).grid(row=0, column=1, sticky='w', pady=5)

        ttk.Label(self.manual_frame, text='Start Time (HH:MM:SS):').grid(row=1, column=0, sticky='w', pady=5)
        self.from_time_var = tk.StringVar(value='00:00:00')
        ttk.Entry(self.manual_frame, textvariable=self.from_time_var, width=30).grid(row=1, column=1, sticky='w', pady=5)

        ttk.Label(self.manual_frame, text='End Date (YYYY-MM-DD):').grid(row=2, column=0, sticky='w', pady=5)
        self.to_date_var = tk.StringVar(value=datetime.now().strftime('%Y-%m-%d'))
        ttk.Entry(self.manual_frame, textvariable=self.to_date_var, width=30).grid(row=2, column=1, sticky='w', pady=5)

        ttk.Label(self.manual_frame, text='End Time (HH:MM:SS):').grid(row=3, column=0, sticky='w', pady=5)
        self.to_time_var = tk.StringVar(value='23:59:59')
        ttk.Entry(self.manual_frame, textvariable=self.to_time_var, width=30).grid(row=3, column=1, sticky='w', pady=5)

        # InfluxDB frame
        self.influxdb_frame = ttk.LabelFrame(main_frame, text='InfluxDB Settings', padding=10)
        if self.time_mode_var.get() == 'influxdb':
            self.influxdb_frame.pack(fill='x', padx=20, pady=10)
        else:
            self.influxdb_frame.pack_forget()

        ttk.Label(self.influxdb_frame, text='InfluxDB URL:').grid(row=0, column=0, sticky='w', pady=5)
        self.influx_url_var = tk.StringVar(value='http://localhost:8086')
        ttk.Entry(self.influxdb_frame, textvariable=self.influx_url_var, width=30).grid(row=0, column=1, sticky='w', pady=5)

        ttk.Label(self.influxdb_frame, text='API Token:').grid(row=1, column=0, sticky='w', pady=5)
        self.influx_token_var = tk.StringVar()
        ttk.Entry(self.influxdb_frame, textvariable=self.influx_token_var, width=30, show='*').grid(row=1, column=1, sticky='w', pady=5)

        ttk.Label(self.influxdb_frame, text='Organization:').grid(row=2, column=0, sticky='w', pady=5)
        self.influx_org_var = tk.StringVar()
        ttk.Entry(self.influxdb_frame, textvariable=self.influx_org_var, width=30).grid(row=2, column=1, sticky='w', pady=5)

        ttk.Label(self.influxdb_frame, text='Bucket:').grid(row=3, column=0, sticky='w', pady=5)
        self.influx_bucket_var = tk.StringVar(value='smartmeter')
        ttk.Entry(self.influxdb_frame, textvariable=self.influx_bucket_var, width=30).grid(row=3, column=1, sticky='w', pady=5)

        # Info frame
        info_frame = ttk.LabelFrame(main_frame, text='Info', padding=10)
        info_frame.pack(fill='both', expand=True, padx=20, pady=10)

        info_text = """Time format examples:
• 2026-03-01 (start of day)
• 2026-03-10 14:30:00 (specific time)

Manual mode: Select date range to republish
Auto-detect: Find gaps in InfluxDB automatically"""

        ttk.Label(info_frame, text=info_text, justify='left').pack(anchor='w')

    def create_progress_tab(self, parent):
        """Create progress tab"""
        main_frame = ttk.Frame(parent, padding=10)
        main_frame.pack(fill='both', expand=True)

        # Progress bar
        ttk.Label(main_frame, text='Publishing Progress:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=5)
        self.progress_var = tk.DoubleVar()
        self.progress_bar = ttk.Progressbar(main_frame, variable=self.progress_var, maximum=100)
        self.progress_bar.pack(fill='x', pady=5)

        # Stats frame
        stats_frame = ttk.LabelFrame(main_frame, text='Statistics', padding=10)
        stats_frame.pack(fill='x', pady=10)

        self.stats_text = tk.StringVar(value='Published: 0\nSkipped: 0\nRate: 0 msg/sec')
        ttk.Label(stats_frame, textvariable=self.stats_text, font=('TkDefaultFont', 10)).pack(anchor='w')

        # Log output
        ttk.Label(main_frame, text='Log Output:', font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=(10, 5))
        self.log_text = scrolledtext.ScrolledText(main_frame, height=20, width=100, state='disabled')
        self.log_text.pack(fill='both', expand=True)

    def update_time_mode(self):
        """Update visibility of time selection frames"""
        if self.time_mode_var.get() == 'manual':
            self.manual_frame.pack(fill='x', padx=20, pady=10)
            self.influxdb_frame.pack_forget()
        else:
            self.manual_frame.pack_forget()
            self.influxdb_frame.pack(fill='x', padx=20, pady=10)

    def select_csv_file(self):
        """Open file browser for CSV selection"""
        filename = filedialog.askopenfilename(
            title='Select CSV File',
            filetypes=[('CSV files', '*.csv'), ('All files', '*.*')]
        )
        if filename:
            self.csv_file_var.set(filename)

    def log(self, message: str):
        """Add message to log"""
        self.log_text.config(state='normal')
        self.log_text.insert('end', message + '\n')
        self.log_text.see('end')
        self.log_text.config(state='disabled')
        self.root.update()

    def test_connection(self):
        """Test MQTT connection"""
        broker = self.mqtt_broker_var.get()
        port = int(self.mqtt_port_var.get())
        user = self.mqtt_user_var.get() or None
        password = self.mqtt_pass_var.get() or None

        def test_thread():
            self.status_label.config(text='Testing...')
            self.log('Testing MQTT connection...')

            client = mqtt.Client()
            if user and password:
                client.username_pw_set(user, password)

            try:
                client.connect(broker, port, keepalive=10)
                client.loop_start()
                time.sleep(2)
                client.loop_stop()
                client.disconnect()
                self.log('✓ MQTT connection successful!')
                self.status_label.config(text='Connection OK')
                messagebox.showinfo('Success', 'MQTT connection test passed!')
            except Exception as e:
                self.log(f'✗ Connection failed: {e}')
                self.status_label.config(text='Connection failed')
                messagebox.showerror('Error', f'Connection failed:\n{e}')

        thread = threading.Thread(target=test_thread, daemon=True)
        thread.start()

    def parse_time_input(self, date_str: str, time_str: str = '00:00:00') -> int:
        """Parse date/time input and return Unix timestamp"""
        try:
            dt_str = f"{date_str} {time_str}"
            dt = datetime.strptime(dt_str, '%Y-%m-%d %H:%M:%S')
            return int(dt.timestamp())
        except ValueError as e:
            raise ValueError(f'Invalid date/time format: {e}')

    def get_time_range(self) -> Tuple[int, int]:
        """Get time range based on selected mode"""
        if self.time_mode_var.get() == 'manual':
            from_time = self.parse_time_input(self.from_date_var.get(), self.from_time_var.get())
            to_time = self.parse_time_input(self.to_date_var.get(), self.to_time_var.get())
            return from_time, to_time
        else:
            # InfluxDB mode
            if not HAS_INFLUXDB:
                raise RuntimeError('InfluxDB mode requires influxdb-client')

            self.log('Connecting to InfluxDB...')
            try:
                client = InfluxDBClient(
                    url=self.influx_url_var.get(),
                    token=self.influx_token_var.get(),
                    org=self.influx_org_var.get()
                )
                query_api = client.query_api()

                device_id = self.device_id_var.get()
                bucket = self.influx_bucket_var.get()

                # Query last 7 days
                now = int(time.time())
                from_time = now - (7 * 24 * 3600)
                to_time = now

                self.log(f'Searching for missing data from {datetime.fromtimestamp(from_time)} to {datetime.fromtimestamp(to_time)}')

                query = f'''
from(bucket: "{bucket}")
  |> range(start: {from_time}s, stop: {to_time}s)
  |> filter(fn: (r) => r._measurement == "smartmeter" and r.device_id == "{device_id}")
  |> keep(columns: ["_time"])
  |> sort(columns: ["_time"])
'''
                tables = query_api.query(query)
                existing_times = []

                for table in tables:
                    for record in table.records:
                        ts = int(record.values["_time"].timestamp())
                        existing_times.append(ts)

                client.close()

                if not existing_times:
                    self.log('No data in InfluxDB, will republish entire range')
                    return from_time, to_time

                # Find first gap
                existing_times = sorted(set(existing_times))
                for i, ts in enumerate(existing_times):
                    if i > 0 and existing_times[i] - existing_times[i-1] > 60:  # 60s gap
                        gap_start = existing_times[i-1]
                        gap_end = existing_times[i]
                        self.log(f'Found gap: {datetime.fromtimestamp(gap_start)} to {datetime.fromtimestamp(gap_end)}')
                        return gap_start, gap_end

                self.log('No gaps found in InfluxDB')
                return from_time, to_time

            except Exception as e:
                raise RuntimeError(f'InfluxDB error: {e}')

    def republish_csv(self, csv_file: str, device_id: str, from_time: int, to_time: int):
        """Republish CSV data to MQTT"""
        if not os.path.isfile(csv_file):
            self.log(f'✗ File not found: {csv_file}')
            return

        count = 0
        skipped = 0
        start_time = time.time()

        try:
            with open(csv_file, 'r') as f:
                reader = csv.DictReader(f)
                if not reader.fieldnames:
                    self.log('✗ Invalid CSV: no header row')
                    return

                for row in reader:
                    if not self.publishing:
                        self.log('Stopped by user')
                        break

                    try:
                        ts_utc = int(row['ts_utc'])

                        if ts_utc < from_time or ts_utc > to_time:
                            skipped += 1
                            continue

                        # Build payload
                        short_id = device_id[-4:].upper() if len(device_id) >= 4 else device_id.upper()
                        data = {'id': short_id, 'ts': ts_utc}

                        for key in ['e_kwh', 'p_w', 'p1_w', 'p2_w', 'p3_w', 'bat_v', 'bat_pct', 'rssi', 'snr']:
                            if key in row and row[key].strip():
                                try:
                                    val = float(row[key]) if '.' in row[key] else int(row[key])
                                    data[key] = val
                                except:
                                    pass

                        # Publish
                        topic = f"smartmeter/{device_id}/state"
                        payload = json.dumps(data)
                        self.mqtt_client.publish(topic, payload)

                        count += 1
                        self.published_count = count

                        # Update UI
                        if count % 10 == 0:
                            elapsed = time.time() - start_time
                            rate = count / elapsed if elapsed > 0 else 0
                            self.stats_text.set(f'Published: {count}\nSkipped: {skipped}\nRate: {rate:.1f} msg/sec')
                            self.log(f'[{count:4d}] {ts_utc} {data.get("p_w", "?")}W')

                        # Rate limiting
                        time.sleep(1.0 / int(self.rate_var.get()))

                    except (ValueError, KeyError):
                        skipped += 1
                        continue

        except Exception as e:
            self.log(f'✗ Error: {e}')

        elapsed = time.time() - start_time
        self.log(f'✓ Completed! Published {count} samples in {elapsed:.1f}s')
        self.published_count = count

    def start_publishing(self):
        """Start republishing data"""
        if not self.csv_file_var.get():
            messagebox.showerror('Error', 'Please select a CSV file')
            return

        if not self.device_id_var.get():
            messagebox.showerror('Error', 'Please enter device ID')
            return

        try:
            port = int(self.mqtt_port_var.get())
        except ValueError:
            messagebox.showerror('Error', 'Invalid MQTT port')
            return

        try:
            rate = int(self.rate_var.get())
            if rate < 1 or rate > 100:
                raise ValueError('Rate must be 1-100')
        except ValueError:
            messagebox.showerror('Error', 'Invalid publish rate')
            return

        self.publishing = True
        self.log_text.config(state='normal')
        self.log_text.delete('1.0', 'end')
        self.log_text.config(state='disabled')
        self.published_count = 0

        def pub_thread():
            try:
                # Get time range
                from_time, to_time = self.get_time_range()
                self.log(f'Time range: {datetime.fromtimestamp(from_time)} to {datetime.fromtimestamp(to_time)}')

                # Connect to MQTT
                self.status_label.config(text='Connecting to MQTT...')
                broker = self.mqtt_broker_var.get()
                port = int(self.mqtt_port_var.get())
                user = self.mqtt_user_var.get() or None
                password = self.mqtt_pass_var.get() or None

                self.mqtt_client = mqtt.Client()
                if user and password:
                    self.mqtt_client.username_pw_set(user, password)

                self.mqtt_client.connect(broker, port, keepalive=60)
                self.mqtt_client.loop_start()
                time.sleep(1)

                self.log('✓ Connected to MQTT broker')
                self.status_label.config(text='Publishing...')

                # Republish
                self.republish_csv(self.csv_file_var.get(), self.device_id_var.get(),
                                  from_time, to_time)

                self.mqtt_client.loop_stop()
                self.mqtt_client.disconnect()
                self.status_label.config(text='Done')
                messagebox.showinfo('Success', f'Published {self.published_count} samples')

            except Exception as e:
                self.log(f'✗ Error: {e}')
                self.status_label.config(text='Error')
                messagebox.showerror('Error', str(e))
            finally:
                self.publishing = False

        thread = threading.Thread(target=pub_thread, daemon=True)
        thread.start()

    def stop_publishing(self):
        """Stop publishing"""
        self.publishing = False
        self.status_label.config(text='Stopping...')


def main():
    root = tk.Tk()
    app = MQTTRepublisherGUI(root)
    root.mainloop()


if __name__ == '__main__':
    main()