Can append files to DB

.gitignore

@@ -2,3 +2,4 @@ db
 db/*
 __pycache__
 __pycache__/*
+__*.py

tools/database.py

 from pathlib import Path
 import sqlite3
-from typing import Dict, List
+from typing import Dict, List, Callable
 import pandas as pd
 import numpy as np
 from .utils import from_class_to_text, Chronometer
@@ -44,6 +44,8 @@ _DB_METADATA_COLUMNS = [
 
 _DB_METADATA_TABLE = "DataFiles"
 
+DBAppendCallback = Callable[[str, int, int], None]
+
 class DatabaseQuery:
     """Represents a simple query for the database, used to filter the 
     requested data.
@@ -284,7 +286,8 @@ class Database:
             return
         
         self.db_file = db
-        self.db = sqlite3.connect(db)
+
+        self.open()
 
     def robot (self, cell: int):
         """Returns a query object for the specified robot data
@@ -308,7 +311,7 @@ class Database:
 
         return pd.read_sql_query(f"SELECT `FileName`, `index` FROM {_DB_METADATA_TABLE}", self.db)
 
-    def append_files (self, *files: DataFile):
+    def append_files (self, *files: DataFile, next: DBAppendCallback | None = None):
         """Adds data to the database
         """
 
@@ -333,6 +336,8 @@ class Database:
         for i, f in enumerate(files):
             Extractor.reset()
             prefix = f"[{i+1}/{I}]"
+            if next is not None:
+                next(f.file_name, i+1, I)
             Chrono.tick()
 
             # Read Excel file
@@ -365,6 +370,10 @@ class Database:
             Chrono.tick(True)
             # raise Exception("")
 
+    def open (self):
+        """Opens the database"""
+        self.db = sqlite3.connect(self.db_file)
+
     def close (self):
         """Closes the database"""
         self.db.close()
@@ -412,4 +421,30 @@ def init_database (db_file: Path | str):
     """
 
     DB = sqlite3.connect(db_file)
+
+    # Initialize metadata table
+    DB.execute("""
+        CREATE TABLE "DataFiles" (
+            "level_0"	INTEGER,
+            "Year"	INTEGER,
+            "Month"	INTEGER,
+            "Day"	INTEGER,
+            "Name"	TEXT,
+            "Start_Speed"	INTEGER,
+            "Stop_Speed"	INTEGER,
+            "Constant_Speed"	INTEGER,
+            "Sampling"	INTEGER,
+            "Class"	INTEGER,
+            "Iteration_A1"	INTEGER,
+            "Iteration_A2"	INTEGER,
+            "Iteration_A3"	INTEGER,
+            "Iteration_A4"	INTEGER,
+            "Iteration_A5"	INTEGER,
+            "Iteration_A6"	INTEGER,
+            "Robot"	TEXT,
+            "FileName"	TEXT,
+            "index"	INTEGER
+        );
+    """)
+
     DB.close()

ui/DB_Metadata.py

@@ -2,9 +2,10 @@
 """
 import PySimpleGUI as sg
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 import numpy as np
 import pandas as pd
+from threading import Thread, Semaphore
 
 from tools.database import Database
 from tools.data_finder import DataFile
@@ -14,6 +15,11 @@ class DBFileManagerWindow (sg.Window):
     def __init__ (self, db: Database):
 
         self.db = db
+        self.popup = None
+
+        self.db_lock = Semaphore()
+        self.db_use_done = False
+        self.db_callback_status = (None, None, None)
 
         super().__init__("Database Source File Manager", self._layout(), finalize=True)
         self.update_display()
@@ -69,6 +75,25 @@ class DBFileManagerWindow (sg.Window):
     def update_display (self):
         """Updates the displayed elements
         """
+
+        # Update the popup with the current append status 
+        self.db_lock.acquire()
+        if self.popup is not None:
+            name, i, I = self.db_callback_status
+            
+            if name is None:
+                # No status
+                self.popup['-filename-'].update(value=f"Waiting ...")
+                self.popup["-progress-"].update(max=10, current_count=0)
+
+            else:
+                # Has status
+                self.popup['-filename-'].update(value=f"Appending {name}")
+                self.popup["-progress-"].update(max=I, current_count=i)
+
+
+            self.popup.read(1) # Update display
+        self.db_lock.release()
         
         pass
 
@@ -133,9 +158,76 @@ class DBFileManagerWindow (sg.Window):
 
         confirmation = sg.popup_ok_cancel(confirmation_message, title="Confirmation prompt")
 
-        print(confirmation)
-        
+        # The user confirms
+        if confirmation == "OK":
+            self.open_popup()
+
+            # The DB can't be used in multiple threads, closing it for 
+            # the main thread. 
+            # It will be re-opened when db_use_done becomes True.
+            self.db.close()
+            
+            # Thread to add files to the DB
+            def append_thread (self: DBFileManagerWindow, valids: List[DataFile]):
+                # Open the DB
+                self.db.open()
+
+                # Add files to the DB
+                print("Starting to append files")
+                self.db.append_files(
+                    *valids,
+                    next=lambda name, i, I: self.set_append_cb_status((name, i, I))
+                )
+
+                # Close the DB
+                self.db.close()
+                self.db_lock.acquire()
+                self.db_use_done = True
+                self.db_callback_status = (None, None, None)
+                self.db_lock.release()
+
+                self.close_popup()
+                print("Done appending files")
+            
+            T = Thread(target=append_thread, daemon=True, args=[self, valids])
+            T.start()
 
+    def set_append_cb_status (self, status: Tuple[str, int, int]):
+        self.db_callback_status = status
+
+    def open_popup (self):
+        """Opens the popup that informs the user from the progress
+        """
+
+        popup = sg.Window("Importing Data", [
+            [ sg.Push(), sg.Text("Appending data to the Database."), sg.Push() ],
+            [ 
+                sg.Push(), 
+                sg.Text(
+                    "This might take a while",
+                    justification="center"
+                ), 
+                sg.Push() 
+            ],
+            [ 
+                sg.Push(), 
+                sg.Text("Waiting ...", key="-filename-"), 
+                sg.Push() 
+            ], 
+            [
+                sg.ProgressBar(10, expand_x=True, key="-progress-")
+            ]
+        ], finalize=True, disable_close=True, disable_minimize=True)
+        popup.read(1)
+        self.popup = popup
+
+    def close_popup (self):
+        """Closes the popup that informs the user from the progress
+        """
+
+        if self.popup is not None:
+            self.popup.close()
+            self.popup = None
 
     def poll (self):
         """Runs the window
@@ -150,14 +242,22 @@ class DBFileManagerWindow (sg.Window):
         if event == sg.WIN_CLOSED:
             return False
         
-        # No interaction
-        if event == sg.TIMEOUT_EVENT:
-            return True
-        
         # User wants to add files
         if event == "-add-":
             self.add_files()
 
+        # Check if the append operation is done 
+        self.db_lock.acquire()
+        if self.db_use_done == True:
+            print("Done using the DB")
+            # Reset the status
+            self.db_use_done = False
+            self.db.open()
+            # Closes this window instead of dynamic update of the file list.
+            # This can be improved
+            self.close()
+        self.db_lock.release()
+
         self.update_display()
 
         return True
\ No newline at end of file

ui/Database.py

@@ -27,7 +27,11 @@ class DBSelectWin (sg.Window):
             "-edit-": sg.Button("Edit Data", key="-edit-"),
             "-plot-": sg.Button("Plot Data", key="-plot-"),
             "-new-": sg.Button("New Database", key="-new-"),
-            "-browse-": sg.FileBrowse(key="-browse-", initial_folder=str(Path(".").absolute()))
+            "-browse-": sg.FileBrowse(
+                key="-browse-", 
+                initial_folder=str(Path(".").absolute()),
+                file_types=(("SQLite Database", "*.sqlite"),)
+            )
         }
 
         self.inputs = {