diff --git a/.gitignore b/.gitignore index ed728e4c49a2b3f1c35710eb090c61add1a3a787..8e80082fbc75dacd4a03c45fd7597293546af5c9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ db db/* __pycache__ __pycache__/* +__*.py diff --git a/tools/database.py b/tools/database.py index 6f3c91d364b7f96c64f0add138a63cc68a1e852e..5ef8bb9e0d44c4206e50fa72b408178a0e28cce2 100644 --- a/tools/database.py +++ b/tools/database.py @@ -1,6 +1,6 @@ from pathlib import Path import sqlite3 -from typing import Dict, List +from typing import Dict, List, Callable import pandas as pd import numpy as np from .utils import from_class_to_text, Chronometer @@ -44,6 +44,8 @@ _DB_METADATA_COLUMNS = [ _DB_METADATA_TABLE = "DataFiles" +DBAppendCallback = Callable[[str, int, int], None] + class DatabaseQuery: """Represents a simple query for the database, used to filter the requested data. @@ -284,7 +286,8 @@ class Database: return self.db_file = db - self.db = sqlite3.connect(db) + + self.open() def robot (self, cell: int): """Returns a query object for the specified robot data @@ -308,7 +311,7 @@ class Database: return pd.read_sql_query(f"SELECT `FileName`, `index` FROM {_DB_METADATA_TABLE}", self.db) - def append_files (self, *files: DataFile): + def append_files (self, *files: DataFile, next: DBAppendCallback | None = None): """Adds data to the database """ @@ -333,6 +336,8 @@ class Database: for i, f in enumerate(files): Extractor.reset() prefix = f"[{i+1}/{I}]" + if next is not None: + next(f.file_name, i+1, I) Chrono.tick() # Read Excel file @@ -365,6 +370,10 @@ class Database: Chrono.tick(True) # raise Exception("") + def open (self): + """Opens the database""" + self.db = sqlite3.connect(self.db_file) + def close (self): """Closes the database""" self.db.close() @@ -412,4 +421,30 @@ def init_database (db_file: Path | str): """ DB = sqlite3.connect(db_file) + + # Initialize metadata table + DB.execute(""" + CREATE TABLE "DataFiles" ( + "level_0" INTEGER, + "Year" INTEGER, + "Month" INTEGER, + "Day" INTEGER, + "Name" TEXT, + "Start_Speed" INTEGER, + "Stop_Speed" INTEGER, + "Constant_Speed" INTEGER, + "Sampling" INTEGER, + "Class" INTEGER, + "Iteration_A1" INTEGER, + "Iteration_A2" INTEGER, + "Iteration_A3" INTEGER, + "Iteration_A4" INTEGER, + "Iteration_A5" INTEGER, + "Iteration_A6" INTEGER, + "Robot" TEXT, + "FileName" TEXT, + "index" INTEGER + ); + """) + DB.close() diff --git a/ui/DB_Metadata.py b/ui/DB_Metadata.py index e607bb722a8cc1776669f2ec12a6a1b916a40d79..064fa9f206d2b96e11c8a3cac5c73b15990ef7fe 100644 --- a/ui/DB_Metadata.py +++ b/ui/DB_Metadata.py @@ -2,9 +2,10 @@ """ import PySimpleGUI as sg from pathlib import Path -from typing import List +from typing import List, Tuple import numpy as np import pandas as pd +from threading import Thread, Semaphore from tools.database import Database from tools.data_finder import DataFile @@ -14,6 +15,11 @@ class DBFileManagerWindow (sg.Window): def __init__ (self, db: Database): self.db = db + self.popup = None + + self.db_lock = Semaphore() + self.db_use_done = False + self.db_callback_status = (None, None, None) super().__init__("Database Source File Manager", self._layout(), finalize=True) self.update_display() @@ -69,6 +75,25 @@ class DBFileManagerWindow (sg.Window): def update_display (self): """Updates the displayed elements """ + + # Update the popup with the current append status + self.db_lock.acquire() + if self.popup is not None: + name, i, I = self.db_callback_status + + if name is None: + # No status + self.popup['-filename-'].update(value=f"Waiting ...") + self.popup["-progress-"].update(max=10, current_count=0) + + else: + # Has status + self.popup['-filename-'].update(value=f"Appending {name}") + self.popup["-progress-"].update(max=I, current_count=i) + + + self.popup.read(1) # Update display + self.db_lock.release() pass @@ -133,9 +158,76 @@ class DBFileManagerWindow (sg.Window): confirmation = sg.popup_ok_cancel(confirmation_message, title="Confirmation prompt") - print(confirmation) - + # The user confirms + if confirmation == "OK": + self.open_popup() + + # The DB can't be used in multiple threads, closing it for + # the main thread. + # It will be re-opened when db_use_done becomes True. + self.db.close() + + # Thread to add files to the DB + def append_thread (self: DBFileManagerWindow, valids: List[DataFile]): + # Open the DB + self.db.open() + + # Add files to the DB + print("Starting to append files") + self.db.append_files( + *valids, + next=lambda name, i, I: self.set_append_cb_status((name, i, I)) + ) + + # Close the DB + self.db.close() + self.db_lock.acquire() + self.db_use_done = True + self.db_callback_status = (None, None, None) + self.db_lock.release() + + self.close_popup() + print("Done appending files") + + T = Thread(target=append_thread, daemon=True, args=[self, valids]) + T.start() + def set_append_cb_status (self, status: Tuple[str, int, int]): + self.db_callback_status = status + + def open_popup (self): + """Opens the popup that informs the user from the progress + """ + + popup = sg.Window("Importing Data", [ + [ sg.Push(), sg.Text("Appending data to the Database."), sg.Push() ], + [ + sg.Push(), + sg.Text( + "This might take a while", + justification="center" + ), + sg.Push() + ], + [ + sg.Push(), + sg.Text("Waiting ...", key="-filename-"), + sg.Push() + ], + [ + sg.ProgressBar(10, expand_x=True, key="-progress-") + ] + ], finalize=True, disable_close=True, disable_minimize=True) + popup.read(1) + self.popup = popup + + def close_popup (self): + """Closes the popup that informs the user from the progress + """ + + if self.popup is not None: + self.popup.close() + self.popup = None def poll (self): """Runs the window @@ -150,14 +242,22 @@ class DBFileManagerWindow (sg.Window): if event == sg.WIN_CLOSED: return False - # No interaction - if event == sg.TIMEOUT_EVENT: - return True - # User wants to add files if event == "-add-": self.add_files() + # Check if the append operation is done + self.db_lock.acquire() + if self.db_use_done == True: + print("Done using the DB") + # Reset the status + self.db_use_done = False + self.db.open() + # Closes this window instead of dynamic update of the file list. + # This can be improved + self.close() + self.db_lock.release() + self.update_display() return True \ No newline at end of file diff --git a/ui/Database.py b/ui/Database.py index 8c248cf2da8f6eaf88ee50e1b6a44b8c8ac27efe..224c960651eece4e27b3aa6897d1e1d53da34bdc 100644 --- a/ui/Database.py +++ b/ui/Database.py @@ -27,7 +27,11 @@ class DBSelectWin (sg.Window): "-edit-": sg.Button("Edit Data", key="-edit-"), "-plot-": sg.Button("Plot Data", key="-plot-"), "-new-": sg.Button("New Database", key="-new-"), - "-browse-": sg.FileBrowse(key="-browse-", initial_folder=str(Path(".").absolute())) + "-browse-": sg.FileBrowse( + key="-browse-", + initial_folder=str(Path(".").absolute()), + file_types=(("SQLite Database", "*.sqlite"),) + ) } self.inputs = {