Skip to content
Snippets Groups Projects
Commit d24d3448 authored by VulcanixFR's avatar VulcanixFR
Browse files

Can append files to DB

parent 4685f5a5
Branches
No related merge requests found
......@@ -2,3 +2,4 @@ db
db/*
__pycache__
__pycache__/*
__*.py
from pathlib import Path
import sqlite3
from typing import Dict, List
from typing import Dict, List, Callable
import pandas as pd
import numpy as np
from .utils import from_class_to_text, Chronometer
......@@ -44,6 +44,8 @@ _DB_METADATA_COLUMNS = [
_DB_METADATA_TABLE = "DataFiles"
DBAppendCallback = Callable[[str, int, int], None]
class DatabaseQuery:
"""Represents a simple query for the database, used to filter the
requested data.
......@@ -284,7 +286,8 @@ class Database:
return
self.db_file = db
self.db = sqlite3.connect(db)
self.open()
def robot (self, cell: int):
"""Returns a query object for the specified robot data
......@@ -308,7 +311,7 @@ class Database:
return pd.read_sql_query(f"SELECT `FileName`, `index` FROM {_DB_METADATA_TABLE}", self.db)
def append_files (self, *files: DataFile):
def append_files (self, *files: DataFile, next: DBAppendCallback | None = None):
"""Adds data to the database
"""
......@@ -333,6 +336,8 @@ class Database:
for i, f in enumerate(files):
Extractor.reset()
prefix = f"[{i+1}/{I}]"
if next is not None:
next(f.file_name, i+1, I)
Chrono.tick()
# Read Excel file
......@@ -365,6 +370,10 @@ class Database:
Chrono.tick(True)
# raise Exception("")
def open (self):
"""Opens the database"""
self.db = sqlite3.connect(self.db_file)
def close (self):
"""Closes the database"""
self.db.close()
......@@ -412,4 +421,30 @@ def init_database (db_file: Path | str):
"""
DB = sqlite3.connect(db_file)
# Initialize metadata table
DB.execute("""
CREATE TABLE "DataFiles" (
"level_0" INTEGER,
"Year" INTEGER,
"Month" INTEGER,
"Day" INTEGER,
"Name" TEXT,
"Start_Speed" INTEGER,
"Stop_Speed" INTEGER,
"Constant_Speed" INTEGER,
"Sampling" INTEGER,
"Class" INTEGER,
"Iteration_A1" INTEGER,
"Iteration_A2" INTEGER,
"Iteration_A3" INTEGER,
"Iteration_A4" INTEGER,
"Iteration_A5" INTEGER,
"Iteration_A6" INTEGER,
"Robot" TEXT,
"FileName" TEXT,
"index" INTEGER
);
""")
DB.close()
......@@ -2,9 +2,10 @@
"""
import PySimpleGUI as sg
from pathlib import Path
from typing import List
from typing import List, Tuple
import numpy as np
import pandas as pd
from threading import Thread, Semaphore
from tools.database import Database
from tools.data_finder import DataFile
......@@ -14,6 +15,11 @@ class DBFileManagerWindow (sg.Window):
def __init__ (self, db: Database):
self.db = db
self.popup = None
self.db_lock = Semaphore()
self.db_use_done = False
self.db_callback_status = (None, None, None)
super().__init__("Database Source File Manager", self._layout(), finalize=True)
self.update_display()
......@@ -69,6 +75,25 @@ class DBFileManagerWindow (sg.Window):
def update_display (self):
"""Updates the displayed elements
"""
# Update the popup with the current append status
self.db_lock.acquire()
if self.popup is not None:
name, i, I = self.db_callback_status
if name is None:
# No status
self.popup['-filename-'].update(value=f"Waiting ...")
self.popup["-progress-"].update(max=10, current_count=0)
else:
# Has status
self.popup['-filename-'].update(value=f"Appending {name}")
self.popup["-progress-"].update(max=I, current_count=i)
self.popup.read(1) # Update display
self.db_lock.release()
pass
......@@ -133,9 +158,76 @@ class DBFileManagerWindow (sg.Window):
confirmation = sg.popup_ok_cancel(confirmation_message, title="Confirmation prompt")
print(confirmation)
# The user confirms
if confirmation == "OK":
self.open_popup()
# The DB can't be used in multiple threads, closing it for
# the main thread.
# It will be re-opened when db_use_done becomes True.
self.db.close()
# Thread to add files to the DB
def append_thread (self: DBFileManagerWindow, valids: List[DataFile]):
# Open the DB
self.db.open()
# Add files to the DB
print("Starting to append files")
self.db.append_files(
*valids,
next=lambda name, i, I: self.set_append_cb_status((name, i, I))
)
# Close the DB
self.db.close()
self.db_lock.acquire()
self.db_use_done = True
self.db_callback_status = (None, None, None)
self.db_lock.release()
self.close_popup()
print("Done appending files")
T = Thread(target=append_thread, daemon=True, args=[self, valids])
T.start()
def set_append_cb_status (self, status: Tuple[str, int, int]):
self.db_callback_status = status
def open_popup (self):
"""Opens the popup that informs the user from the progress
"""
popup = sg.Window("Importing Data", [
[ sg.Push(), sg.Text("Appending data to the Database."), sg.Push() ],
[
sg.Push(),
sg.Text(
"This might take a while",
justification="center"
),
sg.Push()
],
[
sg.Push(),
sg.Text("Waiting ...", key="-filename-"),
sg.Push()
],
[
sg.ProgressBar(10, expand_x=True, key="-progress-")
]
], finalize=True, disable_close=True, disable_minimize=True)
popup.read(1)
self.popup = popup
def close_popup (self):
"""Closes the popup that informs the user from the progress
"""
if self.popup is not None:
self.popup.close()
self.popup = None
def poll (self):
"""Runs the window
......@@ -150,14 +242,22 @@ class DBFileManagerWindow (sg.Window):
if event == sg.WIN_CLOSED:
return False
# No interaction
if event == sg.TIMEOUT_EVENT:
return True
# User wants to add files
if event == "-add-":
self.add_files()
# Check if the append operation is done
self.db_lock.acquire()
if self.db_use_done == True:
print("Done using the DB")
# Reset the status
self.db_use_done = False
self.db.open()
# Closes this window instead of dynamic update of the file list.
# This can be improved
self.close()
self.db_lock.release()
self.update_display()
return True
\ No newline at end of file
......@@ -27,7 +27,11 @@ class DBSelectWin (sg.Window):
"-edit-": sg.Button("Edit Data", key="-edit-"),
"-plot-": sg.Button("Plot Data", key="-plot-"),
"-new-": sg.Button("New Database", key="-new-"),
"-browse-": sg.FileBrowse(key="-browse-", initial_folder=str(Path(".").absolute()))
"-browse-": sg.FileBrowse(
key="-browse-",
initial_folder=str(Path(".").absolute()),
file_types=(("SQLite Database", "*.sqlite"),)
)
}
self.inputs = {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment