Commit 97782a00 authored by l_piel01's avatar l_piel01

Merge branch 'develop' into refactor/410_align-host-manager

parents addad155 7c2a2ee7
......@@ -16,7 +16,7 @@ Barista is an open-source graphical high-level interface for the Caffe deep lear
**Using Barista**
* To start Barista, run the file `main.py`
* A user manual as well as a starter's tutorial on Barista can be found in the wiki of the project repository: https://zivgitlab.uni-muenster.de/pria/Barista
* A user manual as well as a starter's tutorial on Barista can be found in the wiki of the project repository: https://zivgitlab.uni-muenster.de/pria/Barista/wikis/home
If you have any questions regarding Barista, please feel free to contact the developers at barista(at)uni-muenster.de
......
......@@ -45,6 +45,13 @@ def getMultipleHash(dbPaths):
hashValue += getHash(path)
return hashValue
"""
returns the sha256 hash value of a single string
@author j_stru18
@param inputStr : the string that has to be hashed
@return : the hashValue of the string
"""
def getStringHash(inputStr):
hashObject = hashlib.sha256()
hashObject.update(inputStr)
......
......@@ -7,10 +7,11 @@ from backend.barista.utils.logger import Log
class Hdf5Input:
def __init__(self):
def __init__(self, pathOfHdf5Txt=False):
self._db = None
self._path = None
self.logid = Log.getCallerId('HDF5 Input')
self._pathOfHdf5Txt = pathOfHdf5Txt # HDF5TXT files can contain commentary lines that are no paths
self._logid = Log.getCallerId('HDF5 Input')
def __del__(self):
self.close()
......@@ -19,6 +20,10 @@ class Hdf5Input:
'''set the path of the database.'''
self._path = path
def getPath(self):
'''get the path of this HDF5Input object'''
return self._path
def open(self):
'''open the database from the set path.'''
if self._db:
......@@ -28,10 +33,10 @@ class Hdf5Input:
try:
self._db = h5.File(self._path, 'r')
except:
Log.error("File not valid HDF5: " + self._path, self.logid)
Log.error("File not valid HDF5: " + self._path, self._logid)
self._db = None
else:
Log.error("File does not exist: " + self._path, self.logid)
elif not self._pathOfHdf5Txt:
Log.error("File does not exist: " + self._path, self._logid)
def close(self):
if self._db:
......
......@@ -8,7 +8,7 @@ class Hdf5TxtInput:
def __init__(self):
self._db = []
self._path = None
self.logid = Log.getCallerId("HDF5TXT Input")
self._logid = Log.getCallerId("HDF5TXT Input")
self._projectPath = None
def __del__(self):
......@@ -23,20 +23,21 @@ class Hdf5TxtInput:
if self._path is not None:
if os.path.exists(self._path):
lines = [line.rstrip('\n') for line in open(self._path)]
hdf5Count = 0
for line in lines:
if line is not "":
if line[:1] == '.':
line = self._makepath(line)
i = len(self._db)
self._db.append(Hdf5Input())
self._db.append(Hdf5Input(pathOfHdf5Txt=True))
self._db[i].setPath(line)
self._db[i].open()
if not self._db[i].isOpen():
Log.error("File contains invalid HDF5: " + self._path, self.logid)
self._db = None
return
else:
Log.error("File does not exist: " + self._path, self.logid)
if self._db[i].isOpen():
hdf5Count += 1
if hdf5Count == 0:
self._db = None
Log.error("File contained no valid paths to HDF5 files: {}".format(self._path), self._logid)
def close(self):
if self._db:
......@@ -77,7 +78,11 @@ class Hdf5TxtInput:
self._projectPath = path
def _sumDataCount(self, d1, d2):
return {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)}
dataCountDict = {}
if d1 or d2:
dataCountDict = {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)}
return dataCountDict
def _combineDim(self, d1, d2):
return {k: max(d1.get(k, 0), d2.get(k, 0)) for k in set(d1) | set(d2)}
......@@ -87,6 +92,7 @@ class Hdf5TxtInput:
if False in l:
return False
return True
"""
Converts a path that is considered to be relative to the project path into an absolute path.
......
import os
import uuid
from PyQt5 import QtWidgets
from functools import partial
......@@ -205,14 +208,14 @@ class InputManager(ManagerDialog):
filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:",
defaultName)
if filename is "": # Dialog was closed
return
return False
path = os.path.join(os.path.dirname(path), filename) + ".txt"
if os.path.exists(path):
Log.error("File already exists: " + path + ", can't create new HDF5TXT ", self.logid)
QtWidgets.QMessageBox.critical(self, "Can't create new HDF5TXT",
"File already exists:\n" + path)
return
return False
with open(path, 'w') as file:
# add a new line to be sure
file.write(h5path)
......@@ -256,7 +259,7 @@ class InputManager(ManagerDialog):
filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:",
filename)
if filename is "": # Dialog was closed
return
return False
path = os.path.join(os.path.dirname(path), filename) + ".txt"
# we hash the hdf5 file: if it already exists in the input manager, we don't have to create a new HDF5TXT
......@@ -274,7 +277,7 @@ class InputManager(ManagerDialog):
"Creation of File failed:\n"
+ path)
Log.error("Creation of HDF5TXT " + path + " failed", self.logid)
return
return False
else:
Log.error("Remote host sent no message.", self.logid)
......@@ -288,7 +291,6 @@ class InputManager(ManagerDialog):
db["isRemote"] = True
db["host"] = host
db["port"] = port
addingSuccessful = False
if typeWasHDF5:
addingSuccessful = self._addDBbyID(db, hdf5ID)
else:
......@@ -460,7 +462,9 @@ class InputManager(ManagerDialog):
"""
add this new database to the dict. only for databases WITHOUT IDs
generates an ID of the database by hashing the file to ensure no database is added twice
generates an ID of the database by hashing the file to ensure no database is added twice, unless it is a HDF5TXT
without any valid paths. In this case, a new unique ID is generated. This prevents unwanted hash conflicts for these
files.
@author ..., f_prob02, j_stru18, l_hu0002
@param db a dictionary that holds the attributes of a database object
......@@ -472,13 +476,16 @@ class InputManager(ManagerDialog):
if db["path"] is not None:
dbPaths = db_util.getDBPathsByType(db["path"], db["type"])
hashValue = db_util.getMultipleHash(dbPaths)
if hashValue == 0:
hashValue = uuid.uuid4()
id = str(hashValue)
isValidDB = self._addDBbyID(db, id)
return isValidDB
"""
Generates an ID by hashing a remote database and the remote host's port, which is used to add the database to the
input manager.
input manager, unless it is a HDF5TXT without any valid paths. In this case, a new unique ID is generated and
modified with the hash of the port. This prevents hash unwanted conflicts for these files.
WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that duplicate
files on different hosts can be added. This has to be changed afterwards.
......@@ -496,6 +503,8 @@ class InputManager(ManagerDialog):
ret = sendMsgToHost(db["host"], db["port"], msg)
if ret and ret["status"]:
hashValue = ret["hashValue"]
if hashValue == 0:
hashValue = uuid.uuid4().int
portHash = db_util.getStringHash(str(db["port"]))
id = str(hashValue ^ portHash)
isValidDB = self._addDBbyID(db, id)
......@@ -527,8 +536,8 @@ class InputManager(ManagerDialog):
addingSuccessful = True
else:
duplicatePath = self.getPathByID(id)
QtWidgets.QMessageBox.critical(self, "Duplicate Error on Import!", "The selected file was already "
"imported:\n{}".format(duplicatePath))
QtWidgets.QMessageBox.critical(self, "Duplicate Error on Import!", "The selected file {0} was already "
"imported:\n{1}".format(db["path"], duplicatePath))
Log.error("Duplicate error on import.", self.logid)
assert self.dbcontains(id)
return addingSuccessful
......@@ -986,7 +995,7 @@ class InputManager(ManagerDialog):
self.setUsage(usedBy[:-2])
"""
Reload this database widget by opening it again and in case of a local HDF5TXT rehashing it. If the file was
Reload this database widget by opening it again and in case of a HDF5TXT rehashing it. If the file was
changed outside of Barista, a warning is shown.
@author: j_stru18
......@@ -995,9 +1004,29 @@ class InputManager(ManagerDialog):
self.open()
type = self.getType()
path = self.getPath()
if type == "HDF5TXT" and not self.isRemote:
dbpaths = db_util.getHdf5List(path)
hashedID = db_util.getMultipleHash(dbpaths)
if type == "HDF5TXT":
if not self.isRemote:
dbpaths = db_util.getHdf5List(path)
hashValue = db_util.getMultipleHash(dbpaths)
hashedID = str(hashValue)
else:
hostName = self.remoteHost[0]
port = self.remoteHost[1]
hashMsg = {"key": Protocol.GETHASH, "path": self.getPath(), "type": "HDF5TXT"}
hashRet = sendMsgToHost(hostName, port, hashMsg)
if hashRet["status"]:
hashValue = hashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
if hashValue != 0: # file contained paths to HDF5 files
hashedID = str(hashValue ^ portHash)
else:
return
else:
Log.log("Reloading of database {} on remote host failed.".format(self.id),
self.manager.getLogID)
QtWidgets.QMessageBox.warning(self, "Reloading Failed", "Reloading of database on remote host"
" failed.")
return
if hashedID != self.id: # file was changed outside of Barista
QtWidgets.QMessageBox.warning(self, "File was Changed", "This HDF5TXT file's paths were changed "
"outside of Barista - check for duplicate "
......@@ -1204,56 +1233,61 @@ class InputManager(ManagerDialog):
@author j_stru18
"""
def _addHdf5Local(self):
'''add new HDF5 files to the HDF5TXT file'''
if self.getType() == "HDF5TXT":
# ask for type
ret = QtWidgets.QFileDialog.getOpenFileNames(self, "Select a HDF5 Database", QDir.homePath(),
"HDF5 (*.hdf5 *.h5)")
path = self.getPath()
newh5paths = ret[0]
if len(newh5paths) is not 0:
if os.path.exists(path): # there are selected files and the HDF5TXT exists
h5paths = db_util.getHdf5List(path)
oldHash = db_util.getMultipleHash(h5paths)
newHash = oldHash + db_util.getMultipleHash(newh5paths)
oldID = str(oldHash)
newID = str(newHash)
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
updateNeeded = False
if not isFileDuplicate:
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(newID)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
else:
Log.error("File not found: " + path, self.manager.getLogID())
QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path)
else:
if self.getType() != "HDF5TXT": # guard statement for wrong db type
Log.error("No HDF5TXT file was selected.", self.manager.getLogID())
return
ret = QtWidgets.QFileDialog.getOpenFileNames(self, "Select a HDF5 Database", QDir.homePath(),
"HDF5 (*.hdf5 *.h5)")
path = self.getPath()
newh5paths = ret[0]
if len(newh5paths) == 0: # guard statement for empty selection
Log.error("File not found: " + path, self.manager.getLogID())
QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path)
return
if os.path.exists(path): # there are selected files and the HDF5TXT exists
h5paths = db_util.getHdf5List(path)
newHash = db_util.getMultipleHash(newh5paths)
if not h5paths: # file contains no valid paths
oldID = self.id
else:
oldHash = db_util.getMultipleHash(h5paths)
newHash += oldHash
oldID = str(oldHash)
newID = str(newHash)
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
updateNeeded = False
if not isFileDuplicate:
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(newID)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
"""
Adds a new HDF5 File to the local HDF5TXT file:
......@@ -1273,63 +1307,71 @@ class InputManager(ManagerDialog):
def _addHdf5Remote(self):
hostName = self.remoteHost[0]
port = self.remoteHost[1]
path = self.getPath()
rfd = RemoteFileDialog(hostName, port, "Select a HDF5 Database", "HDF5 (*.hdf5 *.h5)")
rfd.exec_()
newPath = rfd.returnvalue
if newPath is not "":
path = self.getPath()
oldHashMsg = {"key": Protocol.GETHASH, "path": path, "type": "HDF5TXT"}
oldHashRet = sendMsgToHost(hostName, port, oldHashMsg)
if oldHashRet["status"]:
oldHashValue = oldHashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
oldID = str(oldHashValue ^ portHash)
newHashMsg = {"key": Protocol.GETHASH, "path": newPath, "type": "HDF5"}
newHashRet = sendMsgToHost(hostName, port, newHashMsg)
if newHashRet["status"]:
updateNeeded = False
newHashValue = oldHashValue + newHashRet["hashValue"]
newID = str(newHashValue ^ portHash)
isFileDupliate = self.manager.dbcontains(newID) # does a db with the same id exist?
if not isFileDupliate:
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else:
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.deletebyID(newID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
else:
Log.error("Hashing of selected HDF5 was not successful", self.manager.getLogID())
else:
Log.error("Hashing of old HDF5TXT not successful.", self.manager.getLogID())
else:
if newPath == "": # guard statement for empty path
Log.error("No HDF5TXT file was selected.", self.manager.getLogID())
return
oldHashMsg = {"key": Protocol.GETHASH, "path": path, "type": "HDF5TXT"}
oldHashRet = sendMsgToHost(hostName, port, oldHashMsg)
newHashMsg = {"key": Protocol.GETHASH, "path": newPath, "type": "HDF5"}
newHashRet = sendMsgToHost(hostName, port, newHashMsg)
if not oldHashRet["status"]: # guard statement for invalid return status #1
Log.error("Hashing of old HDF5TXT not successful.", self.manager.getLogID())
return
if not newHashRet["status"]: # guard statement for invalid return status #2
Log.error("Hashing of selected HDF5 was not successful", self.manager.getLogID())
return
oldHashValue = oldHashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
if oldHashValue == 0: # file contained no paths to HDF5 files
oldID = self.id
else:
oldID = str(oldHashValue ^ portHash)
updateNeeded = False
newHashValue = oldHashValue + newHashRet["hashValue"]
newID = str(newHashValue ^ portHash)
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
if not isFileDuplicate:
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else: # file with identical hashed ID exists
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.deletebyID(newID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
def disableEditing(self, disable):
""" Disable the button to assign a new layer """
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment