Commit b2800200 authored by Johannes Stricker's avatar Johannes Stricker

Merge branch 'develop' into bugfix/461-ctrl

parents 3e9cc736 7c2a2ee7
......@@ -16,7 +16,14 @@ Barista is an open-source graphical high-level interface for the Caffe deep lear
**Using Barista**
* To start Barista, run the file `main.py`
* A user manual as well as a starter's tutorial on Barista can be found in the wiki of the project repository: https://zivgitlab.uni-muenster.de/pria/Barista
* A user manual as well as a starter's tutorial on Barista can be found in the wiki of the project repository: https://zivgitlab.uni-muenster.de/pria/Barista/wikis/home
If you have any questions regarding Barista, please feel free to contact the developers at barista(at)uni-muenster.de
** Citing Barista **
If you use Barista in your research, please cite it using the following reference:
Klemm, S. and Scherzinger, A. and Drees, D. and Jiang, X. Barista - a Graphical Tool for Designing and Training Deep Neural Networks. arXiv preprint. arXiv:1802.04626. http://arxiv.org/abs/1802.04626
......@@ -128,35 +128,49 @@ class MinimumTrainingRequirements:
def _checkUniqueBlobNames(self):
"""Checks for duplicate top blob names in all layers and emits error message if duplicates found except
if in-place is permitted (Activation Layer)"""
nonActivationTopBlobNames = []
activationTopBlobNames = []
if in-place is permitted"""
# find all 'real' sources of blobs, i.e. layers that produce a blob as output and are not in-place
blobGenerators = {} # dictionary that saves the sources for every blob name
if hasattr(self._stateData, '__getitem__'):
# inefficient double looping but activation layers can only be checked after non-activation layers
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
if LayerHelper.isLayerIncludedInTrainingPhase(layer) and ("top" in layer["parameters"]):
if not layer["type"].isActivationLayer():
for name in layer["parameters"]["top"]:
if name not in nonActivationTopBlobNames:
nonActivationTopBlobNames.append(name)
else:
self._errorMessages.append((name + " is used more than once as connection name!",
"Duplicate connector name in " + layer["parameters"]["name"]))
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
if LayerHelper.isLayerIncludedInTrainingPhase(layer) and ("top" in layer["parameters"]):
if layer["type"].isActivationLayer():
for name in layer["parameters"]["top"]:
if name not in activationTopBlobNames:
activationTopBlobNames.append(name)
if name in nonActivationTopBlobNames:
if not LayerHelper.isLayerInPlace(layer):
self._errorMessages.append(
(name + " is used more than once as connection name!",
"Duplicate connector name in " + layer["parameters"]["name"]))
else:
self._errorMessages.append((name + " is used more than once as connection name!",
"Duplicate connector name in " + layer["parameters"]["name"]))
parameters = layer.get("parameters", {})
tops = parameters.get("top", [])
bottoms = parameters.get("bottom", [])
# if at least one top blob is also a bottom, check if layer allows in-place
sourced = [blob for blob in tops if blob not in bottoms]
inPlace = [blob for blob in tops if blob in bottoms]
if len(inPlace) > 0 and not layer["type"].allowsInPlace():
for name in inPlace:
self._errorMessages.append((
"{} is reproduced by {}".format(name, parameters.get("name", "[NO NAME]")),
"{} does not support in-place operation".format(parameters.get("name",
"[NO NAME]"))
))
# check all blobs that are generated in one layer if they are generated only once in each phase.
for name in sourced:
phase = [] # this list can hold train, test and both
p = LayerHelper.getLayerPhase(layer)
if p == "":
phase = [LayerHelper.PHASE_TEST, LayerHelper.PHASE_TRAIN]
else:
phase.append(p)
# if a blob already exists, check if it was generated before in the same Phase
if name in blobGenerators:
found_match = False
for candidate in blobGenerators[name]:
intersection = set(phase).intersection(candidate[1])
if len(intersection) > 0:
found_match = True
self._errorMessages.append(("Sources are {} and {} in phase {}".format(
parameters.get("name", "[NO NAME]"),
candidate[0],
list(intersection)[0]),
"{} is generated by multiple layers".format(name)))
if not found_match:
blobGenerators[name].append((parameters.get("name", "[NO NAME]"), phase))
else:
blobGenerators[name] = [(parameters.get("name", "[NO NAME]"), phase)]
def _checkDataLayerExistence(self):
"""Check whether at least one data layer exists (during the training phase)."""
......
......@@ -45,6 +45,13 @@ def getMultipleHash(dbPaths):
hashValue += getHash(path)
return hashValue
"""
returns the sha256 hash value of a single string
@author j_stru18
@param inputStr : the string that has to be hashed
@return : the hashValue of the string
"""
def getStringHash(inputStr):
hashObject = hashlib.sha256()
hashObject.update(inputStr)
......
......@@ -272,12 +272,16 @@ class LayerType(TopLevelEntityType):
def isActivationLayer(self):
return self._category == self.CATEGORY_ACTIVATION
def allowsInPlace(self):
inPlaceLayers = ['Dropout']
return self._name in inPlaceLayers or self.isActivationLayer()
@staticmethod
def getCategoryByName(layerTypeName):
"""Given its name, determine which category a LayerType does belong to."""
# !!! hard coded list of activation layers could cause problems when more layers are added in caffe
if layerTypeName in ["ReLU", "PReLU", "ELU", "Sigmoid", "TanH", "AbsVal", "Power",
"Exp", "Log", "BNLL", "Threshold", "Bias", "Scale" ]:
"Exp", "Log", "BNLL", "Threshold", "Bias", "Scale"]:
return LayerType.CATEGORY_ACTIVATION
elif "Loss" in layerTypeName:
return LayerType.CATEGORY_LOSS
......
......@@ -7,10 +7,11 @@ from backend.barista.utils.logger import Log
class Hdf5Input:
def __init__(self):
def __init__(self, pathOfHdf5Txt=False):
self._db = None
self._path = None
self.logid = Log.getCallerId('HDF5 Input')
self._pathOfHdf5Txt = pathOfHdf5Txt # HDF5TXT files can contain commentary lines that are no paths
self._logid = Log.getCallerId('HDF5 Input')
def __del__(self):
self.close()
......@@ -19,6 +20,10 @@ class Hdf5Input:
'''set the path of the database.'''
self._path = path
def getPath(self):
'''get the path of this HDF5Input object'''
return self._path
def open(self):
'''open the database from the set path.'''
if self._db:
......@@ -28,10 +33,10 @@ class Hdf5Input:
try:
self._db = h5.File(self._path, 'r')
except:
Log.error("File not valid HDF5: " + self._path, self.logid)
Log.error("File not valid HDF5: " + self._path, self._logid)
self._db = None
else:
Log.error("File does not exist: " + self._path, self.logid)
elif not self._pathOfHdf5Txt:
Log.error("File does not exist: " + self._path, self._logid)
def close(self):
if self._db:
......
......@@ -8,7 +8,7 @@ class Hdf5TxtInput:
def __init__(self):
self._db = []
self._path = None
self.logid = Log.getCallerId("HDF5TXT Input")
self._logid = Log.getCallerId("HDF5TXT Input")
self._projectPath = None
def __del__(self):
......@@ -23,20 +23,21 @@ class Hdf5TxtInput:
if self._path is not None:
if os.path.exists(self._path):
lines = [line.rstrip('\n') for line in open(self._path)]
hdf5Count = 0
for line in lines:
if line is not "":
if line[:1] == '.':
line = self._makepath(line)
i = len(self._db)
self._db.append(Hdf5Input())
self._db.append(Hdf5Input(pathOfHdf5Txt=True))
self._db[i].setPath(line)
self._db[i].open()
if not self._db[i].isOpen():
Log.error("File contains invalid HDF5: " + self._path, self.logid)
self._db = None
return
else:
Log.error("File does not exist: " + self._path, self.logid)
if self._db[i].isOpen():
hdf5Count += 1
if hdf5Count == 0:
self._db = None
Log.error("File contained no valid paths to HDF5 files: {}".format(self._path), self._logid)
def close(self):
if self._db:
......@@ -77,7 +78,11 @@ class Hdf5TxtInput:
self._projectPath = path
def _sumDataCount(self, d1, d2):
return {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)}
dataCountDict = {}
if d1 or d2:
dataCountDict = {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)}
return dataCountDict
def _combineDim(self, d1, d2):
return {k: max(d1.get(k, 0), d2.get(k, 0)) for k in set(d1) | set(d2)}
......@@ -87,6 +92,7 @@ class Hdf5TxtInput:
if False in l:
return False
return True
"""
Converts a path that is considered to be relative to the project path into an absolute path.
......
This diff is collapsed.
......@@ -435,10 +435,10 @@ class NodeEditor:
self.__networkManager.addLayer(type, scenePosX, scenePosY)
def tryToAddBottomBlob(self, layerID):
def tryToAddBottomBlob(self, layerID, blobName):
""" Notifies the NetworkManager to create a new bottom connector for the given layer. """
self.__networkManager.addBottomBlob(layerID)
self.__networkManager.addBottomBlob(layerID, blobName)
def tryToRemoveBottomBlob(self, layerID, blobIndex):
""" Notifies the NetworkManager to try to remove the bottom connector
......
......@@ -228,7 +228,7 @@ class NodeEditorEventHandler:
if ok:
self.nodeEditor.tryToAddTopBlob(pressedItem.getLayerID(), name)
elif action == addBottomAction:
self.nodeEditor.tryToAddBottomBlob(pressedItem.getLayerID())
self.nodeEditor.tryToAddBottomBlob(pressedItem.getLayerID(), "")
elif action == copyItemsAction:
self.nodeEditor.setCopySelection()
elif action == removeAction:
......
......@@ -120,7 +120,7 @@ class NodeEditorScene(QGraphicsScene):
# add empty bottom blob property
if not emptyBlobAvailable and not topBlobFound:
self.__nodeEditor.tryToAddBottomBlob(bottomLayerID)
self.__nodeEditor.tryToAddBottomBlob(bottomLayerID, "")
# connect nodes
connected = self.__nodeEditor.tryToConnect(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)
......@@ -164,16 +164,20 @@ class NodeEditorScene(QGraphicsScene):
for i in range(0, item.getTopConnectorCount()):
self.__nodeEditor.tryToAddTopBlob(newItem.getLayerID(), item.getTopConnectors()[i].getBlobName())
for connection in item.getTopConnectors()[i].getConnections():
for j in range(item.getTopConnectors()[i].getConnectionCount()-1, -1, -1):
connection = item.getTopConnectors()[i].getConnections()[j]
connection.setTopConnector(newItem.getTopConnectors()[i])
item.getTopConnectors()[i].removeConnection(connection)
newItem.getTopConnectors()[i].addConnection(connection)
# Add bottom connectors
for i in range(0, item.getBottomConnectorCount()):
self.__nodeEditor.tryToAddBottomBlob(newItem.getLayerID())
self.__nodeEditor.tryToAddBottomBlob(newItem.getLayerID(), item.getBottomConnectors()[i].getBlobName())
for j in range(item.getBottomConnectors()[i].getConnectionCount()-1, -1, -1):
connection = item.getBottomConnectors()[i].getConnections()[j]
for connection in item.getBottomConnectors()[i].getConnections():
connection.setBottomConnector(newItem.getBottomConnectors()[i])
item.getBottomConnectors()[i].removeConnection(connection)
newItem.getBottomConnectors()[i].addConnection(connection)
......
......@@ -542,7 +542,7 @@ class NetworkManager(QObject):
# Add one to the count of all layers in the application
self.layerCount += 1
def addBottomBlob(self, layerID):
def addBottomBlob(self, layerID, blobName):
'Adds a bottom entry to the layer with the given id'
def intern(stateDict):
network = stateDict["network"]
......@@ -550,7 +550,7 @@ class NetworkManager(QObject):
layerParams = networkHelper.layerParams(layerID)
if "bottom" not in layerParams:
layerParams.giveProperty("bottom",[])
layerParams["bottom"].pushBack("")
layerParams["bottom"].pushBack(blobName)
# TODO: Instead update in onStateUpdate
#self.nodeEditor.addBottomBlob(layerID, "")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment