dict_helper.py 11 KB
Newer Older
1
from backend.caffe.loader import _extract_param
2
import backend.caffe.proto_info as info
3
import uuid
4
import copy
5

6 7 8
from google.protobuf.internal.containers import RepeatedScalarFieldContainer as ProtoList

def isNetDictValid(netdic):
Tim H's avatar
Tim H committed
9
    """ This function returns a tuple (valid, msg). Value is true iff netdic has a valid format.
10 11
    If the format is not valid, msg contains a text which describes the problem.
    """
12 13 14
    if not {"layers", "layerOrder"}.issubset(netdic.keys()):
        missing = {"layers","layerOrder"}.difference(netdic.keys())
        return False, "Needed parameters {} not in dictionary".format(list(missing))
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
    layers =  netdic["layers"]
    order = netdic["layerOrder"]

    #Check if netparameters are valid (except layers)
    netparam = info.CaffeMetaInformation().availableNetParameters().copy()
    del netparam["layers"]
    tmp = netdic.copy()
    del tmp["layers"]
    del tmp["layerOrder"]
    if not _areParameterValid(netparam, tmp):
        return False, "Invalid net paremters"

    # Check for correct types of this special values
    if type(order) != list or type(layers) != dict:
        return False, "layerOrder or layers are invalid types"
    # Test if all ids are in the layerOrder list and vise versa
    if set(layers.keys()) != set(order):
Robin 1's avatar
Robin 1 committed
32 33 34 35
        missingOrder = set(layers.keys()).difference(order)
        missingLayer = set(order).difference(layers.keys())
        return False, "layerOrder and layers does not have same ids - missing ids in layers: {} ; missing ids in order: {}"\
                      .format(missingLayer, missingOrder)
36 37 38 39 40
    # Check every sublayer
    for id in order:
        if not {"parameters","type"}.issubset(layers[id].keys()):
            return False, "parameters or type are not in layers subdictionary"
        params = layers[id]["parameters"]
41 42
        if params is None:
            return False, "parameters is None, should be at least {'name': '..', 'type': '...'}"
43
        layer = layers[id]["type"]
44 45 46 47 48
        if not isinstance(layer, info.LayerType):
            return False, "layers is {}, expected instance of protoinfo.LayerType".format(layer)
        validparam, msg =  _areParameterValid(layer.parameters(), params)
        if not validparam:
            return False, "Invalid parameters for {}: {}".format(id, msg)
49

50
    return True, ""
51

52
def _areParameterValid(allparams, paramdict, prefix=""):
53 54
    """ Checks if all parameters in paramdict are valid.
        allparams should contain all available parameters names as key
Tim H's avatar
Tim H committed
55
        and the proto-meta-info-class as value.
56
    """
57
    def checkType(meta, value, typeprefix):
58 59
        pytype = None
        if meta.isParameterGroup():
60 61 62
            if type(value) != dict:
                return False, "Expected dictionary , found {}".format(type(value))
            return _areParameterValid(meta.parameter(), value, typeprefix)
63 64 65
        elif meta.isBool():
            pytype = [bool]
        elif meta.isString():
66
            pytype = [unicode, str]
67 68 69 70 71 72 73
        elif meta.isInt():
            pytype = [int, long]
        elif meta.isBytes():
            pass # TODO: What type are bytes?
        elif meta.isFloat() or meta.isDouble():
            pytype = [float]
        elif meta.isEnum():
74 75 76 77 78
            pytype = [str, unicode]
        res = type(value) in pytype
        if not res:
            return False, "Expected Type {}, found {}".format(pytype, type(value))
        return True, ""
79
    for param in paramdict:
80
        typeprefix=prefix+"."+param
81
        val = paramdict[param]
82 83
        if not allparams.has_key(param):
            return False, "Parameter {}: Parameter not available in Caffe".format(typeprefix)
84 85
        meta = allparams[param]
        if meta.isRepeated():
Tim H's avatar
Tim H committed
86
            if not (type(val) is ProtoList or type(val) is list):
87 88 89 90 91
                return False, "Parameter {}: Expected repeated entry, found single one".format(typeprefix)
            for i,entry in enumerate(val):
                valid, msg = checkType(meta,entry, typeprefix)
                if not valid:
                    return False, "Parameter {} - idx {}: {}".format(typeprefix, i,msg)
92
        else:
Tim H's avatar
Tim H committed
93
            valid, msg = checkType(meta,val,typeprefix)
94 95 96
            if not valid:
                return False, "Parameter {}: {}".format(typeprefix,msg)
    return True, ""
Tim H's avatar
Tim H committed
97

98

99

100
def bareNet(name):
101
    """ Creates a dictionary of a networks with default values where required. """
102 103
    from backend.caffe.path_loader import PathLoader
    proto = PathLoader().importProto()
104
    net = proto.NetParameter()
105
    descr = info.ParameterGroupDescriptor(net)
106 107 108
    params = descr.parameter().copy()
    del params["layer"]
    del params["layers"]
109 110 111
    res = _extract_param(net, params)
    res["layers"] = {}
    res["layerOrder"] = []
112
    res["name"] = unicode(name)
113 114 115 116
    return res

def _bareLayer(layertype, name):
    """ Creates a dictionary of the given layertype with the given name
117
        initialized with default values if required.
118
    """
119 120
    from backend.caffe.path_loader import PathLoader
    proto = PathLoader().importProto()
121
    res = {"type": layertype}
122
    layerParamInst = proto.LayerParameter()
Tim H's avatar
Tim H committed
123
    res["parameters"] = _extract_param(layerParamInst, info.CaffeMetaInformation().commonParameters())
124
    res["parameters"]["name"] = unicode(name)
125 126 127
    layerparamkeys = layertype.layerParamKeys()
    for layerparamkey in layerparamkeys:
        layerTypeParam = layertype.parameters()[layerparamkey]
128
        layerTypeParamInst = layerTypeParam.protoclass()()
129
        res["parameters"][layerparamkey] = _extract_param(layerTypeParamInst, layerTypeParam.parameter())
130
    res["parameters"]["type"] = unicode(layertype.name())
131 132
    return res

133

134
class DictHelper:
Robin 1's avatar
Robin 1 committed
135
    """ A helper class to simplify the using of a network dictionary """
136 137
    def __init__(self,netdic):
        self._netdic = netdic
Tim H's avatar
Tim H committed
138

139
    def nameOfLayers(self):
Robin 1's avatar
Robin 1 committed
140
        """ Return a list with all layers of the networks """
141
        return [x["parameters"]["name"] for x in self._netdic["layers"].values()]
142

143
    def nameOfTopBlobs(self):
Robin 1's avatar
Robin 1 committed
144
        """ Return a list with all top blob names in use of the current network """
145 146 147 148 149 150 151
        names = []
        for id in self._netdic["layers"]:
            if "top" in self.layerPerId(id)["parameters"]:
                for j in range(0, len(self.layerPerId(id)["parameters"]["top"])):
                    names.append(self.layerPerId(id)["parameters"]["top"][j])
        return names

152
    def netParameters(self):
Robin 1's avatar
Robin 1 committed
153
        """ Return all parameters for the network (excluding layers) """
154
        return dict([(k,self._netdic[k]) for k in self._netdic if not k in ["layers","layerOrder"]])
155

156
    def layerIdsForName(self,name):
Robin 1's avatar
Robin 1 committed
157
        """ Return a list of the ids for every layer with the given name """
158
        return [x for x in self._netdic["layers"] if self._netdic["layers"][x]["parameters"]["name"] == name]
159

160
    def layerType(self,id):
Robin 1's avatar
Robin 1 committed
161
        """ Return the type of the layer with the given id
162
        """
163
        return self._netdic["layers"][id]["type"]
164

165
    def layerParams(self,id):
Robin 1's avatar
Robin 1 committed
166
        """ Return the parameter of the layer with the given id
167
        """
168
        return self._netdic["layers"][id]["parameters"]
169

170
    def layerPerId(self, id):
Robin 1's avatar
Robin 1 committed
171
        """ Return the layer with the given id """
172 173
        return self._netdic["layers"][id]

174
    def hasLayerWithId(self,id):
175
        """ Return true if there is a layer with the given id"""
176 177
        return id in self._netdic["layers"]

178
    def addLayer(self, layertype, name, idx):
179
        """ Add a new Layer from layertype with the given name
180 181 182 183
            and initialize it with the default values if required.
            This new layer is added to the intern dictionary at position idx
            and the layer subdictionary will be returned with the id.

Tim H's avatar
Tim H committed
184
            import backend.caffe.protoinfo as info
185 186
            convLayer = info.CaffeMetaInformation().availableLayerTypes()["Convolution"]
            newLayer, layerId = dichelperinstance.addLayer(convLayer, "Convolution", 3)
187 188
        """
        newLayer = _bareLayer(layertype, name)
189 190
        id = str(uuid.uuid4())
        self._netdic["layerOrder"].insert(idx, id)
191
        self._netdic["layers"][id] = newLayer
192 193 194 195 196 197
        return newLayer, id

    def removeLayer(self, id):
        del self._netdic["layers"][id]
        self._netdic["layerOrder"].remove(id)

198 199 200 201
    def duplicateLayer(self, id, newName, idx=None):
        """ Duplicate Layer with given id. The new layer has a new random id
            and the new given name.
            The layer will be added at position idx in the layerOrder.
Robin 1's avatar
Robin 1 committed
202
            If idx==None, the idx will be directly beside the old layer.
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
            This function returns the new layer as dictionary and its id.
            E.g. newLayerDict, newId = dicthelper.duplicateLayer("somerandomid", "Some awesome name")
        """
        oldLayer = self.layerPerId(id)
        newLayer = {
            "type": oldLayer["type"],
            "parameters": copy.deepcopy(oldLayer["parameters"])
        }
        newLayer["parameters"]["name"] = newName
        newId = str(uuid.uuid4())
        if idx is None:
            idx = self._netdic["layerOrder"].index(id)+1
        self._netdic["layerOrder"].insert(idx, newId)
        self._netdic["layers"][newId] = newLayer
        return newLayer, newId

219 220
    def dictionary(self):
        return self._netdic
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275

    def layerParameterIsSet(self, layerId, parameterKey):
        """ Check whether the given parameter is set for the given layer.

        :param layerId: The id of the layer to check.
        :param parameterKey: The key of the parameter to check. Might represent a nested parameter concatenated with dots.
        :return:
        """

        isSet = "parameters" in self._netdic["layers"][layerId]

        if isSet:
            paramKeyParts = parameterKey.split(".")
            value = self._netdic["layers"][layerId]["parameters"]
            for paramKeyPart in paramKeyParts:
                if paramKeyPart in value:
                    value = value[paramKeyPart]
                else:
                    isSet = False
                    break

        return isSet

    def layerParameter(self, layerId, parameterKey):
        """ Get a specific parameter for one of the layers.

        :param layerId: The id of the layer.
        :param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots.
        :return:
        """

        paramKeyParts = parameterKey.split(".")
        value = self._netdic["layers"][layerId]["parameters"]
        for paramKeyPart in paramKeyParts:
            if paramKeyPart in value:
                value = value[paramKeyPart]
            else:
                value = None
                break

        return value

    def setLayerParameter(self, layerId, parameterKey, value):
        """ Set a specific parameter for one of the layers.

        :param layerId: The id of the layer.
        :param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots.
        :return:
        """

        paramKeyParts = parameterKey.split(".")
        dic = self._netdic["layers"][layerId]["parameters"]
        for key in paramKeyParts[:-1]:
            dic = dic.setdefault(key, {})
        dic[paramKeyParts[-1]] = value