* Implemented own Json codec which should be able to handle large Json files much better

Grinder/Libraries.pro

@@ -22,7 +22,7 @@ win32 {
 	LIBS += libopencv_highgui340
 }
 
-unix {
+linux|macx {
 	LIBS += -lopencv_core
 	LIBS += -lopencv_imgcodecs
 	LIBS += -lopencv_imgproc

Grinder/Version.h

@@ -10,14 +10,14 @@
 
 #define GRNDR_INFO_TITLE		"Grinder"
 #define GRNDR_INFO_COPYRIGHT	"Copyright (c) WWU Muenster"
-#define GRNDR_INFO_DATE			"26.04.2018"
+#define GRNDR_INFO_DATE			"28.04.2018"
 #define GRNDR_INFO_COMPANY		"WWU Muenster"
 #define GRNDR_INFO_WEBSITE		"http://www.uni-muenster.de"
 
 #define GRNDR_VERSION_MAJOR		0
 #define GRNDR_VERSION_MINOR		3
 #define GRNDR_VERSION_REVISION	0
-#define GRNDR_VERSION_BUILD		145
+#define GRNDR_VERSION_BUILD		147
 
 namespace grndr
 {

Grinder/common/serialization/JsonSettingsCodec.cpp

@@ -6,120 +6,321 @@
 #include "Grinder.h"
 #include "JsonSettingsCodec.h"
 #include "SerializationExceptions.h"
+#include "util/StringUtils.h"
 
-JsonSettingsCodec::JsonSettingsCodec(bool compactFormat) :
-	_compactFormat(compactFormat)
+#include <QJsonValue>
+
+void JsonSettingsCodec::encodeSettings(const SettingsContainer& settings, QTextStream& stream)
+{
+	encode(settings, stream, 0, true);
+}
+
+void JsonSettingsCodec::decodeSettings(SettingsContainer& settings, QTextStream& stream)
+{
+	settings.clear();
+
+	try {
+		decode(settings, stream);
+	} catch (SerializationException& e) {
+		throw SerializationException{_EXCPT(QString{"Invalid JSON document: %1"}.arg(e.what()))};
+	}
+}
+
+void JsonSettingsCodec::encode(const SettingsContainer& settings, QTextStream& stream, int level, bool lastEntry) const
 {
+	// Begin a new Json object
+	QString objectName = (!settings.getName().isEmpty() && level > 0 ? QString{"\"%1\": "}.arg(settings.getName()) : "");
+	writeJsonLine(QString{"%1%2"}.arg(objectName).arg(settings.isArray() ? "[" : "{"), stream, level);
+
+	// Write all children of the current container
+	auto children = settings.children();
+
+	for (unsigned int i = 0; i < children.size(); ++i)
+	{
+		const auto& child = children[i];
+		bool lastChild = (i + 1 >= children.size() && settings.values().isEmpty());
+
+		if (settings.isArray())
+		{
+			// Each array element is added as an individual object
+			writeJsonLine(child->isArray() ? "[" : "{", stream, level + 1);
+			encode(*child, stream, level + 2, true);
+			writeJsonLine(QString{"%1%2"}.arg(child->isArray() ? "]" : "}").arg(!lastChild ? "," : ""), stream, level + 1);
+		}
+		else
+			encode(*child, stream, level + 1, lastChild);
+	}
 
+	// Write all settings of the current container
+	writeJsonValues(settings, stream, level + 1);
+
+	// Close the Json object
+	writeJsonLine(QString{"%1%2"}.arg(settings.isArray() ? "]" : "}").arg(!lastEntry ? "," : ""), stream, level);
 }
 
-void JsonSettingsCodec::encodeSettings(const SettingsContainer& settings)
+void JsonSettingsCodec::decode(SettingsContainer& settings, QTextStream& stream) const
 {
-	_document.setObject(encode(settings));
+	// We first need to tokenize the given input stream
+	auto tokens = readJsonDocument(stream);
+
+	// Now parse the tokens using a recursive descent parser
+	if (!tokens.empty())
+		parseJsonDocument(settings, tokens);
 }
 
-void JsonSettingsCodec::decodeSettings(SettingsContainer& settings)
+void JsonSettingsCodec::writeJsonValues(const SettingsContainer& settings, QTextStream& stream, int level) const
 {
-	if (!_document.isObject())
-		throw SerializationException{_EXCPT("Invalid JSON document")};
+	// Write out each value
+	auto keys = settings.values().keys();
 
-	settings.clear();
-	decode(settings, _document.object());
+	for (int i = 0; i < keys.size(); ++i)
+	{
+		auto valueName = keys[i];
+		writeJsonValue(valueName, settings.values()[valueName], stream, level, i + 1 >= keys.size());
+	}
 }
 
-QString JsonSettingsCodec::getText() const
+void JsonSettingsCodec::writeJsonValue(QString valueName, const QVariant& value, QTextStream& stream, int level, bool lastEntry) const
 {
-	if (_document.isNull())
-		throw SerializationException{_EXCPT("Invalid JSON document")};
+	QJsonValue jsonValue = QJsonValue::fromVariant(value);	// Use Qt's Json value class to properly handle the variant type
+	QString formattedValue;
+
+	switch (jsonValue.type())
+	{
+	case QJsonValue::Bool:
+		formattedValue = jsonValue.toBool() ? "true" : "false";
+		break;
+
+	case QJsonValue::Double:
+		formattedValue = QString{"%1"}.arg(jsonValue.toDouble());
+		break;
+
+	default:
+		formattedValue = "\"" + StringUtils::encodeEscapeCharacters(jsonValue.toString()) + "\"";
+		break;
+	}
 
-	return _document.toJson(_compactFormat ? QJsonDocument::Compact : QJsonDocument::Indented);
+	writeJsonLine(QString{"\"%1\": %2%3"}.arg(valueName).arg(formattedValue).arg(!lastEntry ? "," : ""), stream, level);
 }
 
-void JsonSettingsCodec::setText(const QString& text)
+void JsonSettingsCodec::writeJsonLine(const QString& text, QTextStream& stream, int level) const
 {
-	QJsonParseError parseError;
-	_document = QJsonDocument::fromJson(text.toLatin1(), &parseError);
+	QString line = text;
+
+	// Tabify the line
+	if (level > 0)
+		line.insert(0, QString{"\t"}.repeated(level));
 
-	if (_document.isNull())
-		throw SerializationException{_EXCPT(QString{"Invalid JSON document: %1"}.arg(parseError.errorString()))};
+	stream << line << "\n";
 }
 
-QJsonObject JsonSettingsCodec::encode(const SettingsContainer& settings)
+std::vector<JsonSettingsCodec::Token> JsonSettingsCodec::readJsonDocument(QTextStream& stream) const
 {
-	QJsonObject object = QJsonObject::fromVariantMap(settings.values());
+	// Set up the various regular expressions for parsing the Json document
+	struct TokenizerEntry
+	{
+		TokenizerEntry(QString regExp, TokenType type, bool greedy = true) : re{"^" + regExp}, tokenType{type} { re.setMinimal(!greedy); }
 
-	// Add all children of the current container
-	for (const auto& child : settings.children())
+		QRegExp re;
+		TokenType tokenType{TokenType::Undefined};
+	};
+
+	static const std::vector<TokenizerEntry> tokenizers{
+		TokenizerEntry{R"("([^"\\]|\\["\\])*")", TokenType::String, false},
+		TokenizerEntry{R"(\{)", TokenType::ObjectBegin},
+		TokenizerEntry{R"(\})", TokenType::ObjectEnd},
+		TokenizerEntry{R"(\[)", TokenType::ArrayBegin},
+		TokenizerEntry{R"(\])", TokenType::ArrayEnd},
+		TokenizerEntry{R"(:)", TokenType::Colon},
+		TokenizerEntry{R"(,)", TokenType::Comma},
+		TokenizerEntry{R"([^,]+)", TokenType::Value},
+	};
+
+	// Tokenize the entire document, line by line
+	std::vector<Token> tokens;
+
+	while (!stream.atEnd())
 	{
-		if (child->isArray())
+		QString line = stream.readLine();
+
+		while (!line.isEmpty())
 		{
-			QJsonArray array;
+			// Remove leading whitespaces before parsing the line (so that new tokens always begin at position 0)
+			if ((line = line.trimmed()).isEmpty())
+				break;
+
+			bool matchFound = false;
 
-			// Add all children of the current child to the array
-			for (const auto& arrayElem : child->children())
+			// Try each tokenizer for a match; if none was found, the Json document has an invalid syntax
+			for (const auto& tokenizer : tokenizers)
 			{
-				QJsonObject containerObj;
-				containerObj[arrayElem->getName()] = encode(*arrayElem);
-				array.append(containerObj);
+				auto indexFound = tokenizer.re.indexIn(line);
+
+				if (indexFound == 0)	// Matches must begin at the start of the line
+				{
+					int matchedLength = tokenizer.re.matchedLength();
+					QString matchedString = line.left(matchedLength);
+
+					if (tokenizer.tokenType == TokenType::String)
+						matchedString = StringUtils::decodeEscapeCharacters(matchedString);
+
+					tokens.emplace_back(Token{tokenizer.tokenType, matchedString});
+					line.remove(0, matchedLength);
+
+					matchFound = true;
+					break;
+				}
 			}
 
-			object[child->getName()] = array;
+			if (!matchFound)
+				throw SerializationException{QString{"Invalid token '%1'"}.arg(line)};
 		}
-		else
-			object[child->getName()] = encode(*child);
 	}
 
-	return object;
+	return tokens;
 }
 
-void JsonSettingsCodec::decode(SettingsContainer& settings, const QJsonObject& object)
+void JsonSettingsCodec::parseJsonDocument(SettingsContainer& settings, const std::vector<Token>& tokens) const
 {
-	for (const auto& key : object.keys())
+	// Parse the entire document consisting of a large Json object; note that the used grammar is somewhat limited and tailored for how settings containers are stored
+
+	// Doc ::= Object
+	ParseContext parseContext;
+	parseJsonObject(settings, tokens, parseContext);
+}
+
+void JsonSettingsCodec::parseJsonObject(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const
+{
+	// Object ::= '{' ObjectItems '}' | '{' '}'
+	rdpExpect(TokenType::ObjectBegin, tokens, parseContext, [&settings, &parseContext](QString) { parseContext.pushChildSettings(settings); });	// Push a new container for the object on top
+
+	if (!rdpAccept(TokenType::ObjectEnd, tokens, parseContext, nullptr, true))	// Check for an empty object
+		parseJsonObjectItems(settings, tokens, parseContext);
+
+	rdpExpect(TokenType::ObjectEnd, tokens, parseContext, [&parseContext](QString) { parseContext.popChildSettings(); });	// And pop it was we're finished with it
+}
+
+void JsonSettingsCodec::parseJsonObjectItems(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const
+{
+	// ObjectItems ::= ObjectItem {',' ObjectItem}
+	do {
+		parseJsonObjectItem(settings, tokens, parseContext);
+	} while (rdpAccept(TokenType::Comma, tokens, parseContext));
+}
+
+void JsonSettingsCodec::parseJsonObjectItem(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const
+{
+	// ObjectItem ::= String ':' (String|Value|Array|Object)
+	rdpExpect(TokenType::String, tokens, parseContext, [&parseContext](QString value) { removeEnclosingQuotes(value); parseContext.identifier = value; });	// Remember the last identifier
+	rdpExpect(TokenType::Colon, tokens, parseContext);
+
+	// Lambdas to propery set variant values from the Json value
+	auto stringAttribtor = [&parseContext](QString value) { removeEnclosingQuotes(value); parseContext.setSettingsValue(value); };
+	auto valueAttributor = [&parseContext](QString value) {
+		if (value.compare("true", Qt::CaseInsensitive) == 0)
+			parseContext.setSettingsValue(true);
+		else if (value.compare("false", Qt::CaseInsensitive) == 0)
+			parseContext.setSettingsValue(false);
+		else if (value.indexOf('.') != -1)
+			parseContext.setSettingsValue(value.toDouble());
+		else
+			parseContext.setSettingsValue(value.toInt());
+	};
+
+	if (rdpAccept(TokenType::String, tokens, parseContext, stringAttribtor))
 	{
-		const auto& value = object[key];
+		// Nothing else to do
+	}
+	else if (rdpAccept(TokenType::Value, tokens, parseContext, valueAttributor))
+	{
+		// Nothing else to do
+	}
+	else if (rdpAccept(TokenType::ArrayBegin, tokens, parseContext, nullptr, true))
+		parseJsonArray(settings, tokens, parseContext);
+	else if (rdpAccept(TokenType::ObjectBegin, tokens, parseContext, nullptr, true))
+		parseJsonObject(settings, tokens, parseContext);
+}
 
-		if (value.isObject())
-		{
-			SettingsContainer childContainer{key};
-			decode(childContainer, value.toObject());
-			settings << childContainer;
-		}
-		else if (value.isArray())
-		{
-			SettingsContainer childContainer{key, true};
-			QJsonArray array = value.toArray();
+void JsonSettingsCodec::parseJsonArray(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const
+{
+	// Array ::= '[' Object {',' Object} ']' | '[' ']'
+	rdpExpect(TokenType::ArrayBegin, tokens, parseContext, [&settings, &parseContext](QString) { parseContext.pushChildSettings(settings, true); });
+
+	if (!rdpAccept(TokenType::ArrayEnd, tokens, parseContext, nullptr, true))	// Check for an empty array
+	{
+		do {
+			// Handle the array elements which are encapsulated as individual objects
+			SettingsContainer arrayElemContainer;
+			parseContext.settingsStack.push(&arrayElemContainer);
+			parseContext.identifier = "$_array_elem_$";
+			parseJsonObject(settings, tokens, parseContext);
+			parseContext.settingsStack.pop();
 
-			// Iterate over all array elements and create setting containers for each
-			for (const auto& arrayElem : array)
+			// Extract the elements from their encapsulating objects
+			for (auto arrayElem : arrayElemContainer.children("$_array_elem_$"))
 			{
-				if (arrayElem.isObject())
-				{
-					auto containerObj = arrayElem.toObject();
-
-					// The array element must contain a single object which contains the actual (named) object
-					if (containerObj.size() == 1)
-					{
-						QString elemName = containerObj.keys().front();
-
-						if (containerObj[elemName].isObject())
-						{
-							SettingsContainer elemContainer{elemName};
-							decode(elemContainer, containerObj[elemName].toObject());
-							childContainer << elemContainer;
-						}
-						else
-							throw SerializationException{_EXCPT("Invalid array element found")};
-					}
-					else
-						throw SerializationException{_EXCPT("Invalid container object found")};
-				}
+				if (arrayElem->children().size() == 1)	// There must be exactly one child object in the encapsulating object
+					*parseContext.settingsStack.top() << std::move(*arrayElem->children().at(0));
 				else
-					throw SerializationException{_EXCPT("Invalid array found")};
+					throw SerializationException{"Invalid container object found"};
 			}
+		} while (rdpAccept(TokenType::Comma, tokens, parseContext));
+	}
+
+	rdpExpect(TokenType::ArrayEnd, tokens, parseContext, [&parseContext](QString) { parseContext.popChildSettings(); });
+}
 
-			settings << childContainer;
+bool JsonSettingsCodec::rdpAccept(JsonSettingsCodec::TokenType token, const std::vector<Token>& tokens, ParseContext& parseContext, std::function<void(QString)> attributor, bool peek) const
+{
+	if (parseContext.currentToken < tokens.size() && tokens[parseContext.currentToken].type == token)
+	{
+		if (!peek)
+		{
+			// Call the attributor with the current token value
+			if (attributor)
+				attributor(tokens[parseContext.currentToken].value);
+
+			parseContext.currentToken += 1;
 		}
-		else
-			settings[key] = value.toVariant();
+
+		return true;
+	}
+	else
+		return false;
+}
+
+void JsonSettingsCodec::rdpExpect(JsonSettingsCodec::TokenType token, const std::vector<Token>& tokens, ParseContext& parseContext, std::function<void(QString)> attributor) const
+{
+	if (!rdpAccept(token, tokens, parseContext, attributor))
+		throw SerializationException{QString{"Unexpected symbol type (wanted %1, got %2)"}.arg(static_cast<int>(token)).arg(static_cast<int>(!tokens.empty() ? tokens.back().type : TokenType::Undefined))};
+}
+
+void JsonSettingsCodec::removeEnclosingQuotes(QString& text)
+{
+	if (text.left(1) == "\"")
+		text.remove(0, 1);
+
+	if (text.right(1) == "\"")
+		text.chop(1);
+}
+
+void JsonSettingsCodec::ParseContext::pushChildSettings(SettingsContainer& settings, bool isArray)
+{
+	// Create a new child container (or push the root container if the stack is empty)
+	if (!settingsStack.empty())
+	{
+		auto childContainer = settingsStack.top()->createChild(identifier, isArray);
+		settingsStack.push(childContainer);
+	}
+	else
+		settingsStack.push(&settings);
+}
+
+void JsonSettingsCodec::ParseContext::setSettingsValue(const QVariant& value)
+{
+	if (!settingsStack.empty())
+	{
+		auto& currentSettings = *settingsStack.top();
+		currentSettings[identifier] = value;
 	}
 }

Grinder/common/serialization/JsonSettingsCodec.h

@@ -6,7 +6,7 @@
 #ifndef JSONSETTINGSCODEC_H
 #define JSONSETTINGSCODEC_H
 
-#include <QJsonDocument>
+#include <stack>
 
 #include "SettingsCodec.h"
 
@@ -15,24 +15,66 @@ namespace grndr
 	class JsonSettingsCodec : public SettingsCodec
 	{
 	public:
-		JsonSettingsCodec(bool compactFormat = false);
+		using SettingsCodec::SettingsCodec;
 
 	public:
-		virtual void encodeSettings(const SettingsContainer& settings) override;
-		virtual void decodeSettings(SettingsContainer& settings) override;
+		virtual void encodeSettings(const SettingsContainer& settings, QTextStream& stream) override;
+		virtual void decodeSettings(SettingsContainer& settings, QTextStream& stream) override;
 
-	public:
-		virtual QString getText() const override;
-		virtual void setText(const QString& text) override;
+	private:
+		void encode(const SettingsContainer& settings, QTextStream& stream, int level, bool lastEntry) const;
+		void decode(SettingsContainer& settings, QTextStream& stream) const;
 
 	private:
-		QJsonObject encode(const SettingsContainer& settings);
-		void decode(SettingsContainer& settings, const QJsonObject& object);
+		void writeJsonValues(const SettingsContainer& settings, QTextStream& stream, int level) const;
+		void writeJsonValue(QString valueName, const QVariant& value, QTextStream& stream, int level, bool lastEntry) const;
+		void writeJsonLine(const QString& text, QTextStream& stream, int level) const;
 
 	private:
-		QJsonDocument _document;
+		enum class TokenType
+		{
+			Undefined,
+			String,
+			ObjectBegin,
+			ObjectEnd,
+			ArrayBegin,
+			ArrayEnd,
+			Colon,
+			Comma,
+			Value,
+		};
+
+		struct Token
+		{
+			TokenType type{TokenType::Undefined};
+			QString value{""};
+		};
+
+		struct ParseContext
+		{
+			std::stack<SettingsContainer*> settingsStack;
+			unsigned int currentToken{0};
+			QString identifier{""};
+
+			void pushChildSettings(SettingsContainer& settings, bool isArray = false);
+			void popChildSettings() { settingsStack.pop(); }
 
-		bool _compactFormat{false};
+			void setSettingsValue(const QVariant& value);
+		};
+
+		std::vector<Token> readJsonDocument(QTextStream& stream) const;
+
+		void parseJsonDocument(SettingsContainer& settings, const std::vector<Token>& tokens) const;
+		void parseJsonObject(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const;
+		void parseJsonObjectItems(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const;
+		void parseJsonObjectItem(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const;
+		void parseJsonArray(SettingsContainer& settings, const std::vector<Token>& tokens, ParseContext& parseContext) const;
+
+		bool rdpAccept(TokenType token, const std::vector<Token>& tokens, ParseContext& parseContext, std::function<void(QString)> attributor = nullptr, bool peek = false) const;
+		void rdpExpect(TokenType token, const std::vector<Token>& tokens, ParseContext& parseContext, std::function<void(QString)> attributor = nullptr) const;
+
+	private:
+		static void removeEnclosingQuotes(QString& text);
 	};
 }
 

Grinder/common/serialization/SettingsCodec.cpp

@@ -10,14 +10,12 @@
 void SettingsCodec::saveContainer(const SettingsContainer& settings, QString fileName)
 {
 	try {
-		encodeSettings(settings);
-
 		QFile file{fileName};
 
 		if (file.open(QIODevice::WriteOnly|QIODevice::Truncate|QIODevice::Text))
 		{
 			QTextStream out{&file};
-			out << getText();
+			encodeSettings(settings, out);
 		}
 		else
 			throw SerializationException{_EXCPT(QString{"Unable to open file '%1' for writing"}.arg(fileName))};
@@ -37,8 +35,8 @@ void SettingsCodec::loadContainer(SettingsContainer& settings, QString fileName)
 
 		if (file.open(QIODevice::ReadOnly|QIODevice::Text))
 		{
-			setText(file.readAll());
-			decodeSettings(settings);
+			QTextStream in{&file};
+			decodeSettings(settings, in);
 		}
 		else
 			throw SerializationException{_EXCPT(QString{"Unable to open file '%1' for reading"}.arg(fileName))};

Grinder/common/serialization/SettingsCodec.h

@@ -13,15 +13,11 @@ namespace grndr
 	class SettingsCodec
 	{
 	public:
-		virtual void encodeSettings(const SettingsContainer& settings) = 0;
-		virtual void decodeSettings(SettingsContainer& settings) = 0;
+		virtual void encodeSettings(const SettingsContainer& settings, QTextStream& stream) = 0;
+		virtual void decodeSettings(SettingsContainer& settings, QTextStream& stream) = 0;
 
 		void saveContainer(const SettingsContainer& settings, QString fileName);
 		void loadContainer(SettingsContainer& settings, QString fileName);
-
-	public:
-		virtual QString getText() const = 0;
-		virtual void setText(const QString& text) = 0;
 	};
 }
 

Grinder/common/serialization/SettingsContainer.cpp

@@ -46,6 +46,13 @@ SettingsContainer& SettingsContainer::operator <<(const SettingsContainer& child
 	return *this;
 }
 
+SettingsContainer* SettingsContainer::createChild(QString name, bool isArray)
+{
+	auto container = std::make_shared<SettingsContainer>(name, isArray);
+	_childContainers.push_back(container);
+	return container.get();
+}
+
 SettingsContainer& SettingsContainer::operator <<(SettingsContainer&& child)
 {
 	auto container = std::make_shared<SettingsContainer>(std::move(child));

Grinder/common/serialization/SettingsContainer.h

@@ -37,6 +37,8 @@ namespace grndr
 		SettingsContainer& operator <<(const SettingsContainer& child);
 		SettingsContainer& operator <<(SettingsContainer&& child);
 
+		SettingsContainer* createChild(QString name, bool isArray = false);
+
 		SettingsContainer* child(QString name) { return _child<SettingsContainer*>(name); }
 		const SettingsContainer* child(QString name) const { return _child<const SettingsContainer*>(name); }
 		const std::vector<SettingsContainer*> children() { return children(""); }

Grinder/ui/properties/PropertyTreeWidget.cpp

@@ -74,7 +74,7 @@ void PropertyTreeWidget::mouseDoubleClickEvent(QMouseEvent* event)
 
 void PropertyTreeWidget::keyPressEvent(QKeyEvent* event)
 {
-	if (event->modifiers() == Qt::NoModifier && (event->key() == Qt::Key_Return || event->key() == Qt::Key_Space))	// Also edit the property when pressing Return or Space
+	if (event->modifiers() == Qt::NoModifier && event->key() == Qt::Key_Return)	// Also edit the property when pressing Return
 		editCurrentItem();
 	else
 		QTreeWidget::keyPressEvent(event);