Revised debugging in JSONToSimpl classes; added new JsonToSimplPortalFileMaster to read portal system/template files and merge; copied merge code in from Essentials and reworked (note that Essentials does not yet use this merge code); added 'global' object to the top-level objects that are merged

Pepperdash Core/Pepperdash Core/Comm/GenericHttpSseClient.cs

@@ -97,7 +97,6 @@ namespace PepperDash.Core
 
         private void GetRequestStreamCallback(HttpClientRequest request, HTTP_CALLBACK_ERROR error, object status)
         {
-#warning Explore simplifying this later....
 
             try
             {
@@ -136,8 +135,6 @@ namespace PepperDash.Core
 
                 Debug.Console(1, this, "Client Disconnected");
 
-#warning Need to explore using a heartbeat to verify connection to the server
-
                 Stream streamResponse = response.ContentStream;
                 // Object containing various states to be passed back to async callback below
                 RequestState asyncState = new RequestState();

Pepperdash Core/Pepperdash Core/Config/PortalConfigReader.cs

@@ -0,0 +1,177 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Crestron.SimplSharp;
+using Crestron.SimplSharp.CrestronIO;
+
+using PepperDash.Core;
+
+namespace PepperDash.Core.Config
+{
+	public class PortalConfigReader
+	{
+		/// <summary>
+		/// Reads the config file, checks if it needs a merge, merges and saves, then returns the merged Object.
+		/// </summary>
+		/// <returns>JObject of config file</returns>
+		public static void ReadAndMergeFileIfNecessary(string filePath, string savePath)
+		{
+			try
+			{
+				if (!File.Exists(filePath))
+				{
+					Debug.Console(1, Debug.ErrorLogLevel.Error,
+						"ERROR: Configuration file not present. Please load file to {0} and reset program", filePath);
+				}
+
+				using (StreamReader fs = new StreamReader(filePath))
+				{
+					var jsonObj = JObject.Parse(fs.ReadToEnd());
+					if(jsonObj["template"] != null && jsonObj["system"] != null)
+					{
+						// it's a double-config, merge it.
+						var merged = MergeConfigs(jsonObj);
+						if (jsonObj["system_url"] != null)
+						{
+							merged["systemUrl"] = jsonObj["system_url"].Value<string>();
+						}
+
+						if (jsonObj["template_url"] != null)
+						{
+							merged["templateUrl"] = jsonObj["template_url"].Value<string>();
+						}
+
+						jsonObj = merged;
+					}
+
+					using (StreamWriter fw = new StreamWriter(savePath))
+					{
+						fw.Write(jsonObj.ToString(Formatting.Indented));
+						Debug.Console(1, "JSON config merged and saved to {0}", savePath);
+					}
+
+				}
+			}
+			catch (Exception e)
+			{
+				Debug.Console(1, Debug.ErrorLogLevel.Error, "ERROR: Config load failed: \r{0}", e);
+			}
+		}
+
+		/// <summary>
+		/// 
+		/// </summary>
+		/// <param name="doubleConfig"></param>
+		/// <returns></returns>
+		static JObject MergeConfigs(JObject doubleConfig)
+		{
+			var system = JObject.FromObject(doubleConfig["system"]);
+			var template = JObject.FromObject(doubleConfig["template"]);
+			var merged = new JObject();
+
+			// Put together top-level objects
+			if (system["info"] != null)
+				merged.Add("info", Merge(template["info"], system["info"]));
+			else
+				merged.Add("info", template["info"]);
+
+			merged.Add("devices", MergeArraysOnTopLevelProperty(template["devices"] as JArray,
+				system["devices"] as JArray, "uid"));
+
+			if (system["rooms"] == null)
+				merged.Add("rooms", template["rooms"]);
+			else
+				merged.Add("rooms", MergeArraysOnTopLevelProperty(template["rooms"] as JArray,
+					system["rooms"] as JArray, "key"));
+
+			if (system["sourceLists"] == null)
+				merged.Add("sourceLists", template["sourceLists"]);
+			else
+				merged.Add("sourceLists", Merge(template["sourceLists"], system["sourceLists"]));
+
+			// Template tie lines take precedence.  Config tool doesn't do them at system
+			// level anyway...
+			if (template["tieLines"] != null)
+				merged.Add("tieLines", template["tieLines"]);
+			else if (system["tieLines"] != null)
+				merged.Add("tieLines", system["tieLines"]);
+			else
+				merged.Add("tieLines", new JArray());
+
+			if (system["global"] != null)
+				merged.Add("global", Merge(template["global"], system["global"]));
+			else
+				merged.Add("global", template["global"]);
+
+			Debug.Console(2, "MERGED CONFIG RESULT: \x0d\x0a{0}", merged);
+			return merged;
+		}
+
+		/// <summary>
+		/// Merges the contents of a base and a delta array, matching the entries on a top-level property
+		/// given by propertyName.  Returns a merge of them. Items in the delta array that do not have
+		/// a matched item in base array will not be merged. 
+		/// </summary>
+		static JArray MergeArraysOnTopLevelProperty(JArray a1, JArray a2, string propertyName)
+		{
+			var result = new JArray();
+			if (a2 == null)
+				result = a1;
+			else if (a1 != null)
+			{
+				for (int i = 0; i < a1.Count(); i++)
+				{
+					var a1Dev = a1[i];
+					// Try to get a system device and if found, merge it onto template
+					var a2Match = a2.FirstOrDefault(t => t[propertyName].Equals(a1Dev[propertyName]));// t.Value<int>("uid") == tmplDev.Value<int>("uid"));
+					if (a2Match != null)
+					{
+						var mergedItem = Merge(a1Dev, a2Match);// Merge(JObject.FromObject(a1Dev), JObject.FromObject(a2Match));
+						result.Add(mergedItem);
+					}
+					else
+						result.Add(a1Dev);
+				}
+			}
+			return result;
+		}
+
+
+		/// <summary>
+		/// Helper for using with JTokens.  Converts to JObject 
+		/// </summary>
+		static JObject Merge(JToken t1, JToken t2)
+		{
+			return Merge(JObject.FromObject(t1), JObject.FromObject(t2));
+		}
+
+		/// <summary>
+		/// Merge b ONTO a
+		/// </summary>
+		/// <param name="a"></param>
+		/// <param name="b"></param>
+		static JObject Merge(JObject o1, JObject o2)
+		{
+			foreach (var o2Prop in o2)
+			{
+				var o1Value = o1[o2Prop.Key];
+				if (o1Value == null)
+					o1.Add(o2Prop.Key, o2Prop.Value);
+				else
+				{
+					JToken replacement = null;
+					if (o2Prop.Value.HasValues && o1Value.HasValues) // Drill down
+						replacement = Merge(JObject.FromObject(o1Value), JObject.FromObject(o2Prop.Value));
+					else
+						replacement = o2Prop.Value;
+					o1[o2Prop.Key].Replace(replacement);
+				}
+			}
+			return o1;
+		}
+	}
+}

Pepperdash Core/Pepperdash Core/JsonToSimpl/JsonToSimplArrayLookupChild.cs

@@ -67,7 +67,7 @@ namespace PepperDash.Core.JsonToSimpl
 					});
 					if (item == null)
 					{
-						Debug.Console(1, "Child[{0}] Array '{1}' '{2}={3}' not found: ", Key,
+						Debug.Console(1, "JSON Child[{0}] Array '{1}' '{2}={3}' not found: ", Key,
 							PathPrefix, SearchPropertyName, SearchPropertyValue);
 						this.LinkedToObject = false;
 						return false;
@@ -75,17 +75,17 @@ namespace PepperDash.Core.JsonToSimpl
 
 					this.LinkedToObject = true;
 					ArrayIndex = array.IndexOf(item);
-					Debug.Console(1, "Child[{0}] Found array match at index {1}", Key, ArrayIndex);
+					Debug.Console(1, "JSON Child[{0}] Found array match at index {1}", Key, ArrayIndex);
 					return true;
 				}
 				catch (Exception e)
 				{
-					Debug.Console(1, "Child[{0}] Array '{1}' lookup error: '{2}={3}'\r{4}", Key, 
+					Debug.Console(1, "JSON Child[{0}] Array '{1}' lookup error: '{2}={3}'\r{4}", Key, 
 						PathPrefix, SearchPropertyName, SearchPropertyValue, e);
 				}
 			}
 			else
-				Debug.Console(1, "Child[{0}] Path '{1}' is not an array", Key, PathPrefix);
+				Debug.Console(1, "JSON Child[{0}] Path '{1}' is not an array", Key, PathPrefix);
 
 			return false;
 		}

Pepperdash Core/Pepperdash Core/JsonToSimpl/JsonToSimplChildObjectBase.cs

@@ -170,7 +170,7 @@ namespace PepperDash.Core.JsonToSimpl
 		bool Process(string path, out string response)
 		{
 			path = GetFullPath(path);
-			Debug.Console(1, "Child[{0}] Processing {1}", Key, path); 
+			Debug.Console(1, "JSON Child[{0}] Processing {1}", Key, path); 
 			response = "";
 			if (Master == null)
 			{
@@ -266,7 +266,7 @@ namespace PepperDash.Core.JsonToSimpl
 			var path = GetFullPath(keyPath);
 			try
 			{
-				Debug.Console(1, "Child[{0}] Queueing value on master {1}='{2}'", Key, path, valueToSave);
+				Debug.Console(1, "JSON Child[{0}] Queueing value on master {1}='{2}'", Key, path, valueToSave);
 
 				//var token = Master.JsonObject.SelectToken(path);
 				//if (token != null) // The path exists in the file
@@ -274,7 +274,7 @@ namespace PepperDash.Core.JsonToSimpl
 			}
 			catch (Exception e)
 			{
-				Debug.Console(1, "Child[{0}] Failed setting value for path '{1}'\r{2}", Key, path, e);
+				Debug.Console(1, "JSON Child[{0}] Failed setting value for path '{1}'\r{2}", Key, path, e);
 			}
 		}
 

Pepperdash Core/Pepperdash Core/JsonToSimpl/JsonToSimplPortalFileMaster.cs

@@ -0,0 +1,219 @@
+using System;
+//using System.IO;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Crestron.SimplSharp;
+using Crestron.SimplSharp.CrestronIO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using PepperDash.Core.Config;
+
+namespace PepperDash.Core.JsonToSimpl
+{
+    public class JsonToSimplPortalFileMaster : JsonToSimplMaster
+    {
+		/// <summary>
+		///  Sets the filepath as well as registers this with the Global.Masters list
+		/// </summary>
+		public string PortalFilepath { get; private set; }
+
+		string ActualFilePath;
+
+		/*****************************************************************************************/
+		/** Privates **/
+
+		// To prevent multiple same-file access
+		object StringBuilderLock = new object();
+		static object FileLock = new object();
+
+		/*****************************************************************************************/
+
+		/// <summary>
+        /// SIMPL+ default constructor.
+        /// </summary>
+		public JsonToSimplPortalFileMaster()
+        {
+		}
+
+		/// <summary>
+		/// Read, evaluate and udpate status
+		/// </summary>
+		public void EvaluateFile(string portalFilepath)
+		{
+			PortalFilepath = portalFilepath;
+
+			OnBoolChange(false, 0, JsonToSimplConstants.JsonIsValidBoolChange);
+			if (string.IsNullOrEmpty(PortalFilepath))
+			{
+				CrestronConsole.PrintLine("Cannot evaluate file. JSON file path not set");
+				return;
+			}
+
+			// Resolve possible wildcarded filename
+
+			// If the portal file is xyz.json, then 
+			// the file we want to check for first will be called xyz.local.json
+			var localFilepath = Path.ChangeExtension(PortalFilepath, "local.json");
+			Debug.Console(0, this, "Checking for local file {0}", localFilepath);
+			var actualLocalFile = GetActualFileInfoFromPath(localFilepath);
+
+			if (actualLocalFile != null)
+			{
+				ActualFilePath = actualLocalFile.FullName;
+			}
+			// If the local file does not exist, then read the portal file xyz.json
+			// and create the local.
+			else
+			{
+				Debug.Console(1, this, "Local JSON file not found {0}\rLoading portal JSON file", localFilepath);
+				var actualPortalFile = GetActualFileInfoFromPath(portalFilepath);
+				if (actualPortalFile != null)
+				{
+					var newLocalPath = Path.ChangeExtension(actualPortalFile.FullName, "local.json");
+					// got the portal file, hand off to the merge / save method				
+					PortalConfigReader.ReadAndMergeFileIfNecessary(actualPortalFile.FullName, newLocalPath);
+					ActualFilePath = newLocalPath;
+				}
+				else
+				{
+					var msg = string.Format("Portal JSON file not found: {0}", PortalFilepath);
+					Debug.Console(1, this, msg);
+					ErrorLog.Error(msg);
+					return;
+				}
+			}
+
+			// At this point we should have a local file.  Do it.
+			Debug.Console(1, "Reading local JSON file {0}", ActualFilePath);
+
+			string json = File.ReadToEnd(ActualFilePath, System.Text.Encoding.ASCII);
+
+			try
+			{
+				JsonObject = JObject.Parse(json);
+				foreach (var child in Children)
+					child.ProcessAll();
+				OnBoolChange(true, 0, JsonToSimplConstants.JsonIsValidBoolChange);
+			}
+			catch (Exception e)
+			{
+				var msg = string.Format("JSON parsing failed:\r{0}", e);
+				CrestronConsole.PrintLine(msg);
+				ErrorLog.Error(msg);
+				return;
+			}
+		}
+
+		/// <summary>
+		/// Returns the FileInfo object for a given path, with possible wildcards
+		/// </summary>
+		/// <param name="path"></param>
+		/// <returns></returns>
+		FileInfo GetActualFileInfoFromPath(string path)
+		{
+			var dir = Path.GetDirectoryName(path);
+			var localFilename = Path.GetFileName(path);
+			var directory = new DirectoryInfo(dir);
+			// search the directory for the file w/ wildcards
+			return directory.GetFiles(localFilename).FirstOrDefault();
+		}
+
+		/// <summary>
+		/// 
+		/// </summary>
+		/// <param name="level"></param>
+		public void setDebugLevel(int level)
+		{
+			Debug.SetDebugLevel(level);
+		}
+
+		/// <summary>
+		/// 
+		/// </summary>
+		public override void Save()
+		{
+			// this code is duplicated in the other masters!!!!!!!!!!!!!
+			UnsavedValues = new Dictionary<string, JValue>();
+			// Make each child update their values into master object
+			foreach (var child in Children)
+			{
+				Debug.Console(1, "Master [{0}] checking child [{1}] for updates to save", UniqueID, child.Key);
+				child.UpdateInputsForMaster();
+			}
+
+			if (UnsavedValues == null || UnsavedValues.Count == 0)
+			{
+				Debug.Console(1, "Master [{0}] No updated values to save. Skipping", UniqueID);
+				return;
+			}
+			lock (FileLock)
+			{
+				Debug.Console(1, "Saving");
+				foreach (var path in UnsavedValues.Keys)
+				{
+					var tokenToReplace = JsonObject.SelectToken(path);
+					if (tokenToReplace != null)
+					{// It's found
+						tokenToReplace.Replace(UnsavedValues[path]);
+						Debug.Console(1, "JSON Master[{0}] Updating '{1}'", UniqueID, path);
+					}
+					else // No token.  Let's make one 
+					{
+						//http://stackoverflow.com/questions/17455052/how-to-set-the-value-of-a-json-path-using-json-net
+						Debug.Console(1, "JSON Master[{0}] Cannot write value onto missing property: '{1}'", UniqueID, path);
+						
+//                        JContainer jpart = JsonObject;
+//                        // walk down the path and find where it goes
+//#warning Does not handle arrays.
+//                        foreach (var part in path.Split('.'))
+//                        {
+
+//                            var openPos = part.IndexOf('[');
+//                            if (openPos > -1)
+//                            {
+//                                openPos++; // move to number
+//                                var closePos = part.IndexOf(']');
+//                                var arrayName = part.Substring(0, openPos - 1); // get the name
+//                                var index = Convert.ToInt32(part.Substring(openPos, closePos - openPos));
+
+//                                // Check if the array itself exists and add the item if so
+//                                if (jpart[arrayName] != null)
+//                                {
+//                                    var arrayObj = jpart[arrayName] as JArray;
+//                                    var item = arrayObj[index];
+//                                    if (item == null)
+//                                        arrayObj.Add(new JObject());
+//                                }
+								
+//                                Debug.Console(0, "IGNORING MISSING ARRAY VALUE FOR NOW");
+//                                continue;
+//                            }
+//                            // Build the 
+//                            if (jpart[part] == null)
+//                                jpart.Add(new JProperty(part, new JObject()));
+//                            jpart = jpart[part] as JContainer;
+//                        }
+//                        jpart.Replace(UnsavedValues[path]);
+					}
+				}
+				using (StreamWriter sw = new StreamWriter(ActualFilePath))
+				{
+					try
+					{
+						sw.Write(JsonObject.ToString());
+						sw.Flush();
+					}
+					catch (Exception e)
+					{
+						string err = string.Format("Error writing JSON file:\r{0}", e);
+						Debug.Console(0, err);
+						ErrorLog.Warn(err);
+						return;
+					}
+				}
+			}
+		}
+	}
+}

Pepperdash Core/Pepperdash Core/Logging/Debug.cs

@@ -39,7 +39,7 @@ namespace PepperDash.Core
 		/// </summary>
 		static bool ExcludeAllMode;
 
-		static bool ExcludeNoKeyMessages;
+		//static bool ExcludeNoKeyMessages;
 
 		static Dictionary<string, object> IncludedExcludedKeys;
 
@@ -141,14 +141,14 @@ namespace PepperDash.Core
 					IncludedExcludedKeys.Clear();
 					ExcludeAllMode = true;
 				}
-				else if (lkey == "+nokey")
-				{
-					ExcludeNoKeyMessages = false;
-				}
-				else if (lkey == "-nokey")
-				{
-					ExcludeNoKeyMessages = true;
-				}
+				//else if (lkey == "+nokey")
+				//{
+				//    ExcludeNoKeyMessages = false;
+				//}
+				//else if (lkey == "-nokey")
+				//{
+				//    ExcludeNoKeyMessages = true;
+				//}
 				else
 				{
 					string key = null; ;

Pepperdash Core/Pepperdash Core/PepperDash_Core.csproj

@@ -74,7 +74,9 @@
     <Compile Include="Comm\CommunicationGather.cs" />
     <Compile Include="Comm\EventArgs.cs" />
     <Compile Include="Comm\GenericSshClient.cs" />
+    <Compile Include="Config\PortalConfigReader.cs" />
     <Compile Include="CoreInterfaces.cs" />
+    <Compile Include="JsonToSimpl\JsonToSimplPortalFileMaster.cs" />
     <Compile Include="Logging\Debug.cs" />
     <Compile Include="Logging\DebugContext.cs" />
     <Compile Include="Logging\DebugMemory.cs" />