[Fork] [Fork] questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file

CSharp · Console

/Program.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;
using System.Threading;
using System.Collections.Concurrent;
using System.Threading.Tasks;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;	

using NUnit.Framework;

    //https://stackoverflow.com/questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file
    public class PressureMap
    {
        public double[,] PressureMatrix { get; set; }
    }

    public class CalibrationConfiguration 
    {
        // Data model not included in question
    }

    public class RepresentationConfiguration 
    {
        // Data model not included in question
    }

    public class RecordedDataHeader
    {
        public string SoftwareVersion { get; set; }
        public CalibrationConfiguration CalibrationConfiguration { get; set; }
        public RepresentationConfiguration RepresentationConfiguration { get; set; }
    }

    public class RecordedData
    {
        // Ensure the header is serialized first.
        [JsonProperty(Order = 1)]
        public RecordedDataHeader RecordedDataHeader { get; set; }
        // Ensure the pressure data is serialized last.
        [JsonProperty(Order = 2)]
        public IEnumerable<PressureMap> PressureData { get; set; }
    }

    //https://stackoverflow.com/questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file
    class TestClass
    {
        public static void Test()
        {
            var filePath = "Question52311362.json";

            new TestClass { filePath = filePath }.SampleAndWritePressureData();

            Console.WriteLine(File.ReadAllText(filePath));

            File.Delete(filePath);
        }

        public string filePath;

        public void SampleAndWritePressureData()
        {
            var calibrationConfiguration = new CalibrationConfiguration();
            var representationConfiguration = new RepresentationConfiguration();
            var softwareVersion = Assembly.GetExecutingAssembly().GetName().Version.Major.ToString() + "." + Assembly.GetExecutingAssembly().GetName().Version.Minor.ToString();

            var sampleCount = 400;    // Or whatever stopping criterion you prefer
            var sampleInterval = 10;  // in ms

			using (var pressureData = new BlockingCollection<PressureMap>())
			{
				// Adapted from
				// https://docs.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview
				// https://docs.microsoft.com/en-us/dotnet/api/system.collections.concurrent.blockingcollection-1?view=netframework-4.7.2

				// Spin up a Task to sample the pressure maps
			    using (Task t1 = Task.Factory.StartNew(() =>
				{
					for (int i = 0; i < sampleCount; i++)
					{
						var data = GetPressureMap(i);
						Console.WriteLine("Generated sample {0}", i);
						pressureData.Add(data);
						System.Threading.Thread.Sleep(sampleInterval);
					}
					pressureData.CompleteAdding();
				}))
				{
					// Spin up a Task to consume the BlockingCollection
					using (Task t2 = Task.Factory.StartNew(() =>
					{
						var recordedDataHeader = new RecordedDataHeader
						{
							SoftwareVersion = softwareVersion,
							CalibrationConfiguration = calibrationConfiguration,
							RepresentationConfiguration = representationConfiguration,
						};

						var settings = new JsonSerializerSettings
						{
							ContractResolver = new CamelCasePropertyNamesContractResolver(),
						};

						using (var stream = new FileStream(this.filePath, FileMode.Create))
						using (var textWriter = new StreamWriter(stream))
						using (var jsonWriter = new JsonTextWriter(textWriter))
						{
							int j = 0;
							
							var query = pressureData
								.GetConsumingEnumerable()
								.Select(p => 
										{ 
											// Flush the writer periodically in case the process terminates abnormally
											jsonWriter.Flush();
											Console.WriteLine("Serializing item {0}", j++);
											return p;
										});
							
							var recordedData = new RecordedData
							{
								RecordedDataHeader = recordedDataHeader,
								// Since PressureData is declared as IEnumerable<PressureMap>, evaluation will be lazy.
								PressureData = query,
							};							

							Console.WriteLine("Beginning serialization of {0} to {1}:", recordedData, this.filePath);
							JsonSerializer.CreateDefault(settings).Serialize(textWriter, recordedData);
							Console.WriteLine("Finished serialization of {0} to {1}.", recordedData, this.filePath);
						}
					}))
					{
						Task.WaitAll(t1, t2);
					}
				}
			}
	    }
		
        PressureMap GetPressureMap(int count)
        {
            return new PressureMap
            {
                PressureMatrix = new double[,] { { (double)count } }
            };
        }
    }

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: " + Environment.Version);
			Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}