[Fork] questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file

CSharp · Console

/Program.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;
using System.Threading;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;	

using NUnit.Framework;

    //https://stackoverflow.com/questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file
    public class PressureMap
    {
        public double[,] PressureMatrix { get; set; }
    }

    public class CalibrationConfiguration 
    {
        // Data model not included in question
    }

    public class RepresentationConfiguration 
    {
        // Data model not included in question
    }

    public class RecordedDataHeader
    {
        public string SoftwareVersion { get; set; }
        public CalibrationConfiguration CalibrationConfiguration { get; set; }
        public RepresentationConfiguration RepresentationConfiguration { get; set; }
    }

    public class RecordedData
    {
        // Ensure the header is serialized first.
        [JsonProperty(Order = 1)]
        public RecordedDataHeader RecordedDataHeader { get; set; }
        // Ensure the pressure data is serialized last.
        [JsonProperty(Order = 2)]
        public IEnumerable<PressureMap> PressureData { get; set; }
    }

    //https://stackoverflow.com/questions/52311362/c-sharp-outofmemoryexception-saving-a-list-on-a-json-file
    public static partial class JsonExtensions
    {
        // Adapted from the answer to
        // https://stackoverflow.com/questions/44787652/serialize-as-ndjson-using-json-net
        // by dbc https://stackoverflow.com/users/3744182/dbc
		public static void ToNewlineDelimitedJson<T>(Stream stream, IEnumerable<T> items)
		{
			// Let caller dispose the underlying stream 
			using (var textWriter = new StreamWriter(stream, new UTF8Encoding(false, true), 1024, true))
			{
				ToNewlineDelimitedJson(textWriter, items);
			}
		}


        public static void ToNewlineDelimitedJson<T>(TextWriter textWriter, IEnumerable<T> items)
        {
            var serializer = JsonSerializer.CreateDefault();

            foreach (var item in items)
            {
                // Formatting.None is the default; I set it here for clarity.
                using (var writer = new JsonTextWriter(textWriter) { Formatting = Formatting.None, CloseOutput = false })
                {
                    serializer.Serialize(writer, item);
                }
                // http://specs.okfnlabs.org/ndjson/
                // Each JSON text MUST conform to the [RFC7159] standard and MUST be written to the stream followed by the newline character \n (0x0A). 
                // The newline charater MAY be preceeded by a carriage return \r (0x0D). The JSON texts MUST NOT contain newlines or carriage returns.
                textWriter.Write("\n");
            }
        }

        // Adapted from the answer to 
        // https://stackoverflow.com/questions/29729063/line-delimited-json-serializing-and-de-serializing
        // by Yuval Itzchakov https://stackoverflow.com/users/1870803/yuval-itzchakov
        public static IEnumerable<TBase> FromNewlineDelimitedJson<TBase, THeader, TRow>(TextReader reader)
            where THeader : TBase
            where TRow : TBase
        {
            bool first = true;

            using (var jsonReader = new JsonTextReader(reader) { CloseInput = false, SupportMultipleContent = true })
            {
                var serializer = JsonSerializer.CreateDefault();

                while (jsonReader.Read())
                {
                    if (jsonReader.TokenType == JsonToken.Comment)
                        continue;
                    if (first)
                    {
                        yield return serializer.Deserialize<THeader>(jsonReader);
						first = false;
                    }
                    else
                    {
                        yield return serializer.Deserialize<TRow>(jsonReader);
                    }
                }
            }
        }
    }

    class TestClass
    {
        public static void Test()
        {
            var filePath = "Question52311362.json";

            new TestClass { filePath = filePath }.SampleAndWritePressureData();

            //Console.WriteLine(File.ReadAllText(filePath));

            using (var stream = File.OpenRead(filePath))
            using (var textReader = new StreamReader(stream))
            {
                foreach (var obj in JsonExtensions.FromNewlineDelimitedJson<object, RecordedDataHeader, PressureMap>(textReader))
                {
                    if (obj is RecordedDataHeader)
                    {
                        var header = (RecordedDataHeader)obj;
                        // Process the header
                        Console.WriteLine(JsonConvert.SerializeObject(header));
                    }
                    else
                    {
                        var row = (PressureMap)obj;
                        // Process the row.
                        Console.WriteLine(JsonConvert.SerializeObject(row));
                    }
                }
            }

            File.Delete(filePath);
        }

        public string filePath;

        public void SampleAndWritePressureData()
        {
            var calibrationConfiguration = new CalibrationConfiguration();
            var representationConfiguration = new RepresentationConfiguration();
            var softwareVersion = Assembly.GetExecutingAssembly().GetName().Version.Major.ToString() + "." + Assembly.GetExecutingAssembly().GetName().Version.Minor.ToString();

            var sampleCount = 12; // Or whatever
            var sampleInterval = 10;

            var recordedDataHeader = new RecordedDataHeader
            {
                SoftwareVersion = softwareVersion,
                CalibrationConfiguration = calibrationConfiguration,
                RepresentationConfiguration = representationConfiguration,
            };

            var settings = new JsonSerializerSettings
            {
                ContractResolver = new CamelCasePropertyNamesContractResolver(),
            };

            // Write the header
            Console.WriteLine("Beginning serialization of sample data to {0}.", this.filePath);

            using (var stream = new FileStream(this.filePath, FileMode.Create))
            {
                JsonExtensions.ToNewlineDelimitedJson(stream, new[] { recordedDataHeader });
            }

            // Write each sample incrementally

            for (int i = 0; i < sampleCount; i++)
            {
                Thread.Sleep(sampleInterval);
                Console.WriteLine("Performing sample {0} of {1}", i, sampleCount);
                var map = GetPressureMap(i);

                using (var stream = new FileStream(this.filePath, FileMode.Append))
                {
                    JsonExtensions.ToNewlineDelimitedJson(stream, new[] { map });
                }
            }

            Console.WriteLine("Finished serialization of sample data to {0}.", this.filePath);
        }

        PressureMap GetPressureMap(int count)
        {
            return new PressureMap
            {
                PressureMatrix = new double[,] { { (double)count } }
            };
        }
    }

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: " + Environment.Version);
			Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}