questions/49139691/outofmemoryexception-while-reading-large-xml-file-in-c-sharp-windows-forms | C# Online Compiler

questions/49139691/outofmemoryexception-while-reading-large-xml-file-in-c-sharp-windows-forms by dbc_MinLength

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Serialization;
using System.Xml;
using System.Xml.Linq;
using System.IO;
using System.IO.Compression;

    //https://stackoverflow.com/questions/49139691/outofmemoryexception-while-reading-large-xml-file-in-c-sharp-windows-forms

    public class MeanVarianceTestResultsData
    {
        public int ResultsID { get; set; }
        public DateTime DateExecuted { get; set; }
        public string UserExecuted { get; set; }
        public string CorrectedMean { get; set; }
        public string CorrectedVariance { get; set; }
        public bool TestPassed { get; set; }

        public string TestResultImageFile { get; set; }
        public Stream TestResultImage { get; set; }
    }

    public static class MeanVarianceTestResultsDataExtensions
    {
        public static List<MeanVarianceTestResultsData> ReadResultListFrom(XmlReader reader, Func<MeanVarianceTestResultsData, Stream> openStream, Func<Stream, Stream> closeStream)
        {
            return reader.ReadSubtrees("TestResult").Select(r => ReadResultFrom(r, openStream, closeStream)).ToList();
        }

        public static MeanVarianceTestResultsData ReadResultFrom(XmlReader reader, Func<MeanVarianceTestResultsData, Stream> openStream, Func<Stream, Stream> closeStream)
        {
            if (reader == null || openStream == null)
                throw new ArgumentNullException();
            reader.MoveToContent();
            var result = new MeanVarianceTestResultsData();
            var isEmpty = reader.IsEmptyElement;
            // Read the root
            reader.Read();
            if (isEmpty)
                return result;
            while (!reader.EOF)
            {
                if (reader.NodeType == XmlNodeType.EndElement)
                {
                    reader.Read();
                    break;
                }
                else if (reader.NodeType != XmlNodeType.Element)
                    // Comment, text, CDATA, etc.
                    reader.Skip();
                else if (reader.Name == "ResultsID")
                    result.ResultsID = reader.ReadElementContentAsInt();
                else if (reader.Name == "DateExecuted")
                    result.DateExecuted = reader.ReadElementContentAsDateTime();
                else if (reader.Name == "UserExecuted")
                    result.UserExecuted = reader.ReadElementContentAsString();
                else if (reader.Name == "CorrectedMean")
                    result.CorrectedMean = reader.ReadElementContentAsString();
                else if (reader.Name == "TestPassed")
                    result.TestPassed = reader.ReadElementContentAsBoolean();
                else if (reader.Name == "TestResultImage")
                    result.TestResultImage = reader.ReadElementContentAsStream(() => openStream(result), closeStream);
                else
                    reader.Skip();
            }
            return result;
        }
    }

    public static class XmlReaderExtensions
    {
        public static Stream ReadElementContentAsStream(this XmlReader reader, Func<Stream> openStream, Func<Stream, Stream> closeStream)
        {
            if (reader == null || openStream == null)
                throw new ArgumentNullException();
            Stream stream = null;
            try
            {
                stream = openStream();
                byte[] buffer = new byte[4096];
                int readBytes = 0;
                while ((readBytes = reader.ReadElementContentAsBase64(buffer, 0, buffer.Length)) > 0)
                {
                    stream.Write(buffer, 0, readBytes);
                }
            }
            finally
            {
                if (closeStream != null && stream != null)
                    stream = closeStream(stream);
            }
            return stream;
        }

        public static IEnumerable<XmlReader> ReadSubtrees(this XmlReader reader, string name)
        {
            while (reader.ReadToFollowing(name))
            {
                using (var subReader = reader.ReadSubtree())
                    yield return subReader;
            }
        }
    }

    public class TestClass
    {
        public static void Test()
        {
            var fileName = "Question49159697.xml";
            var resultId = 49;
            var bytes = Enumerable.Range(0, 1024).Select(i => unchecked((byte)i)).ToArray();

            foreach (var xml in GetXml(resultId, bytes))
            {
                try
                {
					Console.WriteLine("\nTesting XML: ");
					Console.WriteLine(xml);
                    File.WriteAllText(fileName, xml);
                    Test(fileName, resultId, bytes);
                    TestToFile(fileName, resultId, bytes);
                }
                finally
                {
                    try
                    {
                        if (File.Exists(fileName))
                            File.Delete(fileName);
                    }
                    catch (Exception)
                    {
                    }
                }
            }

            Console.WriteLine("\nAll tests passed.");
        }

        static void TestToFile(string fileName, int resultId, byte[] bytes)
        {
            List<MeanVarianceTestResultsData> results;
            using (var reader = XmlReader.Create(fileName))
            {
                results = MeanVarianceTestResultsDataExtensions.ReadResultListFrom(
                    reader,
                    m => { m.TestResultImageFile = Path.GetTempFileName(); return File.Open(m.TestResultImageFile, FileMode.Create); },
                    s => { s.Dispose(); return null; });
            }

            Console.WriteLine("Read {0} results of type {1}", results.Count, typeof(MeanVarianceTestResultsData));
            Assert.IsTrue(results.Count == 2);

            foreach (var result in results)
            {
                Assert.IsTrue(File.ReadAllBytes(result.TestResultImageFile).SequenceEqual(bytes));
                Assert.IsTrue(result.ResultsID == resultId);
                File.Delete(result.TestResultImageFile);
            }
        }

        static void Test(string fileName, int resultId, byte[] bytes)
        {
            List<MeanVarianceTestResultsData> results;
            using (var reader = XmlReader.Create(fileName))
            {
                results = MeanVarianceTestResultsDataExtensions.ReadResultListFrom(reader, m => new MemoryStream(), s => { s.Position = 0; return s; });
            }

            Console.WriteLine("Read {0} results of type {1}", results.Count, typeof(MeanVarianceTestResultsData));
            Assert.IsTrue(results.Count == 2);
            foreach (var result in results)
            {
                Assert.IsTrue(((MemoryStream)result.TestResultImage).ToArray().SequenceEqual(bytes));
                Assert.IsTrue(result.ResultsID == resultId);
            }
        }

        static IEnumerable<string> GetXml(int resultId, byte [] bytes)
        {
            var template = @"<Root><SubRoot>
  <TestResult>
    <ResultsID>{0}</ResultsID>
    <DateExecuted>2018-02-20T09:36:12.787</DateExecuted>
    <UserExecuted>xxx</UserExecuted>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
    <CorrectedMean>1966.32245</CorrectedMean>
    <CorrectedVariance>19525.66320199{0}</CorrectedVariance>
    <TestPassed>true</TestPassed>
    <TestResultImage>{1}</TestResultImage>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
  </TestResult>
  <TestResult>Ignore This Text
    <!--Have a comment 
        on multiple lines  -->
    <ResultsID>{0}</ResultsID>
    <DateExecuted>2018-02-20T09:36:12.787</DateExecuted>
    <UserExecuted />
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
    <CorrectedMean>1966.32245</CorrectedMean>
    <CorrectedVariance>19525.66320199{0}</CorrectedVariance>
    <TestPassed>true</TestPassed>
    <TestResultImage>{1}</TestResultImage>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss
		<IrrelevantTag>AA<n />{1}</IrrelevantTag>ss><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss
	</IrrelevantTag>
  </TestResult>
</SubRoot></Root>";
            var xml = string.Format(template, resultId, Convert.ToBase64String(bytes));
            return new[]
            {
                xml,
                XDocument.Parse(xml).ToString(SaveOptions.DisableFormatting),
                XDocument.Parse(xml).ToString(),
            };
        }
    }

    public class AssertionFailedException : System.Exception
    {
        public AssertionFailedException() : base() { }

        public AssertionFailedException(string s) : base(s) { }
    }

    public static class Assert
    {
        public static void IsTrue(bool value)
        {
            IsTrue(value, "failed");
        }

        public static void IsTrue(bool value, string message)
        {
            if (value == false)
                throw new AssertionFailedException(message ?? "failed");
        }
    }
						
	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Roslyn 2.0 Compiler; Environment version: " + Environment.Version);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Uncaught exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}

	static class Path
	{
		// Work around the fact that Path.GetTempFileName() does not work on dotnetfiddle
		public static string GetTempFileName()
		{
			return Guid.NewGuid().ToString();
		}
	}

​x
 
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Serialization;
using System.Xml;
using System.Xml.Linq;
using System.IO;
using System.IO.Compression;
​
    //https://stackoverflow.com/questions/49139691/outofmemoryexception-while-reading-large-xml-file-in-c-sharp-windows-forms
​
    public class MeanVarianceTestResultsData
    {
        public int ResultsID { get; set; }
        public DateTime DateExecuted { get; set; }
        public string UserExecuted { get; set; }
        public string CorrectedMean { get; set; }
        public string CorrectedVariance { get; set; }
        public bool TestPassed { get; set; }
​
        public string TestResultImageFile { get; set; }
        public Stream TestResultImage { get; set; }
    }
​
    public static class MeanVarianceTestResultsDataExtensions
    {
        public static List<MeanVarianceTestResultsData> ReadResultListFrom(XmlReader reader, Func<MeanVarianceTestResultsData, Stream> openStream, Func<Stream, Stream> closeStream)
        {
            return reader.ReadSubtrees("TestResult").Select(r => ReadResultFrom(r, openStream, closeStream)).ToList();
        }
​
        public static MeanVarianceTestResultsData ReadResultFrom(XmlReader reader, Func<MeanVarianceTestResultsData, Stream> openStream, Func<Stream, Stream> closeStream)
        {
            if (reader == null || openStream == null)
                throw new ArgumentNullException();
            reader.MoveToContent();
            var result = new MeanVarianceTestResultsData();
            var isEmpty = reader.IsEmptyElement;
            // Read the root
            reader.Read();
            if (isEmpty)
                return result;
            while (!reader.EOF)
            {
                if (reader.NodeType == XmlNodeType.EndElement)
                {
                    reader.Read();
                    break;
                }
                else if (reader.NodeType != XmlNodeType.Element)
                    // Comment, text, CDATA, etc.
                    reader.Skip();
                else if (reader.Name == "ResultsID")
                    result.ResultsID = reader.ReadElementContentAsInt();
                else if (reader.Name == "DateExecuted")
                    result.DateExecuted = reader.ReadElementContentAsDateTime();
                else if (reader.Name == "UserExecuted")
                    result.UserExecuted = reader.ReadElementContentAsString();
                else if (reader.Name == "CorrectedMean")
                    result.CorrectedMean = reader.ReadElementContentAsString();
                else if (reader.Name == "TestPassed")
                    result.TestPassed = reader.ReadElementContentAsBoolean();
                else if (reader.Name == "TestResultImage")
                    result.TestResultImage = reader.ReadElementContentAsStream(() => openStream(result), closeStream);
                else
                    reader.Skip();
            }
            return result;
        }
    }
​
    public static class XmlReaderExtensions
    {
        public static Stream ReadElementContentAsStream(this XmlReader reader, Func<Stream> openStream, Func<Stream, Stream> closeStream)
        {
            if (reader == null || openStream == null)
                throw new ArgumentNullException();
            Stream stream = null;
            try
            {
                stream = openStream();
                byte[] buffer = new byte[4096];
                int readBytes = 0;
                while ((readBytes = reader.ReadElementContentAsBase64(buffer, 0, buffer.Length)) > 0)
                {
                    stream.Write(buffer, 0, readBytes);
                }
            }
            finally
            {
                if (closeStream != null && stream != null)
                    stream = closeStream(stream);
            }
            return stream;
        }
​
        public static IEnumerable<XmlReader> ReadSubtrees(this XmlReader reader, string name)
        {
            while (reader.ReadToFollowing(name))
            {
                using (var subReader = reader.ReadSubtree())
                    yield return subReader;
            }
        }
    }
​
    public class TestClass
    {
        public static void Test()
        {
            var fileName = "Question49159697.xml";
            var resultId = 49;
            var bytes = Enumerable.Range(0, 1024).Select(i => unchecked((byte)i)).ToArray();
​
            foreach (var xml in GetXml(resultId, bytes))
            {
                try
                {
                    Console.WriteLine("\nTesting XML: ");
                    Console.WriteLine(xml);
                    File.WriteAllText(fileName, xml);
                    Test(fileName, resultId, bytes);
                    TestToFile(fileName, resultId, bytes);
                }
                finally
                {
                    try
                    {
                        if (File.Exists(fileName))
                            File.Delete(fileName);
                    }
                    catch (Exception)
                    {
                    }
                }
            }
​
            Console.WriteLine("\nAll tests passed.");
        }
​
        static void TestToFile(string fileName, int resultId, byte[] bytes)
        {
            List<MeanVarianceTestResultsData> results;
            using (var reader = XmlReader.Create(fileName))
            {
                results = MeanVarianceTestResultsDataExtensions.ReadResultListFrom(
                    reader,
                    m => { m.TestResultImageFile = Path.GetTempFileName(); return File.Open(m.TestResultImageFile, FileMode.Create); },
                    s => { s.Dispose(); return null; });
            }
​
            Console.WriteLine("Read {0} results of type {1}", results.Count, typeof(MeanVarianceTestResultsData));
            Assert.IsTrue(results.Count == 2);
​
            foreach (var result in results)
            {
                Assert.IsTrue(File.ReadAllBytes(result.TestResultImageFile).SequenceEqual(bytes));
                Assert.IsTrue(result.ResultsID == resultId);
                File.Delete(result.TestResultImageFile);
            }
        }
​
        static void Test(string fileName, int resultId, byte[] bytes)
        {
            List<MeanVarianceTestResultsData> results;
            using (var reader = XmlReader.Create(fileName))
            {
                results = MeanVarianceTestResultsDataExtensions.ReadResultListFrom(reader, m => new MemoryStream(), s => { s.Position = 0; return s; });
            }
​
            Console.WriteLine("Read {0} results of type {1}", results.Count, typeof(MeanVarianceTestResultsData));
            Assert.IsTrue(results.Count == 2);
            foreach (var result in results)
            {
                Assert.IsTrue(((MemoryStream)result.TestResultImage).ToArray().SequenceEqual(bytes));
                Assert.IsTrue(result.ResultsID == resultId);
            }
        }
​
        static IEnumerable<string> GetXml(int resultId, byte [] bytes)
        {
            var template = @"<Root><SubRoot>
  <TestResult>
    <ResultsID>{0}</ResultsID>
    <DateExecuted>2018-02-20T09:36:12.787</DateExecuted>
    <UserExecuted>xxx</UserExecuted>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
    <CorrectedMean>1966.32245</CorrectedMean>
    <CorrectedVariance>19525.66320199{0}</CorrectedVariance>
    <TestPassed>true</TestPassed>
    <TestResultImage>{1}</TestResultImage>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
  </TestResult>
  <TestResult>Ignore This Text
    <!--Have a comment 
        on multiple lines  -->
    <ResultsID>{0}</ResultsID>
    <DateExecuted>2018-02-20T09:36:12.787</DateExecuted>
    <UserExecuted />
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss</IrrelevantTag>
    <CorrectedMean>1966.32245</CorrectedMean>
    <CorrectedVariance>19525.66320199{0}</CorrectedVariance>
    <TestPassed>true</TestPassed>
    <TestResultImage>{1}</TestResultImage>
    <IrrelevantTag><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss
        <IrrelevantTag>AA<n />{1}</IrrelevantTag>ss><IrrelevantTag>AA<n />{1}</IrrelevantTag>ss
    </IrrelevantTag>
  </TestResult>
</SubRoot></Root>";
            var xml = string.Format(template, resultId, Convert.ToBase64String(bytes));
            return new[]
            {
                xml,
                XDocument.Parse(xml).ToString(SaveOptions.DisableFormatting),
                XDocument.Parse(xml).ToString(),
            };
        }
    }
​
    public class AssertionFailedException : System.Exception
    {
        public AssertionFailedException() : base() { }
​
        public AssertionFailedException(string s) : base(s) { }
    }
​
    public static class Assert
    {
        public static void IsTrue(bool value)
        {
            IsTrue(value, "failed");
        }
​
        public static void IsTrue(bool value, string message)
        {
            if (value == false)
                throw new AssertionFailedException(message ?? "failed");
        }
    }
                        
    public class Program
    {
        public static void Main()
        {
            Console.WriteLine("Roslyn 2.0 Compiler; Environment version: " + Environment.Version);
            Console.WriteLine();
​
            try
            {
                TestClass.Test();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Uncaught exception: ");
                Console.WriteLine(ex);
                throw;
            }
        }
    }
​
    static class Path
    {
        // Work around the fact that Path.GetTempFileName() does not work on dotnetfiddle
        public static string GetTempFileName()
        {
            return Guid.NewGuid().ToString();
        }
    }
​

Environment version: .NET 9.0.0 (9.0.0)
Json.NET version: Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed

{"PolicyHolderInfo":{"ActionDate":"20190912","EmployeeCode":"EPL2","LocalEntityCode":"K","PolicyHolderCode":"1YAAAAC","Name1":"monoliability PDL","Name2":"test","Street1":"-","Street2":"-","City":"-","PostCode":"-","Subdivision":"01","FMEPaymentFrequency1":"00","Reference":"C00-AAC"},"PolicyHolderTrailerInfo":{"RecordCode":"Z","CountryCode":"KSA","SourceIndicator":"EPLC","RecordSequence":0,"LinkNumber":0,"RecordType":"Z","ReturnCode":null,"Reference":"1-AC"},"SourceSystemDate":"2019-09-17T09:48:31.3543314+00:00"}

Cached Result
Last Run:	12:59:26 pm
Compile:	0.015s
Execute:	0.14s
Memory:	16.11Mb
CPU:	0.155s

View IL Code