https://stackoverflow.com/questions/61079767/deserialize-very-large-json-from-a-chunked-array-of-strings-using-system-text-js

CSharp · Console

/Program.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;
using System.Collections.ObjectModel;
using System.Buffers;

using System.Text.Json;
using System.Text.Json.Serialization;

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

//https://stackoverflow.com/questions/61079767/deserialize-very-large-json-from-a-chunked-array-of-strings-using-system-text-js

	public static partial class JsonExtensions
	{
		public static TValue Deserialize<TValue>(IEnumerable<string> buffers, JsonSerializerOptions options = null)
		{
			return Deserialize<TValue>(ToByteArrayChunks(buffers));
		}
		
		public static TValue Deserialize<TValue>(IEnumerable<byte []> buffers, JsonSerializerOptions options = null)
		{
			var sequence = ReadOnlySequenceFactory.Create(buffers);
            var reader = new Utf8JsonReader(sequence, options.GetReaderOptions());						
			return JsonSerializer.Deserialize<TValue>(ref reader, options);
		}

		public static JsonReaderOptions GetReaderOptions(this JsonSerializerOptions options)
		{
			if (options == null)
				return new JsonReaderOptions();
			else
				return new JsonReaderOptions
				{
					AllowTrailingCommas = options.AllowTrailingCommas,
					CommentHandling = options.ReadCommentHandling,
					MaxDepth = options.MaxDepth
				};			
		}
		
		static readonly Encoding encoding = new UTF8Encoding(false);
			
		static IEnumerable<byte []> ToByteArrayChunks(IEnumerable<string> buffers)
		{
			// By using an encoder we can handle the situation in which surrogate pairs enbedded in JSON string literals
			// are split between chunks.
			var encoder = encoding.GetEncoder();
			foreach (var s in buffers)
			{
				ReadOnlySpan<char> charSpan = s;
				var count = encoder.GetByteCount(charSpan, false);
				var bytes = new byte[count];
				Span<byte> byteSpan = bytes;
				encoder.GetBytes(charSpan, byteSpan, false);
				yield return bytes;
			}
		}
	}

    public static class ReadOnlySequenceFactory
    {
		// There is no public concrete implementation of ReadOnlySequenceSegment<T> so we must create one ourselves.
		// This is modeled on https://github.com/dotnet/runtime/blob/master/src/libraries/System.Text.Json/tests/BufferFactory.cs
		// by https://github.com/ahsonkhan
        class ReadOnlyMemorySegment<T> : ReadOnlySequenceSegment<T>
        {
            public static ReadOnlySequence<T> Create(IEnumerable<ReadOnlyMemory<T>> buffers)
            {
                ReadOnlyMemorySegment<T> first = null;
                ReadOnlyMemorySegment<T> current = null;
                foreach (var buffer in buffers)
                {
                    var next = new ReadOnlyMemorySegment<T> { Memory = buffer };
                    if (first == null)
                    {
                        first = next;
                    }
                    else
                    {
                        current.Next = next;
                        next.RunningIndex = current.RunningIndex + current.Memory.Length;
                    }
                    current = next;
                }
                if (first == null)
                {
                    first = current = new ReadOnlyMemorySegment<T>();
                }

                return new ReadOnlySequence<T>(first, 0, current, current.Memory.Length);
            }
        }

        public static ReadOnlySequence<T> Create<T>(IEnumerable<T []> buffers)
        {
            return ReadOnlyMemorySegment<T>.Create(buffers.Select(b => new ReadOnlyMemory<T>(b)));
        }
    }

	public class RootObject
	{
		public int Value { get; set; }
	}

    class TestClass
    {
		const string  hokketonian = "\U00029E3D\u0302";

        public static void Test()
        {
			TestSurrogate();
			
			Test(0, true);
			Test(1, true);
			Test(5, true);
			Test(500, false);
		}
		
        static void Test(int count, bool print)
        {
			var jsonResults = GetJson(count).ToList();

			if (print)
			{
				Console.WriteLine("\nInput JSON:");
				foreach (var s in jsonResults)
					Console.WriteLine(s);
			}
			
			var list = JsonExtensions.Deserialize<List<RootObject>>(jsonResults);
			
			if (print)
			{
				Console.WriteLine("Re-serialized JSON:");
				Console.WriteLine(JsonSerializer.Serialize(list, new JsonSerializerOptions { WriteIndented = true }));
			}
			
			Assert.AreEqual(count, list.Count);
			Assert.IsTrue(Enumerable.Range(0, count).SequenceEqual(list.Select(i => i.Value)));
		}	
		
		static void TestSurrogate()
		{
			int count = 5;

			var expected = string.Concat(Enumerable.Repeat(hokketonian, count));
			
			Console.Write("Expected JSON string: ");
			Console.WriteLine(expected);
			
			var jsonResults = GetJsonWithSurrogateStrings(hokketonian, count);
			
			var s = JsonExtensions.Deserialize<string>(jsonResults);
			
			Console.Write("Actual JSON string  : ");
			Console.WriteLine(s);
			
			Assert.AreEqual(expected, s);
			
		}
		
		static IEnumerable<string> GetJson(int count)
		{
			if (count == 0)
				return new [] { "[]" };
			return new [] { "", "[" }
				.Concat(Enumerable.Range(0, count).Select(i => JsonSerializer.Serialize(new RootObject { Value = i }) + (i < count - 1 ? "," : "")))
				.Concat(new [] { "]" });
		}
		
		static IEnumerable<string> GetJsonWithSurrogateStrings(string input, int count)
		{
			var quote = new [] { "\"" };
			
			return quote.Concat(Enumerable.Repeat(input, 5).SelectMany(s => s).Select(c => c.ToString())).Concat(quote);
		}
	}

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: {0} ({1})", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription , GetNetCoreVersion());
			Console.WriteLine("{0} version: {1}", typeof(JsonSerializer).Assembly.GetName().Name,  typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
		
		public static string GetNetCoreVersion()
		{
			//https://techblog.dorogin.com/how-to-detect-net-core-version-in-runtime-ecd65ad695be
			//
			var assembly = typeof(System.Runtime.GCSettings).GetTypeInfo().Assembly;
			var assemblyPath = assembly.CodeBase.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
			int netCoreAppIndex = Array.IndexOf(assemblyPath, "Microsoft.NETCore.App");
			if (netCoreAppIndex > 0 && netCoreAppIndex < assemblyPath.Length - 2)
				return assemblyPath[netCoreAppIndex + 1];
			return null;
		}
	}