#define DEBUG

​

using System;

using System.Collections;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Runtime.Serialization.Formatters;

using System.IO;

using System.Data;

using System.IO.Compression;

using System.Threading.Tasks;

using Microsoft.IO;

using System.Threading;

​

using System.Text.Json;

using System.Text.Json.Serialization;

using System.Text.Json.Serialization.Metadata;

using System.Text.Json.Nodes;

​

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

​

//https://stackoverflow.com/questions/79221507/how-do-i-efficiently-deserialize-a-compressed-list-of-objects-add-to-it-and-co

#nullable enable

​

    public static class JsonExtensions

    {

        // nugets required: 

        // System.Linq.Async, https://www.nuget.org/packages/System.Linq.Async/

        // Microsoft.IO.RecyclableMemoryStream https://www.nuget.org/packages/microsoft.io.recyclablememorystream/

        public static readonly RecyclableMemoryStreamManager MemoryStreamManager = new RecyclableMemoryStreamManager();

        // 2x buffer sized as recommended by Bradley Grainger, https://faithlife.codes/blog/2012/06/always-wrap-gzipstream-with-bufferedstream/

        // But anything smaller than 85,000 bytes should be OK, since objects larger than that go on the large object heap.  See:

        // https://learn.microsoft.com/en-us/dotnet/standard/garbage-collection/large-object-heap

        const int BufferSize = 16384;       

​

        // Compressed copy + serialize

        public static async Task<byte []> CopyAndAddToCompressedByteArrayAsync<TItem>(byte [] input, IEnumerable<TItem> newItems, JsonSerializerOptions? options = default)

        {

            using var inputStream = new MemoryStream(input);

            using var outputStream = MemoryStreamManager.GetStream();

            await CopyAndAddToCompressedStreamAsync(inputStream, outputStream, newItems, options);

            return outputStream.ToArray();

        }

        public static async Task CopyAndAddToCompressedFileAsync<TItem>(string inputPath, string outputPath, IEnumerable<TItem> newItems, JsonSerializerOptions? options = default)

        {

            await using var input = File.OpenRead(inputPath);

            await using var output = File.OpenWrite(outputPath);

            await CopyAndAddToCompressedStreamAsync(input, output, newItems, options);

        }

        public static async Task CopyAndAddToCompressedStreamAsync<TItem>(Stream input, Stream output, IEnumerable<TItem> newItems, JsonSerializerOptions? options = default)

        {

            options ??= JsonSerializerOptions.Default;

            await using var inputDecompressor = new GZipStream(input, CompressionMode.Decompress, leaveOpen : true);

            await using var outputCompressor = new GZipStream(output, CompressionMode.Compress, leaveOpen : true);

            await using var outputBuffer = new BufferedStream(outputCompressor, BufferSize);

​

#if DEBUG

            var length = output.Length;

#endif

​

            // ToAsyncEnumerable() and  oldItemsAsync!.Concat(newItemsAsync) are from System.Linq.Async, https://www.nuget.org/packages/System.Linq.Async/

            var oldItemsAsync = JsonSerializer.DeserializeAsyncEnumerable<TItem>(inputDecompressor, options) ?? Enumerable.Empty<TItem>().ToAsyncEnumerable();

            var newItemsAsync = newItems.ToAsyncEnumerable();

            var allItemsAsync = oldItemsAsync!.Concat(newItemsAsync)

#if DEBUG

                .Select(i =>

                        {

                            var newLength = output.Length;

                            if (newLength != length)

                                Console.WriteLine(newLength);

                            length = newLength;

​

                            return i;

                        })

#endif

                ;

​

            // https://learn.microsoft.com/en-us/dotnet/core/compatibility/serialization/6.0/iasyncenumerable-serialization

            await JsonSerializer.SerializeAsync(outputBuffer, allItemsAsync, options);

        }

​

        // Compressed serialize (initial creation)

        public static async Task<byte []> SerializeToCompressedByteArrayAsync<TValue>(TValue value, JsonSerializerOptions? options = default)

        {

            using var output = JsonExtensions.MemoryStreamManager.GetStream();

            await JsonExtensions.SerializeToCompressedStreamAsync(output, value, options);

            return output.ToArray();

        }

​

        public static async Task SerializeToCompressedFileAsync<TValue>(string path, TValue value, JsonSerializerOptions? options = default)

        {

            await using var output = File.OpenWrite(path);

            await SerializeToCompressedStreamAsync(output, value, options);

        }

        public static async Task SerializeToCompressedStreamAsync<TValue>(Stream utf8Json, TValue value, JsonSerializerOptions? options = default)

        {

            await using var outputCompressor = new GZipStream(utf8Json, CompressionMode.Compress, leaveOpen : true);

            await using var outputBuffer = new BufferedStream(outputCompressor, BufferSize);

            await JsonSerializer.SerializeAsync<TValue>(outputBuffer, value, options);

        }

    }

​

    public static class FileExtensions

    {

        static readonly UTF8Encoding UTF8 = new UTF8Encoding(false, false);

        public static async Task<string> ReadAllTextCompressedAsync(string path)

        {

            await using var input = File.OpenRead(path);

            await using (var inputDecompressor = new GZipStream(input, CompressionMode.Decompress))

            using (var textReader = new StreamReader(inputDecompressor, Encoding.UTF8, leaveOpen : true))

            {

                return await textReader.ReadToEndAsync();

            }

        }

​

        public static async Task WriteAllTextCompressedAsync(string path, string text, bool bestCompression = false)

        {

            await using var output = File.OpenWrite(path);

            await using (var outputCompressor = new GZipStream(output, CompressionMode.Compress))

            {

                if (bestCompression)

                {

                    outputCompressor.Write(UTF8.GetBytes(text));

                }

                else

                {

                    using (var textWriter = new StreamWriter(outputCompressor, Encoding.UTF8, leaveOpen : true))

                    {

                        await textWriter.WriteAsync(text);

                    }

                }

            }

        }

    }

​

    public record SipTraceRecord(string name, string value);

​

    class TestClass

    {

        public static async Task Test()

        {

            Console.WriteLine("Testing byte streaming");

            await TestByteArrays();

            Console.WriteLine("\nTesting file streaming");

            await TestFiles();

        }

​

        static async Task TestFiles()

        {

            var filePath = "Question79221507.json";

​

            List<SipTraceRecord> initialList = /*Initialize this however you do currently */ Enumerable.Range(0, 10).Select(i => new SipTraceRecord("Name " + i, "Value " + i)).ToList();

​

            var options = new JsonSerializerOptions

            {

                // Add whatever you need here

            };

            await JsonExtensions.SerializeToCompressedFileAsync(filePath, initialList, options);

​

            List<SipTraceRecord> addList = /*Initialize this however you do currently */ new(Enumerable.Range(10, 100000).Select(i => new SipTraceRecord("Added " + i, "Added " + i)));

​

            var tempPath = Path.GetTempFileName();

            await JsonExtensions.CopyAndAddToCompressedFileAsync(filePath, tempPath, addList, options);

            File.Move(tempPath, filePath, true);

​

            var text = await FileExtensions.ReadAllTextCompressedAsync(filePath);

            //Console.WriteLine(text);

            await FileExtensions.WriteAllTextCompressedAsync(tempPath, text, true);

            var fileLength = new System.IO.FileInfo(filePath).Length;

            var fullyCompressedLength = new System.IO.FileInfo(tempPath).Length;

            Console.WriteLine("  Streaming compressed file length: {0}",  fileLength);

            Console.WriteLine("  Non-streaming compressed file length: {0}",  fullyCompressedLength);

​

            // Assert that the streaming file is no more than 1% larger than the file that was compressed all at once.

            Assert.That(fileLength < 1.01 * fullyCompressedLength);

            File.Delete(filePath);

            File.Delete(tempPath);

        }   

​

        static async Task TestByteArrays()

        {

            List<SipTraceRecord> initialList = Enumerable.Range(0, 10).Select(i => new SipTraceRecord("Byte Array Name " + i, "Value " + i)).ToList();

            var options = new JsonSerializerOptions

            {

                // Add whatever you need here

            };

            var initialBytes = await JsonExtensions.SerializeToCompressedByteArrayAsync(initialList, options);

​

            List<SipTraceRecord> list = new(Enumerable.Range(10, 10).Select(i => new SipTraceRecord("Byte Array Name " + i, "Added " + i)));

            var appendedBytes = await JsonExtensions.CopyAndAddToCompressedByteArrayAsync(initialBytes, list, options);

​

            using (var input = new MemoryStream(appendedBytes))

            using (var inputDecompressor = new GZipStream(input, CompressionMode.Decompress))

            using (var textReader = new StreamReader(inputDecompressor, Encoding.UTF8, leaveOpen : true))

            {

                var text = await textReader.ReadToEndAsync();

                Console.WriteLine(text);

            }

        }

    }

​

    public class Program

    {

        public static async Task Main()

        {

            Console.WriteLine("Environment version: {0} ({1}), {2}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription , Environment.Version, Environment.OSVersion);

            Console.WriteLine("System.Text.Json version: " + typeof(JsonSerializer).Assembly.FullName);

            Console.WriteLine();

​

            try

            {

                await TestClass.Test();

            }

            catch (Exception ex)

            {

                Console.WriteLine("Failed with unhandled exception: ");

                Console.WriteLine(ex);

                throw;

            }

        }

    }

​