CSharp · Console
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.ComponentModel.DataAnnotations;
using System.Threading.Tasks;
using System.Threading;
using System.Runtime.CompilerServices;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
using NUnit.Framework; // Throws NUnit.Framework.AssertionException
#nullable enable
//https://stackoverflow.com/questions/75159862/performance-issue-with-deserlizing-large-json-file-into-objects-and-add-them-to
public static partial class JsonExtensions
{
/// <summary>
/// Asynchronously load and synchronously deserialize values from a stream containing a JSON array. The root object of the JSON stream must in fact be an array, or an exception is thrown
/// </summary>
public static async IAsyncEnumerable<T?> DeserializeAsyncEnumerable<T>(Stream stream, JsonSerializerSettings? settings = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var serializer = JsonSerializer.CreateDefault(settings);
var loadSettings = new JsonLoadSettings { LineInfoHandling = LineInfoHandling.Ignore }; // For performance do not load line info.
// StreamReader and JsonTextReader do not implement IAsyncDisposable so let the caller dispose the stream.
using (var textReader = new StreamReader(stream, leaveOpen : true))
using (var reader = new JsonTextReader(textReader) { CloseInput = false })
{
await foreach (var token in LoadAsyncEnumerable(reader, loadSettings, cancellationToken ).ConfigureAwait(false))
yield return token.ToObject<T>(serializer);
}
}
/// <summary>
/// Asynchronously load and return JToken values from a stream containing a JSON array. The root object of the JSON stream must in fact be an array, or an exception is thrown
/// </summary>
public static async IAsyncEnumerable<JToken> LoadJTokenAsyncEnumerable(Stream stream, JsonLoadSettings? settings = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
// StreamReader and JsonTextReader do not implement IAsyncDisposable so let the caller dispose the stream.
using (var textReader = new StreamReader(stream, leaveOpen : true))
using (var reader = new JsonTextReader(textReader) { CloseInput = false })
{
await foreach (var token in LoadAsyncEnumerable(reader, settings, cancellationToken).ConfigureAwait(false))
yield return token;
}
}
/// <summary>
/// Asynchronously load and return JToken values from a stream containing a JSON array. The root object of the JSON stream must in fact be an array, or an exception is thrown
/// </summary>
public static async IAsyncEnumerable<JToken> LoadAsyncEnumerable(JsonTextReader reader, JsonLoadSettings? settings = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
(await reader.MoveToContentAndAssertAsync().ConfigureAwait(false)).AssertTokenType(JsonToken.StartArray);
cancellationToken.ThrowIfCancellationRequested();
while ((await reader.ReadToContentAndAssert(cancellationToken).ConfigureAwait(false)).TokenType != JsonToken.EndArray)
{
cancellationToken.ThrowIfCancellationRequested();
yield return await JToken.LoadAsync(reader, settings, cancellationToken).ConfigureAwait(false);
}
cancellationToken.ThrowIfCancellationRequested();
}
public static JsonReader AssertTokenType(this JsonReader reader, JsonToken tokenType) =>
reader.TokenType == tokenType ? reader : throw new JsonSerializationException(string.Format("Unexpected token {0}, expected {1}", reader.TokenType, tokenType));
public static async Task<JsonReader> ReadToContentAndAssert(this JsonReader reader, CancellationToken cancellationToken = default) =>
await (await reader.ReadAndAssertAsync(cancellationToken).ConfigureAwait(false)).MoveToContentAndAssertAsync(cancellationToken).ConfigureAwait(false);
public static async Task<JsonReader> MoveToContentAndAssertAsync(this JsonReader reader, CancellationToken cancellationToken = default)
{
if (reader == null)
throw new ArgumentNullException();
if (reader.TokenType == JsonToken.None) // Skip past beginning of stream.
await reader.ReadAndAssertAsync(cancellationToken).ConfigureAwait(false);
while (reader.TokenType == JsonToken.Comment) // Skip past comments.
await reader.ReadAndAssertAsync(cancellationToken).ConfigureAwait(false);
return reader;
}
public static async Task<JsonReader> ReadAndAssertAsync(this JsonReader reader, CancellationToken cancellationToken = default)
{
if (reader == null)
throw new ArgumentNullException();
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
throw new JsonReaderException("Unexpected end of JSON stream.");
return reader;
}
}
public class Country
{
[Key]
//[DatabaseGenerated(DatabaseGeneratedOption.None)]
public int Id { get; set; }
public string Name { get; set; }
public string ISO2 { get; set; }
public string ISO3 { get; set; }
public ICollection<City> Cities { get; set; }
}
public class City
{
[Key]
//[DatabaseGenerated(DatabaseGeneratedOption.None)]
public int Id { get; set; }
public string Name { get; set; }
public int CountryId { get; set; }
public Country Country { get; set; }
}
class TestClass
{
public static async Task Test()
{
var fileName = "Question75159862.json";
await File.WriteAllTextAsync(fileName, GetJson());
var countries = new List<Country>();
await using var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true);
await foreach (var country in JsonExtensions.DeserializeAsyncEnumerable<Country>(stream))
{
if (country == null)
continue;
if (country.Cities != null)
foreach (var city in country.Cities)
{
// Do you need another instance of City? If so you may clone the current one as follows:
var clone = JToken.FromObject(city).ToObject<City>();
city.CountryId = country.Id;
//context.Cities.Add(cities);
//context.Entry(cities).State = Microsoft.EntityFrameworkCore.EntityState.Detached;
}
countries.Add(country);
}
Console.WriteLine("Deserialized and re-serialized {0}:", countries);
Console.WriteLine(JsonConvert.SerializeObject(countries, Formatting.Indented));
}
static string GetJson() =>
"""
[
{
"id":1,
"name":"Afghanistan",
"iso3":"AFG",
"iso2":"AF",
"cities":[
{
"id":141,
"name":"‘Alāqahdārī Dīshū"
},
{
"id":53,
"name":"Aībak"
},
{
"id":50,
"name":"Andkhoy"
},
{
"id":136,
"name":"Āqchah"
},
{
"id":137,
"name":"Ārt Khwājah"
},
{
"id":51,
"name":"Asadabad"
},
{
"id":52,
"name":"Ashkāsham"
},
{
"id":138,
"name":"Āsmār"
},
{
"id":54,
"name":"Baghlān"
},
{
"id":55,
"name":"Balkh"
}
]
}
]
""";
}
public class Program
{
public static async Task Main(string[] args)
{
Console.WriteLine("Environment version: {0} ({1}), {2}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription , Environment.Version, Environment.OSVersion);
Console.WriteLine("{0} version: {1}", typeof(JsonSerializer).Namespace, typeof(JsonSerializer).Assembly.FullName);
Console.WriteLine();
try
{
await TestClass.Test();
}
catch (Exception ex)
{
Console.WriteLine("Failed with unhandled exception: ");
Console.WriteLine(ex);
throw;
}
}
}