[Fork] [Fork] [Fork] [Fork] [Fork] https://stackoverflow.com/questions/74027865/read-single-row-and-attributes-from-huge-json-file-via-jsontextreader

VbNet · Console

/Program.vb

' https://dotnetfiddle.net/Fw3myB
Imports System
Imports System.Linq
Imports System.Collections
Imports System.Collections.Generic
Imports System.IO
Imports System.Xml.Linq
Imports System.Xml
Imports System.Text
Imports System.Text.RegularExpressions

Imports Newtonsoft.Json
Imports Newtonsoft.Json.Linq
Imports Newtonsoft.Json.Converters
Imports Newtonsoft.Json.Serialization

Imports NUnit.Framework

	' https://stackoverflow.com/questions/74027865/read-single-row-and-attributes-from-huge-json-file-via-jsontextreader

Public Module JsonExtensions
	Iterator Function DeserializeItemsByPath(Of T)(ByVal stream As Stream, regex as Regex, ByVal Optional settings As JsonSerializerSettings = Nothing) As IEnumerable(Of T)
		Using textReader = New StreamReader(stream, Encoding.UTF8, true, 1024, true)
			For Each item as T In DeserializeItemsByPath(Of T)(textReader, regex, settings)
				Yield item
			Next
        End Using
    End Function

	' Adapted from this answer https://stackoverflow.com/a/56411253/3744182
	' By https://stackoverflow.com/users/14357/spender
	' To https://stackoverflow.com/questions/43747477/how-to-parse-huge-json-file-as-stream-in-json-net
	Iterator Function DeserializeItemsByPath(Of T)(ByVal textReader As TextReader, ByVal regex as Regex, ByVal Optional settings As JsonSerializerSettings = Nothing) As IEnumerable(Of T)
        Dim serializer = JsonSerializer.CreateDefault(settings)
        Using reader = New JsonTextReader(textReader) With { .CloseInput = False }
            While reader.Read()
				If reader.TokenType <> JsonToken.PropertyName And regex.IsMatch(reader.Path)
                    Yield serializer.Deserialize(Of T)(reader)
                End If
            End While
        End Using
    End Function
End Module

Friend Class ListingsIntermediateDTO
	Property publishdate As String
	Property facilities as JToken
End Class

Public Class ListingsDTO
	' kindLabel
	Property publishdate As String
	' .totalSize.value
	Property totalSizeValue as Decimal?
	' .bedrooms.value
	Property bedroomsValue as Decimal?
End Class

Public Class TestClass	
	Shared Friend Sub Test()
		For Each jsonString in GetJson()
			Test1(jsonString)
			Console.WriteLine()
		Next
	End Sub

	Shared Friend Sub Test1(ByVal json As String)
		Dim fileName = "Question74027865.json"
		File.WriteAllText(fileName, json)

		For Each value in GetListings(fileName)
			Console.WriteLine("result: {0}", JsonConvert.SerializeObject(value))
		Next

		File.Delete(FileName)
	End Sub

	Public Shared Iterator Function GetListings(ByVal fileName as String) as IEnumerable(Of ListingsDTO)
		Using stream = File.OpenRead(FileName)
			Dim regex as Regex = new Regex("^(listings\[[0-9]+\]|\[[0-9]+\])$", _
										   RegexOptions.CultureInvariant Or RegexOptions.Compiled Or RegexOptions.Singleline)
			Console.WriteLine(regex)
			For Each value in _
				JsonExtensions.DeserializeItemsByPath(Of ListingsIntermediateDTO)(stream, regex) _
					.Select(Function(dto) new ListingsDTO With { 
						.publishdate = dto.publishdate, 
						.totalSizeValue = dto.facilities.SelectTokens("totalSize.value").Select(Function(t) t.ToObject(Of Decimal?)()).FirstOrDefault(),
						.bedroomsValue = dto.facilities.SelectTokens("bedrooms.value").Select(Function(t) t.ToObject(Of Decimal?)()).FirstOrDefault()
					})
				Yield value
			Next
		End Using
	End Function 
	
	Shared Function GetJson() as IEnumerable(Of String)
		Dim array = { <![CDATA[
{
   "generatedAt":"2022-05-02 02:03:25",
   "listings":[
      {
         "countryid":1,
         "publishdate":"2022-04-02 02:03:25",
         "location":{
            "neighborhood":"Finthen",
            "city":"Mainz",
            "country":"Germany"
         },
         "facilities":{
            "bedrooms":{
               "value":2
            },
            "totalSize":{
               "value":"100"
            }
         }
      },
      {
         "countryid":2,
         "publishdate":"2022-02-02 02:03:24",
         "location":{
            "neighborhood":"Ubers",
            "city":"NYC",
            "country":"USA"
         },
         "facilities":{
            "bedrooms":{
               "value":3
            },
            "totalSize":{
               "value":"150"
            }
         }
      }
   ],
   "count":1077
}
]]>.Value, _
<![CDATA[
[
   {
	  "publishdate":"2022-04-02 02:03:25",
      "countryid":1,
      "price":2997,
      "facilities":{
         "totalSize":{
            "value":80
         }
      }
   },
   {
	  "publishdate":"2022-04-02 04:03:25",
      "countryid":1,
      "price":250,
      "facilities":{
         "totalSize":{
            "value":30
         }
      }
   }
]
]]>.Value
}
		Return array
	End Function
End Class

	Public Module Module1	
		Public Sub Main()
			Console.WriteLine("Environment version: " & Environment.Version.ToString())
			Console.WriteLine("Json.NET version: " & GetType(JsonSerializer).Assembly.FullName)
            Console.WriteLine("")

			Try

				TestClass.Test()

			Catch ex as Exception

				Console.WriteLine("Unhandled exception: ")
				Console.WriteLine(ex)
				Throw

			End Try
			
		End Sub	
	End Module