using System;

using System.Collections;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Runtime.Serialization.Formatters;

using System.IO;

using System.Data;

using System.ComponentModel.DataAnnotations;

using System.Globalization;

using System.Reflection;

​

using Newtonsoft.Json;

using Newtonsoft.Json.Linq;

using Newtonsoft.Json.Converters;

using Newtonsoft.Json.Serialization;    

​

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

​

    //https://stackoverflow.com/questions/54579522/how-to-filter-objects-in-a-json-array-as-they-are-read-from-a-large-file

​

    public static partial class JsonExtensions

    {

        public static IEnumerable<T> DeserializeArrayItems<T>(this JsonSerializer serializer, JsonReader reader)

        {

            if (reader.MoveToContent().TokenType == JsonToken.Null)

                yield break;

            if (reader.TokenType != JsonToken.StartArray)

                throw new JsonSerializationException(string.Format("Current token {0} is not an array at path {1}", reader.TokenType, reader.Path));

            // Process the collection items

            while (reader.Read())

            {

                switch (reader.TokenType)

                {

                    case JsonToken.EndArray:

                        yield break;

​

                    case JsonToken.Comment:

                        break;

​

                    default:

                        yield return serializer.Deserialize<T>(reader);

                        break;

                }

            }

            // Should not come here.

            throw new JsonReaderException(string.Format("Unclosed array at path {0}", reader.Path));

        }

​

        public static JsonReader MoveToContent(this JsonReader reader)

        {

            if (reader.TokenType == JsonToken.None)

                reader.Read();

            while (reader.TokenType == JsonToken.Comment && reader.Read())

                ;

            return reader;

        }

    }

​

    public class PurpleAirData

    {

        public PurpleAirData(DateTime createdAt, double airQuality)

        {

            this.CreatedAt = createdAt;

            this.AirQuality = airQuality;

        }

        // Required properties

        public DateTime CreatedAt { get; set; }

        public double AirQuality { get; set; }

​

        // Optional properties, thus nullable

        public double? Temperature { get; set; }

        public double? Humidity { get; set; }

    }

​

    public class RootObject

    {

        public Channel channel { get; set; } // Define this using http://json2csharp.com/

        public List<PurpleAirData> feeds { get; set; }

    }

​

    class PurpleAirListConverter : JsonConverter

    {

        class PurpleAirDataDTO

        {

            // Required properties

            [JsonProperty("created_at")]

            public DateTime? CreatedAt { get; set; }

            [JsonProperty("Field8")]

            public double? AirQuality { get; set; }

​

            // Optional properties

            [JsonProperty("Field6")]

            public double? Temperature { get; set; }

            [JsonProperty("Field7")]

            public double? Humidity { get; set; }

        }

​

        public override bool CanConvert(Type objectType)

        {

            return objectType == typeof(List<PurpleAirData>);

        }

​

        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)

        {

            if (reader.MoveToContent().TokenType == JsonToken.Null)

                return null;

            var list = existingValue as List<PurpleAirData> ?? new List<PurpleAirData>();

​

            var query = from dto in serializer.DeserializeArrayItems<PurpleAirDataDTO>(reader)

                        where dto != null && dto.CreatedAt != null && dto.AirQuality != null

                        select new PurpleAirData(dto.CreatedAt.Value, dto.AirQuality.Value) { Humidity = dto.Humidity, Temperature = dto.Temperature };

​

            list.AddRange(query);

​

            return list;

        }

​

        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)

        {

            throw new NotImplementedException();

        }

    }

​

    class TestClass

    {

        public static void Test()

        {

            RootObject root;

            using (var textReader = new StringReader(GetJson()))

            {

                root = DeserializePurpleAirDataFile(textReader);

            }

​

            var newJson = JsonConvert.SerializeObject(root, Formatting.Indented);

            Console.WriteLine("Deserialised purple air data: ");

            Console.WriteLine(newJson);

            Assert.IsTrue(root.feeds.Count == JToken.Parse(GetJson()).SelectTokens("feeds[?(@.created_at && @.field8)]").Count());

        }

​

        static RootObject DeserializePurpleAirDataFile(TextReader textReader)

        {

            var settings = new JsonSerializerSettings

            {

                Converters = { new PurpleAirListConverter() },

                NullValueHandling = NullValueHandling.Ignore,

            };

            var serializer = JsonSerializer.CreateDefault(settings);

            using (var reader = new JsonTextReader(textReader) { CloseInput = false })

            {

                return serializer.Deserialize<RootObject>(reader);

            }

        }

​

        static string GetJson()

        {

            var json = @"{

""channel"": {

    ""id"": 622370,

    ""name"": ""AirMonitor_e81a"",

    ""latitude"": ""0.0"",

    ""longitude"": ""0.0"",

    ""field1"": ""PM1.0 (ATM)"",

    ""field2"": ""PM2.5 (ATM)"",

    ""field3"": ""PM10.0 (ATM)"",

    ""field4"": ""Uptime"",

    ""field5"": ""RSSI"",

    ""field6"": ""Temperature"",

    ""field7"": ""Humidity"",

    ""field8"": ""PM2.5 (CF=1)"",

    ""created_at"": ""2018-11-09T00:35:34Z"",

    ""updated_at"": ""2018-11-09T00:35:35Z"",

    ""last_entry_id"": 65435

},

""feeds"": [

    {

        ""created_at"": ""2019-01-10T23:56:09Z"",

        ""entry_id"": 56401,

        ""field1"": ""1.00"",

        ""field2"": ""1.80"",

        ""field3"": ""1.80"",

        ""field4"": ""369"",

        ""field5"": ""-30"",

        ""field6"": ""66.00"",

        ""field7"": ""59.00"",

        ""field8"": ""1.80""

    },

    {

        ""created_at"": ""2019-01-10T23:57:29Z"",

        ""entry_id"": 56402,

        ""field1"": ""1.08"",

        ""field2"": ""2.44"",

        ""field3"": ""3.33"",

        ""field4"": ""371"",

        ""field5"": ""-32"",

        ""field6"": ""66.00"",

        ""field7"": ""59.00"",

        ""field8"": ""2.44""

    },

    {

        //""created_at"": ""2019-01-10T23:57:29Z"",

        ""entry_id"": 56402,

        ""field1"": ""1.08"",

        ""field2"": ""2.44"",

        ""field3"": ""3.33"",

        ""field4"": ""371"",

        ""field5"": ""-32"",

        ""field6"": ""66.00"",

        ""field7"": ""59.00"",

        ""field8"": ""2.44""

    },

    {

        ""created_at"": ""2019-01-10T23:57:29Z"",

        ""entry_id"": 56402,

        ""field1"": ""1.08"",

        ""field2"": ""2.44"",

        ""field3"": ""3.33"",

        ""field4"": ""371"",

        ""field5"": ""-32"",

        ""field6"": ""66.00"",

        ""field7"": ""59.00"",

        //""field8"": ""2.44""

    },

   {

        ""created_at"": ""2019-01-26T00:14:04Z"",

        ""entry_id"": 64400,

        ""field1"": ""0.27"",

        ""field2"": ""0.95"",

        ""field3"": ""1.25"",

        ""field4"": ""213"",

        ""field5"": ""-27"",

        ""field6"": ""72.00"",

        ""field7"": ""40.00"",

        ""field8"": ""0.95""

    }

]}";

            return json;

        }

    }

​

    public class Channel

    {

        public int id { get; set; }

        public string name { get; set; }

        public string latitude { get; set; }

        public string longitude { get; set; }

        public string field1 { get; set; }

        public string field2 { get; set; }

        public string field3 { get; set; }

        public string field4 { get; set; }

        public string field5 { get; set; }

        public string field6 { get; set; }

        public string field7 { get; set; }

        public string field8 { get; set; }

        public DateTime created_at { get; set; }

        public DateTime updated_at { get; set; }

        public int last_entry_id { get; set; }

    }

​

    public class Program

    {

        public static void Main()

        {

            Console.WriteLine("Environment version: " + Environment.Version);

            Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);

            Console.WriteLine();

​

            try

            {

                TestClass.Test();

            }

            catch (Exception ex)

            {

                Console.WriteLine("Failed with unhandled exception: ");

                Console.WriteLine(ex);

                throw;

            }

        }

    }

​