using System;

using System.Collections.Generic;

using System.Threading.Tasks;

using System.Collections.Concurrent;

using System.Collections.Generic;

using System.Linq;

using System.Diagnostics;

​

public class Program

{

    public static async Task Main()

    {

        FillList();

        var stopWatch1 = Stopwatch.StartNew();

        var results1 = await QueryApiAsParallel();

        stopWatch1.Stop();

        var stopWatch2 = Stopwatch.StartNew();

        var results2 = await QueryApiAsParallelOptimized();

        stopWatch2.Stop();

        Console.WriteLine($"Query1: {stopWatch1.ElapsedMilliseconds} ms ({results1.Select(s => int.Parse(s)).Sum()})");

        Console.WriteLine(string.Join(',', results1));

        Console.WriteLine("");

        Console.WriteLine($"Query2: {stopWatch2.ElapsedMilliseconds} ms ({results2.Select(s => int.Parse(s)).Sum()})");

        Console.WriteLine(string.Join(',', results2));

    }

    public static async Task<string[]> QueryApiAsParallelOptimized() {

        int maximum = 10000; 

        int rangeSize = 1000;

​

        var results = new ConcurrentBag<string[]>();

        var partitions = Enumerable.Range(0, maximum);

        var chunks = partitions.Chunk(rangeSize);

        var parallelOptions = new ParallelOptions()

        {

            MaxDegreeOfParallelism = 1000

        };

        await Parallel.ForEachAsync(chunks, parallelOptions, async (range, token) =>

        {

            int skip = range.First();

            string[] names = await CallApiAsync(skip, rangeSize);

            results.Add(names);

        });

        string[] flattened = results.SelectMany(x => x).ToArray();

​

        return flattened;

    }

    public static async Task<string[]> QueryApiAsParallel() {

        int maximum = 10000; // I don't want to query more than 10000 results even I know that are a lot more results

        int rangeSize = 1000; // maximum number that can be received via API

​

        Task<string[]>[] tasks = Partitioner.Create(0, maximum, rangeSize).AsParallel().Select(async (range, index) => 

        {

            int skip = range.Item1;

            string[] names = await CallApiAsync(skip, rangeSize);

​

            return names;

        }).ToArray();

​

        string[][] tasksCompleted = await Task.WhenAll(tasks);

​

        string[] flattened = tasksCompleted.SelectMany(x => x).ToArray();

​

        return flattened;

    }

    private static async Task<string[]> CallApiAsync(int skip, int take) 

    {

        await Task.Delay(100);

        return _list.Skip(skip).Take(take).ToArray();

    }

    private static List<string> _list = new List<string>();

    private static Random _rnd = new Random();

    private static void FillList(){

        for(var i = 0; i< 100000; i++){

            _list.Add($"{_rnd.Next(100,200)}");

        }

    }

}