using System;

using System.Collections.Generic;

using System.Linq;

​

namespace MedianMADExample

{

    // 1. Schema for your data

    public class WTJobTypeLabel

    {

        public string JobTypeLabel { get; set; }

        public Guid MasterID { get; set; }

        public int Count { get; set; }

    }

​

    class Program

    {

        public class JobTypeLabelType

    {

        public string JobTypeLabel { get; set; }

        public bool NonWater { get; set; }

    }

        public static readonly List<JobTypeLabelType> JobTypeLabelList = new List<JobTypeLabelType>

    {

        { new JobTypeLabelType { JobTypeLabel = "Water Mitigation", NonWater = false } },

        { new JobTypeLabelType { JobTypeLabel = "Rebuild", NonWater = true } },

        { new JobTypeLabelType { JobTypeLabel = "Mold", NonWater = true } },

        { new JobTypeLabelType { JobTypeLabel = "Asbestos", NonWater = true } },

        { new JobTypeLabelType { JobTypeLabel = "Abatement", NonWater = true } },

        { new JobTypeLabelType { JobTypeLabel = "General", NonWater = true } }

    };

        static void Main(string[] args)

        {

            // 2. Hard-coded example data (20 records)

            //    We’ll use two MasterIDs for variety.

            Guid masterId1 = Guid.Parse("11111111-1111-1111-1111-111111111111");

            Guid masterId2 = Guid.Parse("22222222-2222-2222-2222-222222222222");

            Guid masterId3 = Guid.Parse("22222222-2222-2222-2222-222222222223");

​

            // Feel free to adjust these counts to see how the outlier logic behaves.

            var data = new List<WTJobTypeLabel>

            {

                new WTJobTypeLabel { JobTypeLabel = "LabelA", MasterID = masterId1, Count = 2 },

                new WTJobTypeLabel { JobTypeLabel = "LabelB", MasterID = masterId1, Count = 5 },

                new WTJobTypeLabel { JobTypeLabel = "LabelC", MasterID = masterId1, Count = 12 },

                new WTJobTypeLabel { JobTypeLabel = "LabelD", MasterID = masterId1, Count = 0 },

                new WTJobTypeLabel { JobTypeLabel = "LabelE", MasterID = masterId1, Count = 100 },

                new WTJobTypeLabel { JobTypeLabel = "LabelF", MasterID = masterId1, Count = 1 },

                new WTJobTypeLabel { JobTypeLabel = "LabelG", MasterID = masterId1, Count = 250 },

                new WTJobTypeLabel { JobTypeLabel = "LabelH", MasterID = masterId1, Count = 9 },

                new WTJobTypeLabel { JobTypeLabel = "LabelI", MasterID = masterId1, Count = 300 },

                new WTJobTypeLabel { JobTypeLabel = "LabelJ", MasterID = masterId1, Count = 50 },

​

                new WTJobTypeLabel { JobTypeLabel = "LabelK", MasterID = masterId2, Count = 7 },

                new WTJobTypeLabel { JobTypeLabel = "LabelL", MasterID = masterId2, Count = 8 },

                new WTJobTypeLabel { JobTypeLabel = "LabelM", MasterID = masterId2, Count = 24 },

                new WTJobTypeLabel { JobTypeLabel = "LabelN", MasterID = masterId2, Count = 4 },

                new WTJobTypeLabel { JobTypeLabel = "LabelO", MasterID = masterId2, Count = 20 },

                new WTJobTypeLabel { JobTypeLabel = "LabelP", MasterID = masterId2, Count = 10 },

                new WTJobTypeLabel { JobTypeLabel = "LabelQ", MasterID = masterId2, Count = 2 },

                new WTJobTypeLabel { JobTypeLabel = "LabelR", MasterID = masterId2, Count = 4 },

                new WTJobTypeLabel { JobTypeLabel = "LabelS", MasterID = masterId2, Count = 1 },

                new WTJobTypeLabel { JobTypeLabel = "LabelT", MasterID = masterId2, Count = 4 },

                new WTJobTypeLabel { JobTypeLabel = "LabelK", MasterID = masterId3, Count = 1 },

                new WTJobTypeLabel { JobTypeLabel = "LabelL", MasterID = masterId3, Count = 2 },

                new WTJobTypeLabel { JobTypeLabel = "LabelM", MasterID = masterId3, Count = 4 },

                new WTJobTypeLabel { JobTypeLabel = "LabelN", MasterID = masterId3, Count = 3 },

                new WTJobTypeLabel { JobTypeLabel = "LabelO", MasterID = masterId3, Count = 5 },

                new WTJobTypeLabel { JobTypeLabel = "LabelP", MasterID = masterId3, Count = 7 },

                new WTJobTypeLabel { JobTypeLabel = "LabelQ", MasterID = masterId3, Count = 6 },

                new WTJobTypeLabel { JobTypeLabel = "LabelR", MasterID = masterId3, Count = 100 },

                new WTJobTypeLabel { JobTypeLabel = "LabelS", MasterID = masterId3, Count = 1000 },

                new WTJobTypeLabel { JobTypeLabel = "LabelS", MasterID = masterId3, Count = 900 },

            };

​

            // 3. Call the filtering method for masterId1

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #1 ===");

            List<WTJobTypeLabel> filteredForMaster1 = GetSmartFilteredJobTypeLabelsWithMAD(data, masterId1);

            PrintResults(filteredForMaster1);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #1 ===");

            List<WTJobTypeLabel> filteredForMaster2 = GetSmartFilteredJobTypeLabels2(data, masterId1);

            PrintResults(filteredForMaster2);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #1 ===");

            List<WTJobTypeLabel> filteredForMaster22 = GetSmartFilteredJobTypeLabels3(data, masterId1);

            PrintResults(filteredForMaster22);

​

            Console.WriteLine();

​

            // 4. Call the filtering method for masterId2

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #2 ===");

            List<WTJobTypeLabel> filteredForMaster3 = GetSmartFilteredJobTypeLabelsWithMAD(data, masterId2);

            PrintResults(filteredForMaster3);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #2 ===");

            List<WTJobTypeLabel> filteredForMaster4 = GetSmartFilteredJobTypeLabels2(data, masterId2);

            PrintResults(filteredForMaster4);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #2 ===");

            List<WTJobTypeLabel> filteredForMaster44 = GetSmartFilteredJobTypeLabels3(data, masterId2);

            PrintResults(filteredForMaster44);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #3 ===");

            List<WTJobTypeLabel> filteredForMaster333 = GetSmartFilteredJobTypeLabels3(data, masterId3);

            PrintResults(filteredForMaster333);

            Console.WriteLine("=== Filtered & Sorted Results for MasterID #3 ===");

            List<WTJobTypeLabel> filteredForMaster3333 = GetSmartFilteredJobTypeLabelsWithIQR(data, masterId3);

            PrintResults(filteredForMaster3333);

​

        }

​

        /// <summary>

        /// Filters the data to a given MasterID, removes only "low outliers" using Median + MAD,

        /// and returns the Top 10 by Count (descending).

        /// </summary>

        public static List<WTJobTypeLabel> GetSmartFilteredJobTypeLabelsWithMAD(

            List<WTJobTypeLabel> allData,

            Guid masterId)

        {

            // 1. Filter and sort descending by Count

            var rawResults = allData

                .Where(label => label.MasterID == masterId)

                .OrderByDescending(label => label.Count)

                .ToList();

​

            // If 0 or 1 records, just return top 10 as is.

            if (rawResults.Count <= 1)

            {

                return rawResults.Take(10).ToList();

            }

​

            // 2. Extract the counts as a sorted list of doubles

            var sortedCounts = rawResults

                .Select(r => (double)r.Count)

                .OrderBy(c => c)

                .ToList();

​

            // 3. Calculate the median

            double median = GetMedian(sortedCounts);

​

            // 4. Calculate the absolute deviations from the median

            var absoluteDeviations = sortedCounts

                .Select(c => Math.Abs(c - median))

                .ToList();

​

            // 5. Calculate the median of those absolute deviations (MAD)

            double mad = GetMedian(absoluteDeviations);

​

            // 6. Scale MAD by 1.4826 (makes it roughly comparable to std. dev)

            double scaledMAD = mad * 1.4826;

            Console.WriteLine(median);

            Console.WriteLine(absoluteDeviations);

            Console.WriteLine(mad);

            Console.WriteLine(scaledMAD);

​

            // 7. Define the lower cutoff: median - (3 * scaledMAD)

            //    Only *low* outliers are removed, so no upper cutoff is used here.

            double lowerCutoff = median - 2.0 * scaledMAD;

​

            // 8. Filter out those below the cutoff

            var filteredResults = rawResults

                .Where(r => r.Count >= lowerCutoff)

                .ToList();

​

            // 9. Return top 10 after filtering

            return filteredResults.Take(10).ToList();

        }

        public static List<WTJobTypeLabel> GetSmartFilteredJobTypeLabels2(

            List<WTJobTypeLabel> allData,

            Guid masterId)

        {

    // 1. Filter and sort descending by Count

            var rawResults = allData

                .Where(label => label.MasterID == masterId)

                .OrderByDescending(label => label.Count)

                .ToList();

​

    // If there are no records or just one, outlier detection doesn’t make much sense

    if (rawResults.Count <= 1)

    {

        // Return all (or handle however you want)

        return rawResults.Take(10).ToList();

    }

​

    // 2. Compute mean of the counts

    var counts = rawResults.Select(r => (double)r.Count).ToArray();

    double mean = counts.Average();

​

    // 3. Compute standard deviation

    double sumOfSquares = counts.Sum(v => Math.Pow(v - mean, 2));

    double variance = sumOfSquares / (counts.Length - 1);

    double stdDev = Math.Sqrt(variance);

​

    // 4. Define a cutoff for “low outliers”: mean - 2 * stdDev

    //    (Adjust the multiplier to your needs: 1.5, 2, 2.5, 3, etc.)

    double lowerCutoff = mean - 1.5 * stdDev;

​

    // 5. Filter out low outliers

    var filteredResults = rawResults

        .Where(r => r.Count >= lowerCutoff)

        .ToList();

​

    // 6. Return the top 10

    return filteredResults.Take(10).ToList();

}

​

​

        /// <summary>

        /// Gets the median of a *sorted* list of double values.

        /// </summary>

        private static double GetMedian(List<double> sortedValues)

        {

            int count = sortedValues.Count;

            if (count == 0) return 0.0;

​

            if (count % 2 == 0)

            {

                // Even number of elements: average the two middle ones

                return (sortedValues[count / 2 - 1] + sortedValues[count / 2]) / 2.0;

            }

            else

            {

                // Odd number of elements: return the middle

                return sortedValues[count / 2];

            }

        }

​

        /// <summary>

        /// Helper method to print the results to the console.

        /// </summary>

        private static void PrintResults(List<WTJobTypeLabel> results)

        {

            foreach (var item in results)

            {

                Console.WriteLine(

                    $"Label: {item.JobTypeLabel}, MasterID: {item.MasterID}, Count: {item.Count}");

            }

        }

        public static List<WTJobTypeLabel> GetSmartFilteredJobTypeLabels3(

            List<WTJobTypeLabel> allData,

            Guid masterId)

        {

        // 1) Query data and sort descending by count

        var rawResults = allData

                .Where(label => label.MasterID == masterId)

                .OrderByDescending(label => label.Count)

                .ToList();

​

        // If 0 or 1 records, no real filtering needed

        if (rawResults.Count <= 1) 

            return rawResults;

​

        // 2) Identify "high outliers" via log-based IQR

        //    We'll get indices of all items considered "high outliers".

        //    For IQR, we need ascending-sorted counts, so let's prep that:

        var ascendingCounts = rawResults

            .Select(r => (double) r.Count)

            .OrderBy(c => c)

            .ToList();

​

        var highOutlierThreshold = GetHighOutlierThresholdLogIQR(ascendingCounts, 1.5); 

        // '1.5' is the "k" typically used; adjust if needed.

​

        // Now figure out which actual items (in descending order) are "high outliers".

        // We compare each .Count to that threshold:

        var highOutlierIndices = new HashSet<int>();

        for (int i = 0; i < rawResults.Count; i++)

        {

            if (rawResults[i].Count > highOutlierThreshold)

                highOutlierIndices.Add(i);

        }

​

        // 3) For ratio-based cutoff, we skip high outliers to find ratioMax.

        //    That is, we want the largest "non-outlier" count to define the ratio basis.

        //    If everything is an outlier except the smallest ones, we'll fallback to

        //    the top non-outlier or at least the second item, etc.

        //    (In practice, you can decide the fallback logic you prefer.)

        int firstNonOutlierIndex = Enumerable.Range(0, rawResults.Count)

            .FirstOrDefault(idx => !highOutlierIndices.Contains(idx));

        // If *all* items are high outliers, we just fallback to the top item. 

        // Or you could fallback to the median or second item. This is your choice.

        long ratioMax = 1; // fallback

        if (firstNonOutlierIndex < rawResults.Count)

        {

            ratioMax = rawResults[firstNonOutlierIndex].Count;

        }

        else

        {

            // Everything is an outlier, fallback to top item

            ratioMax = rawResults[0].Count;

        }

​

        // 4) Apply ratio-based cutoff to remove low outliers

        //    Example: keep items >= 10% of ratioMax

        double fraction = 0.10; 

        double cutoff = fraction * ratioMax;

​

        var filtered = rawResults

            .Where((item, index) => 

                // Keep if it's a high outlier (we always keep big guys) 

                // OR if it's above the ratio cutoff

                highOutlierIndices.Contains(index) || (item.Count >= cutoff)

            )

            .ToList();

​

        // 5) Return top 10

        return filtered.Take(10).ToList();

    }

​

    // --------------- HELPER METHOD ---------------

    /// <summary>

    /// Returns the "high outlier" threshold using a log-based IQR approach.

    /// 

    /// Steps:

    /// 1. Transform each value to log(value+1).

    /// 2. Compute Q1, Q3 in log space.

    /// 3. Define upper fence = Q3 + k*(Q3 - Q1).

    /// 4. Exponentiate that fence back to original space => outlierThreshold.

    ///    i.e. outlierThreshold = exp(upperFence) - 1

    ///

    /// Values ABOVE outlierThreshold are considered "high outliers".

    ///

    /// This function returns that numeric threshold. 

    /// </summary>

    /// <param name="sortedValues">Must be ascending sorted raw values.</param>

    /// <param name="kIqrMultiplier">1.5 is a typical default. Increase to be more lenient.</param>

    private static double GetHighOutlierThresholdLogIQR(List<double> sortedValues, double kIqrMultiplier)

    {

        if (sortedValues == null || sortedValues.Count == 0)

            return double.MaxValue; // trivially no threshold

​

        // 1) log-transform 

        var logValues = sortedValues.Select(v => Math.Log(v + 1.0)).ToList();

        logValues.Sort(); // ensure ascending in log space

​

        // 2) Compute Q1 and Q3 in log space

        double q1 = GetPercentile(logValues, 0.25);

        double q3 = GetPercentile(logValues, 0.75);

        double iqr = q3 - q1;

​

        // 3) Upper fence in log space

        double upperFenceLog = q3 + kIqrMultiplier * iqr;

​

        // 4) Transform back (exponentiate), subtract 1

        //    So anything > (exp(upperFenceLog) - 1) is considered a high outlier

        double threshold = Math.Exp(upperFenceLog) - 1.0;

​

        return threshold;

    }

​

    // --------------- ANOTHER HELPER ---------------

    /// <summary>

    /// Returns the p-th percentile of an ascending-sorted list.

    /// E.g. p=0.25 => Q1, p=0.75 => Q3.

    /// Uses a simple interpolation approach.

    /// </summary>

    private static double GetPercentile(List<double> sortedValues, double p)

    {

        if (sortedValues == null || sortedValues.Count == 0) return 0;

        if (p <= 0) return sortedValues.First();

        if (p >= 1) return sortedValues.Last();

​

        double n = sortedValues.Count;

        double pos = (n - 1) * p;

        int i = (int) pos;

        double frac = pos - i;

​

        if (i + 1 < sortedValues.Count)

            return sortedValues[i] * (1 - frac) + sortedValues[i + 1] * frac;

        else

            return sortedValues[i];

    }

        public static List<WTJobTypeLabel> GetSmartFilteredJobTypeLabelsWithIQR(

            List<WTJobTypeLabel> allData,

            Guid masterId)

        {

        // 1) Query data and sort descending by count

        var rawResults = allData

                .Where(label => label.MasterID == masterId)

                .OrderByDescending(label => label.Count)

                .ToList();

​

        // If 0 or 1 records, no real filtering needed

        if (rawResults.Count <= 1)

            return rawResults.Take(10).ToList();

​

        // 2) Identify "low outliers" using IQR without log

        var sortedCountsAsc = rawResults

            .Select(r => (double)r.Count)

            .OrderBy(c => c)

            .ToList();

​

        var iqr = GetIQR(sortedCountsAsc, out double q1, out double q3);

        double lowerFence = q1 - 1.5 * iqr; // Adjust multiplier as needed

​

        // Any count below lowerFence is a low outlier

        var lowOutlierCounts = sortedCountsAsc

            .Where(c => c < lowerFence)

            .ToList();

​

        // 3) Optionally identify high outliers (if needed)

        // For your specific case, you want to keep high outliers but not use them in ratio

        double upperFence = q3 + 1.5 * iqr;

        var highOutlierThreshold = upperFence;

        var highOutlierCounts = sortedCountsAsc

            .Where(c => c > upperFence)

            .ToList();

​

        // 4) Determine ratioMax by excluding high outliers

        // Find the maximum non-high outlier count

        var nonHighOutlierCounts = sortedCountsAsc

            .Where(c => c <= upperFence)

            .ToList();

​

        double ratioMax = nonHighOutlierCounts.Any() ? nonHighOutlierCounts.Max() : rawResults.Max(r => r.Count);

​

        // Define the fraction for cutoff (e.g., 10%)

        double fraction = 0.10;

        double cutoff = fraction * ratioMax;

​

        // 5) Filter the raw results

        var filtered = rawResults

            .Where(r => 

                // Keep if it's a high outlier (always keep) 

                // OR if it's >= cutoff

                r.Count > upperFence || r.Count >= cutoff

            )

            .Take(10)

            .ToList();

​

        return filtered;

    }

​

    // --------------- HELPER METHODS ---------------

    /// <summary>

    /// Computes the Interquartile Range (IQR), Q1, and Q3.

    /// Assumes sortedValues is sorted in ascending order.

    /// </summary>

    private static double GetIQR(List<double> sortedValues, out double q1, out double q3)

    {

        q1 = GetPercentile(sortedValues, 0.25);

        q3 = GetPercentile(sortedValues, 0.75);

        return q3 - q1;

    }

    }

}

​