AutoML.NET Culture Bug 2020-07-25

CSharp · Console

/Program.cs

using System;
using System.Globalization;
using Microsoft.ML;
using Microsoft.ML.AutoML;
using Microsoft.ML.Data;
using System.Linq;

public class Program
{
	const string TrainDataPath = "train.csv";
	const string ValidationDataPath = "valid.csv";
	const string TestDataPath = "test.csv";
	
	public static void Main()
	{
		WriteDataset(TrainDataPath, 10, true);
		WriteDataset(ValidationDataPath, 30, false);
		WriteDataset(TestDataPath, 200, false);
		
		CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("ru-RU"); // Fails near iteration 44
		//CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); // Works
		RunAutoML();
	}
	
	private static void WriteDataset(string filename, int rows, bool header)
	{
		
		using (System.IO.StreamWriter file = new System.IO.StreamWriter(filename))
        {
            var rnd = new Random();
			var culture = CultureInfo.InvariantCulture; // Make sure we're writing a dataset with floats as "0.1234"
			if (header) 
				file.WriteLine($"Label,Feat1,Feat2,Feat3");
			for (int i = 0; i < rows; i++)
            {
				var label = rnd.NextDouble();
				file.WriteLine($"{label.ToString("F4", culture)},{(label + rnd.NextDouble()).ToString("F4", culture)},{(label * rnd.NextDouble()).ToString("F4", culture)},{(label * rnd.NextDouble()).ToString("F4", culture)}");
            }
        }
			
		//Console.WriteLine($"Contents of {filename}:\n{System.IO.File.ReadAllText(filename)}");
	}
	
	private static void RunAutoML()
	{
		var mlContext = new MLContext();

		// Infer columns in the dataset with AutoML.
		ColumnInferenceResults columnInference = mlContext.Auto().InferColumns(TrainDataPath, "Label", groupColumns: false);
		ColumnInformation columnInformation = columnInference.ColumnInformation;
			
		// Load data from files using inferred columns.
		TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderOptions);
        IDataView TrainDataView = textLoader.Load(TrainDataPath);
        IDataView ValidationDataView = textLoader.Load(ValidationDataPath);
        IDataView TestDataView = textLoader.Load(TestDataPath);

		// Run an AutoML experiment on the dataset.
		var progressHandler = new RegressionExperimentProgressHandler();
		var experimentSettings = new RegressionExperimentSettings();
        experimentSettings.MaxExperimentTimeInSeconds = 9;
		experimentSettings.Trainers.Clear();
		experimentSettings.Trainers.Add(RegressionTrainer.LightGbm); // LightGBM fails on .NET Fiddle
		experimentSettings.Trainers.Add(RegressionTrainer.LbfgsPoissonRegression);
		experimentSettings.Trainers.Add(RegressionTrainer.FastTree);
		
		var experiment = mlContext.Auto().CreateRegressionExperiment(experimentSettings);
        
		Console.WriteLine($"\nRunning AutoML regression experiment...");
		ExperimentResult<RegressionMetrics> experimentResult = experiment.Execute(TrainDataView, ValidationDataView, columnInformation, null, progressHandler);
        
		var topRuns = experimentResult.RunDetails
        	.Where(r => r.ValidationMetrics != null && !double.IsNaN(r.ValidationMetrics.RSquared))
			.OrderByDescending(r => r.ValidationMetrics.RSquared).Take(5);

         Console.WriteLine("\nTop models ranked by rsquared --");
         
		for (var _iterationIndex = 0; _iterationIndex < topRuns.Count(); _iterationIndex++)
        {
            var iterationResult = topRuns.ElementAt(_iterationIndex);
			Console.WriteLine("|" + $"{_iterationIndex,-4} {iterationResult.TrainerName,-35} {iterationResult.ValidationMetrics?.RSquared ?? double.NaN,8:F4} {iterationResult.ValidationMetrics?.MeanAbsoluteError ?? double.NaN,13:F2} {iterationResult.ValidationMetrics?.MeanSquaredError ?? double.NaN,12:F2} {iterationResult.ValidationMetrics?.RootMeanSquaredError ?? double.NaN,8:F2} {iterationResult.RuntimeInSeconds,9:F1}".PadRight(112) + "|");
        }
    	
		Console.WriteLine("\n===== Evaluating model's rsquared with test data =====");
        IDataView predictions = experimentResult.BestRun.Model.Transform(TestDataView);
        var metrics = mlContext.Regression.Evaluate(predictions, labelColumnName: "Label", scoreColumnName: "Score");
		Console.WriteLine("|" + $"{"-",-4} {experimentResult.BestRun.TrainerName,-35} {metrics?.RSquared ?? double.NaN,8:F4} {metrics?.MeanAbsoluteError ?? double.NaN,13:F2} {metrics?.MeanSquaredError ?? double.NaN,12:F2} {metrics?.RootMeanSquaredError ?? double.NaN,8:F2} {experimentResult.BestRun.RuntimeInSeconds,9:F1}".PadRight(112) + "|");
		
		// Re-fit best pipeline on train and validation data, to produce 
        // a model that is trained on as much data as is available while
		// still having test data for the final estimate of how well the
		// model will do in production.
        Console.WriteLine("\n===== Refitting on train+valid and evaluating model's rsquared with test data =====");
		var TrainPlusValidationDataView = textLoader.Load(new MultiFileSource(TrainDataPath, ValidationDataPath));
		var refitModel1 = experimentResult.BestRun.Estimator.Fit(TrainPlusValidationDataView);
		IDataView predictionsRefitOnTrainPlusValidation = refitModel1.Transform(TestDataView);
        var metricsRefitOnTrainPlusValidation = mlContext.Regression.Evaluate(predictionsRefitOnTrainPlusValidation, labelColumnName: "Label", scoreColumnName: "Score");
		Console.WriteLine("|" + $"{"-",-4} {experimentResult.BestRun.TrainerName,-35} {metricsRefitOnTrainPlusValidation?.RSquared ?? double.NaN,8:F4} {metricsRefitOnTrainPlusValidation?.MeanAbsoluteError ?? double.NaN,13:F2} {metricsRefitOnTrainPlusValidation?.MeanSquaredError ?? double.NaN,12:F2} {metricsRefitOnTrainPlusValidation?.RootMeanSquaredError ?? double.NaN,8:F2} {"-",9}".PadRight(112) + "|");
		
		// Re-fit best pipeline on train, validation, and test data, to 
		// produce a model that is trained on as much data as is available.
        // This is the final model that can be deployed to production.
		Console.WriteLine("\n===== Refitting on train+valid+test to get the final model to launch to production =====");
        var TrainPlusValidationPlusTestDataView = textLoader.Load(new MultiFileSource(TrainDataPath, ValidationDataPath, TestDataPath));
		var refitModel2 = experimentResult.BestRun.Estimator.Fit(TrainPlusValidationPlusTestDataView);
		
		// Save / persist the refit model to a.ZIP file.
		Console.WriteLine("\n=============== Saving the model ===============");
        mlContext.Model.Save(refitModel2, TrainDataView.Schema, "model.zip");
        Console.WriteLine($"The model is saved to model.zip");	
	}
}

public class RegressionExperimentProgressHandler : IProgress<RunDetail<RegressionMetrics>>
{
	private int _iterationIndex;

	public void Report(RunDetail<RegressionMetrics> iterationResult)
    {
		if (_iterationIndex++ == 0)
			Console.WriteLine("|" + $"{"",-4} {"Trainer",-35} {"RSquared",8} {"Absolute-loss",13} {"Squared-loss",12} {"RMS-loss",8} {"Duration",9}".PadRight(112) + "|");
			
		if (iterationResult.Exception != null)
			Console.WriteLine($"Exception during AutoML iteration: {iterationResult.Exception}");
		else
			Console.WriteLine("|" + $"{_iterationIndex,-4} {iterationResult.TrainerName,-35} {iterationResult.ValidationMetrics?.RSquared ?? double.NaN,8:F4} {iterationResult.ValidationMetrics?.MeanAbsoluteError ?? double.NaN,13:F2} {iterationResult.ValidationMetrics?.MeanSquaredError ?? double.NaN,12:F2} {iterationResult.ValidationMetrics?.RootMeanSquaredError ?? double.NaN,8:F2} {iterationResult.RuntimeInSeconds,9:F1}".PadRight(112) + "|");
	}
}
							
;