using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Diagnostics.Contracts;
using System.Collections.Generic;
public static void Main()
var tasks = new List<Task>();
tasks.Add(Task.Factory.StartNew(() => Run()));
tasks.Add(Task.Factory.StartNew(() => Run()));
tasks.Add(Task.Factory.StartNew(() => Run()));
tasks.Add(Task.Factory.StartNew(() => Run()));
Task.WaitAll(tasks.ToArray());
Console.WriteLine("All threads complete");
var url = "image/AlexNetPrepOnnx/AlexNetPreprocess.onnx.BADLINK";
var fileName = "AlexNetPreprocess.onnx";
var ensureModel = Microsoft.ML.Internal.Utilities.ResourceManagerUtils.EnsureResourceAsync( url, fileName, dir, timeout);
var errorResult = Microsoft.ML.Internal.Utilities.ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
var directory = Path.GetDirectoryName(errorResult.FileName);
var name = Path.GetFileName(errorResult.FileName);
Console.WriteLine($"{errorMessage}\nMeta file could not be downloaded! " +
$@"Please copy the model file '{name}' from '{url}' to '{directory}'.");
public class WebClientResponseUri : WebClient
public Uri ResponseUri { get; private set; }
protected override WebResponse GetWebResponse(WebRequest request)
WebResponse response = null;
response = base.GetWebResponse(request);
ResponseUri = response.ResponseUri;
namespace Microsoft.ML.Internal.Utilities
internal sealed class ResourceManagerUtils
private static volatile ResourceManagerUtils _instance;
private static readonly object instanceLock = new object();
public static ResourceManagerUtils Instance
Interlocked.CompareExchange(ref _instance, new ResourceManagerUtils(), null) ??
private const string DefaultUrl = "https://aka.ms/mlnet-resources/";
private static string MlNetResourcesUrl
var envUrl = Environment.GetEnvironmentVariable(CustomResourcesUrlEnvVariable);
if (!string.IsNullOrEmpty(envUrl))
public const string TimeoutEnvVariable = "MICROSOFTML_RESOURCE_TIMEOUT";
public const string CustomResourcesUrlEnvVariable = "MICROSOFTML_RESOURCE_URL";
public sealed class ResourceDownloadResults
public readonly string FileName;
internal readonly string ErrorMessage;
internal readonly string DownloadUrl;
public ResourceDownloadResults(string fileName, string errorMessage, string downloadUrl = null)
ErrorMessage = errorMessage;
DownloadUrl = downloadUrl;
private ResourceManagerUtils()
public static string GetUrl(string suffix)
return $"{MlNetResourcesUrl}{suffix}";
public static async Task<ResourceDownloadResults> EnsureResourceAsync( string relativeUrl, string fileName, string dir, int timeout)
var filePath = GetFilePath( fileName, dir, out var error);
if (File.Exists(filePath) || !string.IsNullOrEmpty(error))
return new ResourceDownloadResults(filePath, error);
if (!Uri.TryCreate(Path.Combine(MlNetResourcesUrl, relativeUrl), UriKind.Absolute, out var absoluteUrl))
return new ResourceDownloadResults(filePath,
$"Could not create a valid URI from the base URI '{MlNetResourcesUrl}' and the relative URI '{relativeUrl}'");
return new ResourceDownloadResults(filePath,
await DownloadFromUrlWithRetryAsync( absoluteUrl.AbsoluteUri, fileName, timeout, filePath), absoluteUrl.AbsoluteUri);
private static async Task<string> DownloadFromUrlWithRetryAsync( string url, string fileName,
int timeout, string filePath, int retryTimes = 5)
for (int i = 0; i < retryTimes; ++i)
var thisDownloadResult = await DownloadFromUrlAsync( url, fileName, timeout, filePath);
if (string.IsNullOrEmpty(thisDownloadResult))
return thisDownloadResult;
downloadResult += thisDownloadResult + @"\n";
if (thisDownloadResult.Contains("does not exist"))
private static async Task<string> DownloadFromUrlAsync( string url, string fileName, int timeout, string filePath)
using (var webClient = new WebClientResponseUri())
using (var downloadCancel = new CancellationTokenSource())
bool deleteNeeded = false;
(object sender, EventArgs e) =>
if (File.Exists(filePath) && deleteNeeded)
webClient.Disposed += disposed;
var t = Task.Run(() => DownloadResource( webClient, new Uri(url), filePath, fileName, downloadCancel.Token));
UpdateTimeout(ref timeout);
var timeoutTask = Task.Delay(timeout).ContinueWith(task => default(Exception), TaskScheduler.Default);
Console.WriteLine($"Downloading {fileName} from {url} to {filePath}");
var completedTask = await Task.WhenAny(t, timeoutTask);
if (completedTask != t || completedTask.CompletedResult() != null)
Console.WriteLine("completedTask.CompletedResult() = " + completedTask.CompletedResult());
Console.WriteLine("(await t).Message = " + (await t).Message);
return (await t).Message;
private static void TryDelete( string filePath, bool warn = true)
Console.WriteLine($"File '{filePath}' could not be deleted: {e.Message}");
private static void UpdateTimeout(ref int timeout)
var envTimeout = Environment.GetEnvironmentVariable(TimeoutEnvVariable);
if (!string.IsNullOrWhiteSpace(envTimeout) && int.TryParse(envTimeout, out var res))
private static string GetFilePath( string fileName, string dir, out string error)
var appDataBaseDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
var appDataDir = Path.Combine(appDataBaseDir, "mlnet-resources");
var absDir = Path.Combine(string.IsNullOrEmpty(envDir) ? appDataDir : envDir, dir);
var filePath = Path.Combine(absDir, fileName);
if (!Directory.Exists(appDataBaseDir))
Directory.CreateDirectory(appDataBaseDir);
if (Environment.OSVersion.Platform == PlatformID.Unix)
chmod(appDataBaseDir, 448);
error = "blah1" + e.Message;
if (!Directory.Exists(absDir))
Directory.CreateDirectory(absDir);
private static Exception DownloadResource( WebClientResponseUri webClient, Uri uri, string path, string fileName, CancellationToken ct)
var mutex = new Mutex(false, "Resource" + fileName);
Guid guid = Guid.NewGuid();
string tempPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path), "temp-resource-" + guid.ToString()));
using (var s = webClient.OpenRead(uri))
using (var fs = File.Create(tempPath))
using (var ws = new StreamWriter(fs))
var headers = webClient.ResponseHeaders.GetValues("Content-Length");
var requestUri = webClient.ResponseUri;
Console.WriteLine("requestUri = " + requestUri);
if (requestUri.Host == "www.microsoft.com" && requestUri.ToString().Length < 60)
Console.WriteLine($"The url '{uri}' does not exist. Url was redirected to '{requestUri}'.");
return new Exception($"The url '{uri}' does not exist. Url was redirected to '{requestUri}'.");
if ( !long.TryParse(headers[0], out var size))
long printFreq = (long)(size / 10.0);
var buffer = new byte[4096];
while ((count = s.Read(buffer, 0, 4096)) > 0)
ws.Write(buffer.ToString(), 0, count);
if ((total - (total / printFreq) * printFreq) <= 4096)
Console.WriteLine($"{fileName}: Downloaded {total} bytes out of {size}");
if (ct.IsCancellationRequested)
Console.WriteLine($"{fileName}: Download timed out");
return new Exception("Download timed out");
File.Move(tempPath, path);
Console.WriteLine($"{fileName}: Download complete");
Console.WriteLine($"{fileName}: Could not download. WebClient returned the following error: {e.Message}");
TryDelete( tempPath, warn: false);
public static ResourceDownloadResults GetErrorMessage(out string errorMessage, params ResourceDownloadResults[] result)
var errorResult = result.FirstOrDefault(res => !string.IsNullOrEmpty(res.ErrorMessage));
else if (string.IsNullOrEmpty(errorResult.DownloadUrl))
errorMessage = $"Error downloading resource: {errorResult.ErrorMessage}";
errorMessage = $"Error downloading resource from '{errorResult.DownloadUrl}': {errorResult.ErrorMessage}";
#pragma warning disable IDE1006
[DllImport("libc", SetLastError = true)]
private static extern int chmod(string pathname, int mode);
#pragma warning restore IDE1006
internal static class TaskExtensions
public static TResult CompletedResult<TResult>(this Task<TResult> task)