using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
namespace SampleConsoleApp
private static object m_Lock = new Object();
private static string logFileName;
public static void Main(string[] args)
const string username = "";
const string password = "";
const string remoteFolderPath = "Reports";
string localFolderPath = System.Environment.CurrentDirectory + @"\DownloadedFiles";
Task.Factory.StartNew((Cntr) =>
int Cnt = ((dynamic)Cntr).value;
LogMsg(string.Format("Interval {0} started at {1}", Cnt, DateTime.Now), Cnt);
using (SftpClient sftp = new SftpClient(host, port, username, password))
LogMsg(string.Format("Attempting to connect to SFTP remote...."), Cnt);
LogMsg(string.Format("SFTP connection established"), Cnt);
bool recursiveDownload = true;
DownloadDirectory(Cnt, "client_reports", sftp, remoteFolderPath, localFolderPath, recursiveDownload);
LogMsg(string.Format("An exception has been caught " + er.ToString()), Cnt);
if (sftp != null) sftp.Disconnect();
LogMsg(string.Format("Interval {0} completed at {1}{2}", Cnt, DateTime.Now, Environment.NewLine), Cnt);
}, new { value = Counter });
private static void DownloadDirectory(int Cnt, string FileNameToLook, SftpClient client, string source, string destination, bool recursive = false)
if (!Directory.Exists(destination)) Directory.CreateDirectory(destination);
LogMsg("Getting list of files from SFTP remote folder....", Cnt);
var initialRemoteList = client.ListDirectory(source).ToList<SftpFile>();
LogMsg("files list received from SFTP remote folder....", Cnt);
var remoteFiles = initialRemoteList.Where(s => s.Name.Contains(FileNameToLook)).ToDictionary(f => f.Name, f => new Dictionary<string, object>() { { "LastWriteTime", f.LastWriteTime }, { "LastWriteTimeUtc", f.LastWriteTimeUtc } });
var localFiles = (new DirectoryInfo(destination)).GetFiles().Where(s => s.Name.Contains(FileNameToLook)).ToDictionary(f => f.Name, f => f.LastWriteTimeUtc);
var finalFiles = remoteFiles.GroupJoin(localFiles, rmt => rmt.Key, src => src.Key, (remoteItem, localItem) => new { remoteItem, localItem }).Where(f => ((f.localItem.Count() == 0) || (f.remoteItem.Key == f.localItem.FirstOrDefault().Key && f.remoteItem.Value.Where(t => t.Key == "LastWriteTimeUtc").Select(v => DateTime.Parse(v.Value.ToString())).FirstOrDefault() > f.localItem.FirstOrDefault().Value))).Select(fl => new { FileName = fl.remoteItem.Key });
var files = initialRemoteList.Join(finalFiles, initial => initial.Name, final => final.FileName, (initialItem, finalItem) => initialItem);
LogMsg(string.Format("Files to be downloaded - Count : {0}{1}{2}", files.Count(), Environment.NewLine, string.Join(", ", files.Select(s => s.Name))), Cnt);
files.ToList().ForEach((SftpFile file) =>
if (!file.IsDirectory && !file.IsSymbolicLink)
DownloadFile(Cnt, client, file, destination, remoteFiles.Where(s => s.Key == file.Name).Select(t => t.Value).FirstOrDefault().ToDictionary(f => f.Key, f => f.Value));
else if (file.IsSymbolicLink)
LogMsg(string.Format("Symbolic link ignored: {0}", file.FullName), Cnt);
else if (file.Name != "." && file.Name != "..")
var dir = Directory.CreateDirectory(Path.Combine(destination, file.Name));
DownloadDirectory(Cnt, FileNameToLook, client, file.FullName, dir.FullName);
private static void DownloadFile(int Cnt, SftpClient client, SftpFile file, string directory, Dictionary<string,object> remoteFileAttr)
LogMsg(string.Format("Downloading {0}", file.FullName), Cnt);
if (File.Exists(Path.Combine(directory, file.Name)))
LogMsg(string.Format("File {0} is available at source and is being archived", file.Name), Cnt);
if (!Directory.Exists(directory + "/Archive")) Directory.CreateDirectory(directory + "/Archive");
var fileNameWithoutExtn = file.Name.Substring(0, file.Name.IndexOf("."));
File.Copy(Path.Combine(directory, file.Name), string.Format("{0}/{1}/{2}", directory, "Archive", file.Name.Replace(fileNameWithoutExtn, fileNameWithoutExtn + "_" + DateTime.Now.ToString("_yyyyMMdd_HHmmss_fff")), true));
File.Delete(Path.Combine(directory, file.Name));
LogMsg(string.Format("File {0} at source is archived", file.Name), Cnt);
using (Stream fileStream = File.OpenWrite(Path.Combine(directory, file.Name)))
client.DownloadFile(file.FullName, fileStream);
File.SetLastWriteTimeUtc(Path.Combine(directory, file.Name), remoteFileAttr.Where(s => s.Key == "LastWriteTimeUtc").Select(r => DateTime.Parse(r.Value.ToString())).FirstOrDefault());
File.SetLastWriteTime(Path.Combine(directory, file.Name), remoteFileAttr.Where(s => s.Key == "LastWriteTime").Select(r => DateTime.Parse(r.Value.ToString())).FirstOrDefault());
LogMsg(string.Format("Downloading {0} complete!!", file.FullName), Cnt);
private static void WriteToFile(string FilePath, string msg)
using (FileStream file = new FileStream(FilePath, FileMode.Append, FileAccess.Write, FileShare.Read))
using (StreamWriter writer = new StreamWriter(file, Encoding.GetEncoding(1250)))
writer.WriteLine(WebUtility.HtmlDecode(msg));
static Action<Action, int> WithRetry = (Action action, int timeoutMs) =>
timeoutMs = (timeoutMs == null ? 1000 : timeoutMs);
var time = Stopwatch.StartNew();
while (time.ElapsedMilliseconds < timeoutMs)
static Func<string> getFileName = () => Path.Combine(System.Environment.CurrentDirectory, string.Format("log{0}.log", DateTime.Now.ToString("_yyyyMMdd_HHmmss_fff")));
static Action<string, int> LogMsg = (msg, counter) =>
if (string.IsNullOrEmpty(logFileName)) logFileName = getFileName();
if (File.Exists(logFileName)) { if ((new FileInfo(logFileName)).Length > 1048576) logFileName = getFileName(); }
if(!File.Exists(logFileName)) File.Create(logFileName);
WriteToFile(logFileName, string.Format("{0}|Interval - {1}|{2}", DateTime.Now, counter, msg));
Console.WriteLine(string.Format("{0}|Interval - {1}|{2}", DateTime.Now, counter, msg));