public class LogFieldService : ILogFieldService
private readonly ILogFieldProvider _source;
private readonly IS3Store _s3Store;
private readonly IConfigurationProvider _configurationProvider;
private readonly IDependencyContainer _dependencyContainer;
private readonly ValidatorFactory _validatorFactory;
private readonly IHttpContextAccessor _contextAccessor;
private List<LogFieldValidationResult> ValidationResult { get; set; }
private List<FeatureFieldValidationResult> FeatureFieldsValidationResult { get; set; }
public LogFieldService(IConfigurationProvider configurationProvider,
IDependencyContainer dependencyContainer)
_configurationProvider = configurationProvider;
ValidationResult = new List<LogFieldValidationResult>();
FeatureFieldsValidationResult = new List<FeatureFieldValidationResult>();
_dependencyContainer = dependencyContainer;
_validatorFactory = dependencyContainer.GetInstance<ValidatorFactory>();
_contextAccessor = dependencyContainer.GetInstance<IHttpContextAccessor>();
_source = GetLogFieldProvider();
_s3Store = GetS3Provider();
public async Task<List<LogField>> GetAllLogFields()
var config = await GetConfigurationAsync();
var logFields = await _source.GetLogFields(config.TargetBranch);
public async Task<LogFieldValidationResponse> ValidateFieldsUIAsync(LogFieldSearchRequest logFieldSearchRequest)
var logFields = await GetAllLogFields();
var fieldToValidate = new List<LogField>();
var validationErrors = new List<ErrorInfo>();
fieldToValidate.Add(new LogField()
Name = logFieldSearchRequest.FieldName,
IsExternal = logFieldSearchRequest.IsExternal
var eNonHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.ExternalNoHeader, logFields,
validationErrors.AddRange(eNonHeaderFieldValidator.Validate());
var iNonHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.InternalNoHeader, logFields,
validationErrors.AddRange(iNonHeaderFieldValidator.Validate());
var eHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.ExternalHeader,
logFields, fieldToValidate);
validationErrors.AddRange(eHeaderFieldValidator.Validate());
var iHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.InternalHeader,
logFields, fieldToValidate);
validationErrors.AddRange(iHeaderFieldValidator.Validate());
var featureValidator = _validatorFactory.GetValidator(Constants.ValidationField.FeatureBranch,
logFields, fieldToValidate);
if (validationErrors.Any())
var errorByValidationType = validationErrors.GroupBy(a => a.ValidationType).ToDictionary(k => k.Key,
foreach (var validationError in errorByValidationType)
var result = new LogFieldValidationResult
ValidationType = validationError.Key,
ErrorInfo = validationError.Value,
Count = validationError.Value.Count
ValidationResult.Add(result);
var response = new LogFieldValidationResponse()
Result = ValidationResult,
IsSuccessful = ValidationResult.Any() ? false : true
response.LogsUrl = await LogValidationResultAsync(ValidationResult, response);
ExceptionPolicy.HandleException(ex, KeyStore.LogPolicies.LogOnly);
public async Task<BaseResponse> ValidateFields(LogFieldValidationRequest request)
using (new ProfileContext(
$"Validate develop log field for repo - {Constants.RepoName}, Branch- {request.Branch}"))
_dependencyContainer.GetInstanceOrDefault<INotifier>(BaseContext.Current.RequestOrigin.Name);
await notifier.NotifyPendingAsync();
var config = await GetConfigurationAsync();
var task1 = _source.GetLogFields(request.Branch);
var task2 = _source.GetLogFields(config.TargetBranch);
await Task.WhenAll(task1, task2);
var featureFields = task1.Result;
var developFields = task2.Result;
if (featureFields == null || featureFields.Count < 1 || developFields == null)
throw new Exception("Source and/or target values can't be null");
var newlyAddedFields = featureFields.ExceptAll(developFields, LogFieldComparer.Comparer).ToList();
if (newlyAddedFields.Any())
var res = await ValidateFeatureBranch(developFields, newlyAddedFields);
return await Validate(developFields, featureFields, newlyAddedFields);
public async Task<FeatureFieldValidationResponse> ValidateFeatureBranch(List<LogField> developFields, List<LogField> newlyAddedFeatureFields)
var validationErrors = new List<ErrorInfo>();
using (new ProfileContext(
$"Validate feature log field for repo - {Constants.RepoName}"))
var featureValidator = _validatorFactory.GetValidator(Constants.ValidationField.FeatureBranch, developFields, newlyAddedFeatureFields);
validationErrors.AddRange(featureValidator.Validate());
if (!validationErrors.Any())
return new FeatureFieldValidationResponse { IsSuccessful = true };
return await ToFeatureFieldValidationResponse(validationErrors);
public async Task<LogFieldSyncResponse> SyncLogFields(LogFieldSyncRequest logFieldDetails)
using (new ProfileContext(
$"Sync log field for repo - {Constants.RepoName}, Branch- {logFieldDetails.Branch}"))
LogFieldSyncResponse response = new LogFieldSyncResponse();
var config = await GetConfigurationAsync();
var task1 = _source.GetLogFields(config.TargetBranch);
var task2 = _s3Store.GetLogFields(Constants.StandardLogFieldFileNameS3);
await Task.WhenAll(task1, task2);
var s3LogFieldList = task2.Result;
var developBranchStandardFieldList = task1.Result;
var s3LogFieldNames = s3LogFieldList.Select(a => a.Name);
var fieldsDifference = developBranchStandardFieldList.Where(x => !s3LogFieldNames.Contains(x.Name))
if (fieldsDifference != null && fieldsDifference.Any())
var existingSync = _dependencyContainer.GetInstance<ISyncLogFields>(Constants.ExistingLogFields);
var existingFields = await existingSync.Sync(fieldsDifference, _s3Store);
var standardSync = _dependencyContainer.GetInstance<ISyncLogFields>(Constants.StandardLogFields);
var standardFields = await standardSync.Sync(developBranchStandardFieldList, _s3Store);
response.IsSuccessful = true;
response.Message = "Log fields sync success";
response.LogsUrl = await LogSyncResultAsync(fieldsDifference.Count,
developBranchStandardFieldList.Count, existingFields.Count, response);
return new LogFieldSyncResponse { IsSuccessful = true, Message = "Log fields are already in synced." };
public async Task<List<LogField>> GetLogFields(string repoName, string branchName)
using (new ProfileContext(
$"Sync log field for repo - {repoName}, Branch- {branchName}"))
return await _source.GetLogFields(branchName);
ExceptionPolicy.HandleException(ex, KeyStore.LogPolicies.LogOnly);
public async Task<bool> SaveLogFields(List<LogField> LogFields, string fileName)
string file = string.Empty;
file = Constants.MockDevelopStandardLogFieldFile;
file = Constants.MockFeatureStandardLogFieldFile;
file = Constants.MockS3ExistingLogFieldFile;
file = Constants.MockS3ExistingLogFieldFile;
return await _s3Store.AddLogFields(LogFields, $"mock/{file}");
private ILogFieldProvider GetLogFieldProvider()
if ((bool)(_contextAccessor.HttpContext?.Request?.Headers?.TryGetValue("mock", out StringValues headers)))
var value = headers[0].ToLower();
if (value.Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase))
return _dependencyContainer.GetInstance<ILogFieldProvider>(Constants.Mock);
return _dependencyContainer.GetInstance<ILogFieldProvider>(Constants.Default);
private IS3Store GetS3Provider()
if ((bool)(_contextAccessor.HttpContext?.Request?.Headers?.TryGetValue("mock", out StringValues headers)))
var value = headers[0].ToLower();
if (value.Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase))
return _dependencyContainer.GetInstance<IS3Store>(Constants.Mock);
return _dependencyContainer.GetInstance<IS3Store>(Constants.Default);
private async Task<LogFieldValidationResponse> Validate(List<LogField> developFields, List<LogField> featureFields, List<LogField> newlyAddedFields)
var validationErrors = new List<ErrorInfo>();
var compositeValidator = _validatorFactory.GetValidator(Constants.ValidationField.Composite, developFields.ToList(), featureFields);
validationErrors.AddRange(compositeValidator.Validate());
if (newlyAddedFields?.Count > 0)
var eNonHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.ExternalNoHeader, developFields.ToList(),
validationErrors.AddRange(eNonHeaderFieldValidator.Validate());
var iNonHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.InternalNoHeader, developFields.ToList(),
validationErrors.AddRange(iNonHeaderFieldValidator.Validate());
var eHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.ExternalHeader,
developFields.ToList(), newlyAddedFields);
validationErrors.AddRange(eHeaderFieldValidator.Validate());
var iHeaderFieldValidator =
_validatorFactory.GetValidator(Constants.ValidationField.InternalHeader,
developFields.ToList(), newlyAddedFields);
validationErrors.AddRange(iHeaderFieldValidator.Validate());
if (validationErrors.Any())
var errorByValidationType = validationErrors.GroupBy(a => a.ValidationType).ToDictionary(k => k.Key,
foreach (var validationError in errorByValidationType)
var result = new LogFieldValidationResult
ValidationType = validationError.Key,
ErrorInfo = validationError.Value,
Count = validationError.Value.Count
ValidationResult.Add(result);
var response = new LogFieldValidationResponse()
Result = ValidationResult,
IsSuccessful = ValidationResult.Any() ? false : true
response.LogsUrl = await LogValidationResultAsync(ValidationResult, response);
private async Task<FeatureFieldValidationResponse> ToFeatureFieldValidationResponse(List<ErrorInfo> validationErrors)
FeatureFieldValidationResponse featureResponse = new FeatureFieldValidationResponse();
var errorByValidationType = validationErrors.GroupBy(a => a.ValidationType).ToDictionary(k => k.Key,
foreach (var validationError in errorByValidationType)
var result = new FeatureFieldValidationResult()
ValidationType = validationError.Key,
Fields = validationError.Value.Select(x => x.ExistingStandardField).Distinct().ToList(),
FeatureFieldsValidationResult.Add(result);
var response = new FeatureFieldValidationResponse()
ValidationMessage = "Feature branch validation failed for below fields",
Result = FeatureFieldsValidationResult,
IsSuccessful = FeatureFieldsValidationResult.Any() ? false : true
response.LogsUrl = await LogFeatureValidationResultAsync(FeatureFieldsValidationResult, response);
private async Task<string> LogValidationResultAsync(List<LogFieldValidationResult> logFieldValidationResult,
LogFieldValidationResponse logFieldValidationResponse)
var log = new TraceLog { Category = "log_field_validate_action" };
log.SetValue("status", logFieldValidationResponse.IsSuccessful ? "success" : "failure");
log.SetValue("log_field_validation_response",
new Payload(ByteConverter.ToByteArray(logFieldValidationResponse)));
await TraceLogger.TraceAsync(log);
return await GetLogsUrlAsync();
private async Task<string> LogFeatureValidationResultAsync(List<FeatureFieldValidationResult> logFieldValidationResult,
FeatureFieldValidationResponse logFieldValidationResponse)
var log = new TraceLog { Category = "log_field_validate_action" };
log.SetValue("status", logFieldValidationResponse.IsSuccessful ? "success" : "failure");
log.SetValue("log_field_validation_response",
new Payload(ByteConverter.ToByteArray(logFieldValidationResponse)));
await TraceLogger.TraceAsync(log);
return await GetLogsUrlAsync();
private async Task<string> LogSyncResultAsync(int newAddedCount, int totalStandard, int existingFieldCount,
var log = new TraceLog { Category = "log_field_sync_action" };
log.SetValue("total_new_added_standard_fields", newAddedCount);
log.SetValue("total_existing_fields", existingFieldCount);
log.SetValue("total_standard_fields", totalStandard);
log.SetValue("status", response.IsSuccessful ? "success" : "failure");
log.SetValue("log_field_sync_response", new Payload(ByteConverter.ToByteArray(response)));
await TraceLogger.TraceAsync(log);
return await GetLogsUrlAsync();
private async Task<string> GetLogsUrlAsync()
var appSettingSection = Common.Constants.Configuration.Section.AppSetting;
var key = Common.Constants.Configuration.Key.TraceLogsUrl;
var logsUrl = await _configurationProvider.GetGlobalConfigurationAsStringAsync(appSettingSection, key);
var fromDate = DateTime.UtcNow.AddMinutes(-15);
var toDate = DateTime.UtcNow.AddMinutes(15);
logsUrl = string.Format(logsUrl, fromDate.ToString(Common.Constants.KibanaSupportedTimeFormat),
toDate.ToString(Common.Constants.KibanaSupportedTimeFormat), BaseContext.Current.CorrelationId);
public List<LogField> GetExistingLogFields(bool includeStandardFields)
var fileProdName = "Only_Prod_Fields.csv";
var fileQAName = "Only_QA_Fields.csv";
var fileStageName = "Only_Stage_Fields.csv";
var standardFileName = "log-standardization-fields.json";
var fieldFilePath = Path.Combine($"{AppDomain.CurrentDomain.BaseDirectory}LogFieldFiles", fileProdName);
var fields = File.ReadAllLines(fieldFilePath).Skip(1).ToList();
var lstFields = new List<LogField>();
HashSet<string> added = new HashSet<string>();
List<LogField> standardList = new List<LogField>();
if (includeStandardFields)
var standardPath = Path.Combine($"{AppDomain.CurrentDomain.BaseDirectory}LogFieldFiles",
var standardFields = File.ReadAllText(standardPath);
standardList = JsonConvert.DeserializeObject<List<LogField>>(standardFields);
foreach (var item in fields)
LogField item1 = new LogField(item);
var qaPath = Path.Combine($"{AppDomain.CurrentDomain.BaseDirectory}LogFieldFiles", fileQAName);
var qaFields = File.ReadAllLines(qaPath).Skip(1).ToList();
foreach (var item in qaFields)
LogField item1 = new LogField(item);
if (!added.Contains(item1.Name))
var stagePath = Path.Combine($"{AppDomain.CurrentDomain.BaseDirectory}LogFieldFiles", fileStageName);
var stageFields = File.ReadAllLines(stagePath).Skip(1).ToList();
foreach (var item in stageFields)
LogField item1 = new LogField(item);
if (!added.Contains(item1.Name))
public async Task<GitHubConfiguration> GetConfigurationAsync()
var section = Common.Constants.Configuration.Section.AdaptorConfigurations;
var key = Common.Constants.Configuration.Key.Github;
var configuration = await _configurationProvider.GetGlobalConfigurationAsync<GitHubConfiguration>(section, key);
configuration.Validate();