using System.Collections.Generic;
using System.Globalization;
using Microsoft.Azure.Cis.Activities.Contracts;
using Microsoft.Azure.Topology.DcmtLib;
using Microsoft.WindowsAzure.Bootstrap.Common;
using Microsoft.WindowsAzure.Cis.Workflow.SystemActivities.DCMTKustoLoader.Common;
using Microsoft.WindowsAzure.Cis.Workflow.SystemActivities.DCMTKustoLoader.DataModel;
using Microsoft.WindowsAzure.Topology.Utilities;
namespace Microsoft.WindowsAzure.Cis.Workflow.SystemActivities.DCMTKustoLoader.Business
public class DcmtQueryHelper
public DcmtQueryHelper() {
private DcmtDepotHelper dcmtDepotHelper;
private AzureCloudEnv cloudEnv;
private AvailabilityZones azs;
private DatacenterRegions regions;
private CloudGeographies geo;
const string Delimiter = ",";
static Dictionary<string, string> DCtoAZMappings = new Dictionary<string, string>();
static Dictionary<string, string> DCtoAZGuidMappings = new Dictionary<string, string>();
static Dictionary<string, string> DCtoAGRegionIds = new Dictionary<string, string>();
static Dictionary<string, string> DCtoRegionMapping = new Dictionary<string, string>();
static Dictionary<string, string> DCtoExceptionMapping = new Dictionary<string, string>();
static Dictionary<string, string> ClustertoExceptionMapping = new Dictionary<string, string>();
string createTime = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
private const string suppressedClustersKustoQuery = "let v1 = AzureBuildoutMasterJobSnapshot | where Workflow == \"Buildout\" and isnotempty(ClusterName)" +
" and StateName != \"Finished\" and isnull(CIS_RTWDate) and isnull( BuildOut_ActualLiveDate) | project ClusterName;" +
" let v2 = AzureDecomMasterJobSnapshot | where StateName != \"Finished\" | project ClusterName;" +
" let v3 = ControlPlaneMasterJobSnapshot | where isnotempty(ClusterName) and State != \"Finished\" | project ClusterName;" +
" let v4 = ControlPlaneMasterJobSnapshot | where isempty(ClusterName) and State != \"Finished\" | parse DisplayName with \"ControlPlaneBuildout-MainJob-\" CN | project CN | project ClusterName = CN;" +
" let suppressedCluster = v1 | union v2, v3, v4 | distinct ClusterName;" +
" DCMTClusterInfo() | join kind = leftouter suppressedCluster on $left.Id == $right.ClusterName | project Id, IsSuppressed, ClusterName | where IsSuppressed == 1 or isnotempty(ClusterName) ";
protected CloudModel GetCloudModel(DcmtDepotHelper dcmtDepotHelper, String cloudType)
CloudEnvironments AzureClouds = CloudEnvironments.Load(dcmtDepotHelper, false);
cloudEnv = AzureClouds.GetCloudByName(cloudType);
return new CloudModel(cloudEnv, createTime);
protected CloudGeographies GetGeo(DcmtDepotHelper dcmtDepotHelper)
azs = AvailabilityZones.Load(dcmtDepotHelper, null);
regions = azs.GetAllDatacenterRegions();
protected DatacenterRegions GetDatacenterRegions()
protected AzureCloudEnv GetCloudEnv()
public void GetCloudTopology(KeyValuePair<string, string> dcmtRoot, IActivityContext context, TopologyModel topology)
var cloudType = dcmtRoot.Key;
var dcmtPathForCloud = dcmtRoot.Value;
context.TraceInfo($"\t\tGet Cloud Topology for {cloudType} - from dcmt path {dcmtPathForCloud}");
var DCtoAZMapping = new Dictionary<string, string>();
var DCtoAZGuidMapping = new Dictionary<string, string>();
if (DCtoAGRegionIds == null)
DCtoAGRegionIds = new Dictionary<string, string>();
dcmtDepotHelper = new DcmtDepotHelper(dcmtPathForCloud);
CloudModel cloudModel = GetCloudModel(dcmtDepotHelper, cloudType);
string cloudInfo = cloudModel.ToString();
context.TraceInfo($"\t\tCloud Info: {cloudInfo} get successfully");
topology.clouds.Add(cloudInfo);
if (DCtoExceptionMapping == null)
DCtoExceptionMapping = new Dictionary<string, string>();
if (ClustertoExceptionMapping == null)
ClustertoExceptionMapping = new Dictionary<string, string>();
foreach (RegionWithAvailabilityZones rg in azs.Regions)
int? dc_rg_count = rg.ExceptionsInRegion?.AZDatacenterExceptions?.Count;
if (dc_rg_count != null && dc_rg_count > 0)
foreach (var dc_excep in rg.ExceptionsInRegion.AZDatacenterExceptions)
var dc_excep_id = dc_excep.ID.ToLowerInvariant().Trim();
var dc_except_info = dc_excep.ExceptionInfo.ToLowerInvariant().Trim();
if (!DCtoExceptionMapping.ContainsKey(dc_excep_id))
DCtoExceptionMapping.Add(dc_excep_id, dc_except_info);
int? cluster_rg_count = rg.ExceptionsInRegion?.AZClusterExceptions?.Count;
if (cluster_rg_count != null && cluster_rg_count > 0)
foreach (var cluster_excep in rg.ExceptionsInRegion.AZClusterExceptions)
var cluster_excep_id = cluster_excep.Name.ToLowerInvariant().Trim();
var cluster_excep_info = cluster_excep.ExceptionInfo.ToLowerInvariant().Trim();
if (!ClustertoExceptionMapping.ContainsKey(cluster_excep_id))
ClustertoExceptionMapping.Add(cluster_excep_id, cluster_excep_info);
foreach (CloudGeography g in geo.Geographies)
GeoModel geoModel = new GeoModel(g, cloudType, GetCloudEnv(), createTime);
topology.geos.Add(geoModel.ToString());
foreach (DatacenterRegion dcRegion in regions.Regions)
CloudGeography g = geo.GetGeographyByRegionID(dcRegion.ID);
RegionModel regionModel = new RegionModel(dcRegion, cloudType, g, GetCloudEnv(), createTime);
topology.regions.Add(regionModel.ToString());
IEnumerable<AvailabilityZone> it = azs.GetAllZonesInRegion(dcRegion.ID);
foreach (AvailabilityZone az in it)
AvailabilityZoneModel azModel = new AvailabilityZoneModel(az, dcRegion, createTime);
topology.availabilityZones.Add(azModel.ToString());
string[] dcList = (az.ArtifactDatacenters ?? "").Split(';');
foreach (string dc in dcList)
if (!DCtoAZMapping.ContainsKey(dc.ToLowerInvariant().Trim()))
DCtoAZMapping.Add(dc.ToLowerInvariant().Trim(), az.ID.ToLowerInvariant());
if (!DCtoAZGuidMapping.ContainsKey(dc.ToLowerInvariant().Trim()))
DCtoAZGuidMapping.Add(dc.ToLowerInvariant().Trim(), az.Guid.ToString());
foreach (DatacenterInRegion dc in dcRegion.Datacenters)
DataCenterModel dcModel = new DataCenterModel(dc, dcRegion, createTime);
string AZId = string.Empty;
string AZGuid = string.Empty;
string AZ_Exceptions = string.Empty;
if (DCtoAZMapping.ContainsKey(dc.ID.ToLowerInvariant()))
AZId = DCtoAZMapping[dc.ID.ToLowerInvariant()];
string key = dc.ID.ToUpperInvariant() + cloudType.ToUpperInvariant();
string value = dcRegion.ID.ToUpperInvariant() + "_" + dc.ID.ToUpperInvariant() + "_" + cloudType.ToUpperInvariant();
if (DCtoAZGuidMapping.ContainsKey(dc.ID.ToLowerInvariant()))
AZGuid = DCtoAZGuidMapping[dc.ID.ToLowerInvariant()];
if (DCtoExceptionMapping.ContainsKey(dc.ID.ToLowerInvariant()))
AZ_Exceptions = DCtoExceptionMapping[dc.ID.ToLowerInvariant()];
dcModel.AvailabilityZoneId = AZId;
dcModel.AZExceptions = AZ_Exceptions ?? "";
if (!DCtoAGRegionIds.ContainsKey(key))
DCtoAGRegionIds.Add(key, value);
topology.datacenters.Add(dcModel.ToString());
string filePath = dcmtDepotHelper.GetEdgeZonesXmlLocation(dc.ID);
if (File.Exists(filePath))
Status status = IOHelper.Deserialize<EdgeZones>(filePath, out EdgeZones ezs, warnAboutUnknownItems: false);
throw new ArgumentException(String.Format("Failed to load '{0}'", filePath), nameof(filePath));
foreach (EdgeZone edgeZone in ezs.EdgeZonesList)
EdgeZoneModel ezModel = new EdgeZoneModel(edgeZone, createTime, dcRegion.Guid.ToString(), dc.Guid.ToString());
topology.edgeZones.Add(ezModel.ToString());
public static HashSet<string> LoadCloudVipClusterList(string cloud, IActivityContext context, string dcmtPath, string suppClusterPath, string cloudVipPath, Dictionary<string, string> DCtoAGRegionIds, Dictionary<string, string> ClustertoExceptionMapping)
var SuppressCluster = new HashSet<string>();
List<string> clusters = new List<string>();
if (!GetAllSuppressedClusters(context, out clusters))
throw new DCMTKustoQueryException("Error executing Kusto query for suppresed clusters");
foreach (string cluster in clusters)
string cl = (cluster ?? "").ToLowerInvariant().Trim();
if (!SuppressCluster.Contains(cl))
HashSet<string> cloudVipClusterList = new HashSet<string>();
string dcmtPublicCloudClusterFilePath = Path.Combine(dcmtPath, cloudVipPath);
if (!File.Exists(dcmtPublicCloudClusterFilePath))
return cloudVipClusterList;
using (StreamReader sr = new StreamReader(dcmtPublicCloudClusterFilePath))
for (string inptLine = sr.ReadLine(); inptLine != null; inptLine = sr.ReadLine())
if (string.IsNullOrWhiteSpace(inptLine))
if (inptLine.StartsWith(";"))
if (inptLine.StartsWith("localhost", StringComparison.OrdinalIgnoreCase))
if (inptLine.StartsWith("pilotfish", StringComparison.OrdinalIgnoreCase))
int index = inptLine.IndexOf(',');
string[] array = inptLine.Split(',');
string clusterInfo = string.Empty;
string cluster = array[0].Trim().ToLowerInvariant();
string key = array[3].Trim().ToUpperInvariant() + cloud.ToUpperInvariant();
string AGDCId = string.Empty;
if (DCtoAGRegionIds.ContainsKey(key))
AGDCId = DCtoAGRegionIds[key];
string clusterException = string.Empty;
if (ClustertoExceptionMapping.ContainsKey(cluster))
clusterException = ClustertoExceptionMapping[cluster];
for (int i = 0; i < array.Length; i++)
string temp = array[i].Trim();
if (!cloudVipClusterList.Contains(clusterInfo))
if (SuppressCluster.Contains(cluster))
cloudVipClusterList.Add(clusterInfo + ",true," + AGDCId + "," + clusterException);
cloudVipClusterList.Add(clusterInfo + ",false," + AGDCId + "," + clusterException);
return cloudVipClusterList;
public bool GetClustersFromCloudVip(string cloud, IActivityContext context, string dcmtPath, string suppClusterPath, string cloudVipPath, out List<string> clusters)
if (DCtoAGRegionIds == null || DCtoAGRegionIds.Count == 0)
context.TraceError("Please run GetPhysicalAssets method first.");
if (ClustertoExceptionMapping == null || ClustertoExceptionMapping.Count == 0)
context.TraceError("Please run GetPhysicalAssets method first.");
clusters = new List<string>();
HashSet<string> CloudsVip = LoadCloudVipClusterList(cloud, context, dcmtPath, suppClusterPath, cloudVipPath, DCtoAGRegionIds, ClustertoExceptionMapping);
foreach (string line in CloudsVip)
string newLine = line + "," + createTime;
context.TraceError(e.ToString());
public static bool GetAllSuppressedClusters(IActivityContext context, out List<string> suppressedClusters)
suppressedClusters = new List<string>();
List<string> queryResult = new List<string>();
var kustoConnector = new KustoConnector(Config.kustoCluster, context);
if (!kustoConnector.ExecuteQuery(suppressedClustersKustoQuery, out queryResult))
context.TraceError("Error executing Kusto query\n");
suppressedClusters = queryResult.Select(row => row.Split(',')[0]).ToList();
public static void CreateNonBlockingIcm(string title, string errorMessage, IActivityContext context)
long incidentId = context.FileIncident(
new Dictionary<IncidentParameters, string>
{ IncidentParameters.IncidentTitle, title },
{ IncidentParameters.IncidentDescription, errorMessage }
ignoreIncidentTitlePrefix: false,
createWorkflowIncident: false,
createOperationHistory: true);
context.TraceInfo($"Created non-blocking incident: {incidentId}");