public class BackPropProgram
public static void Main(string[] args)
Console.WriteLine("\nBegin neural network back-propagation demo");
Console.WriteLine("\nGenerating " + numRows +
" artificial data items with " + numInput + " features");
double[][] allData = MakeAllData(numInput, numHidden, numOutput,
Console.WriteLine("Done");
Console.WriteLine("\nCreating train (80%) and test (20%) matrices");
SplitTrainTest(allData, 0.80, seed, out trainData, out testData);
Console.WriteLine("Done\n");
Console.WriteLine("Training data:");
ShowMatrix(trainData, 4, 2, true);
Console.WriteLine("Test data:");
ShowMatrix(testData, 4, 2, true);
Console.WriteLine("Creating a " + numInput + "-" + numHidden +
"-" + numOutput + " neural network");
NeuralNetwork nn = new NeuralNetwork(numInput, numHidden, numOutput);
Console.WriteLine("\nSetting maxEpochs = " + maxEpochs);
Console.WriteLine("Setting learnRate = " + learnRate.ToString("F2"));
Console.WriteLine("Setting momentum = " + momentum.ToString("F2"));
Console.WriteLine("\nStarting training");
double[] weights = nn.Train(trainData, maxEpochs, learnRate, momentum);
Console.WriteLine("Done");
Console.WriteLine("\nFinal neural network model weights and biases:\n");
ShowVector(weights, 2, 10, true);
double trainAcc = nn.Accuracy(trainData);
Console.WriteLine("\nFinal accuracy on training data = " +
trainAcc.ToString("F4"));
double testAcc = nn.Accuracy(testData);
Console.WriteLine("Final accuracy on test data = " +
Console.WriteLine("\nEnd back-propagation demo\n");
public static void ShowMatrix(double[][] matrix, int numRows,
int decimals, bool indices)
int len = matrix.Length.ToString().Length;
for (int i = 0; i < numRows; ++i)
Console.Write("[" + i.ToString().PadLeft(len) + "] ");
for (int j = 0; j < matrix[i].Length; ++j)
Console.Write(v.ToString("F" + decimals) + " ");
if (numRows < matrix.Length)
Console.WriteLine(". . .");
int lastRow = matrix.Length - 1;
Console.Write("[" + lastRow.ToString().PadLeft(len) + "] ");
for (int j = 0; j < matrix[lastRow].Length; ++j)
double v = matrix[lastRow][j];
Console.Write(v.ToString("F" + decimals) + " ");
public static void ShowVector(double[] vector, int decimals,
int lineLen, bool newLine)
for (int i = 0; i < vector.Length; ++i)
if (i > 0 && i % lineLen == 0) Console.WriteLine("");
if (vector[i] >= 0) Console.Write(" ");
Console.Write(vector[i].ToString("F" + decimals) + " ");
static double[][] MakeAllData(int numInput, int numHidden,
int numOutput, int numRows, int seed)
Random rnd = new Random(seed);
int numWeights = (numInput * numHidden) + numHidden +
(numHidden * numOutput) + numOutput;
double[] weights = new double[numWeights];
for (int i = 0; i < numWeights; ++i)
weights[i] = 20.0 * rnd.NextDouble() - 10.0;
Console.WriteLine("Generating weights and biases:");
ShowVector(weights, 2, 10, true);
double[][] result = new double[numRows][];
for (int i = 0; i < numRows; ++i)
result[i] = new double[numInput + numOutput];
new NeuralNetwork(numInput, numHidden, numOutput);
for (int r = 0; r < numRows; ++r)
double[] inputs = new double[numInput];
for (int i = 0; i < numInput; ++i)
inputs[i] = 20.0 * rnd.NextDouble() - 10.0;
double[] outputs = gnn.ComputeOutputs(inputs);
double[] oneOfN = new double[numOutput];
double maxValue = outputs[0];
for (int i = 0; i < numOutput; ++i)
if (outputs[i] > maxValue)
for (int i = 0; i < numInput; ++i)
result[r][c++] = inputs[i];
for (int i = 0; i < numOutput; ++i)
result[r][c++] = oneOfN[i];
static void SplitTrainTest(double[][] allData, double trainPct,
int seed, out double[][] trainData, out double[][] testData)
Random rnd = new Random(seed);
int totRows = allData.Length;
int numTrainRows = (int)(totRows * trainPct);
int numTestRows = totRows - numTrainRows;
trainData = new double[numTrainRows][];
testData = new double[numTestRows][];
double[][] copy = new double[allData.Length][];
for (int i = 0; i < copy.Length; ++i)
for (int i = 0; i < copy.Length; ++i)
int r = rnd.Next(i, copy.Length);
for (int i = 0; i < numTrainRows; ++i)
for (int i = 0; i < numTestRows; ++i)
testData[i] = copy[i + numTrainRows];
public class NeuralNetwork
private double[][] ihWeights;
private double[] hBiases;
private double[] hOutputs;
private double[][] hoWeights;
private double[] oBiases;
private double[] outputs;
public NeuralNetwork(int numInput, int numHidden, int numOutput)
this.numInput = numInput;
this.numHidden = numHidden;
this.numOutput = numOutput;
this.inputs = new double[numInput];
this.ihWeights = MakeMatrix(numInput, numHidden, 0.0);
this.hBiases = new double[numHidden];
this.hOutputs = new double[numHidden];
this.hoWeights = MakeMatrix(numHidden, numOutput, 0.0);
this.oBiases = new double[numOutput];
this.outputs = new double[numOutput];
this.rnd = new Random(0);
this.InitializeWeights();
private static double[][] MakeMatrix(int rows,
double[][] result = new double[rows][];
for (int r = 0; r < result.Length; ++r)
result[r] = new double[cols];
for (int i = 0; i < rows; ++i)
for (int j = 0; j < cols; ++j)
private void InitializeWeights()
int numWeights = (numInput * numHidden) +
(numHidden * numOutput) + numHidden + numOutput;
double[] initialWeights = new double[numWeights];
for (int i = 0; i < initialWeights.Length; ++i)
initialWeights[i] = (0.001 - 0.0001) * rnd.NextDouble() + 0.0001;
this.SetWeights(initialWeights);
public void SetWeights(double[] weights)
int numWeights = (numInput * numHidden) +
(numHidden * numOutput) + numHidden + numOutput;
if (weights.Length != numWeights)
throw new Exception("Bad weights array in SetWeights");
for (int i = 0; i < numInput; ++i)
for (int j = 0; j < numHidden; ++j)
ihWeights[i][j] = weights[k++];
for (int i = 0; i < numHidden; ++i)
hBiases[i] = weights[k++];
for (int i = 0; i < numHidden; ++i)
for (int j = 0; j < numOutput; ++j)
hoWeights[i][j] = weights[k++];
for (int i = 0; i < numOutput; ++i)
oBiases[i] = weights[k++];
public double[] GetWeights()
int numWeights = (numInput * numHidden) +
(numHidden * numOutput) + numHidden + numOutput;
double[] result = new double[numWeights];
for (int i = 0; i < ihWeights.Length; ++i)
for (int j = 0; j < ihWeights[0].Length; ++j)
result[k++] = ihWeights[i][j];
for (int i = 0; i < hBiases.Length; ++i)
result[k++] = hBiases[i];
for (int i = 0; i < hoWeights.Length; ++i)
for (int j = 0; j < hoWeights[0].Length; ++j)
result[k++] = hoWeights[i][j];
for (int i = 0; i < oBiases.Length; ++i)
result[k++] = oBiases[i];
public double[] ComputeOutputs(double[] xValues)
double[] hSums = new double[numHidden];
double[] oSums = new double[numOutput];
for (int i = 0; i < xValues.Length; ++i)
this.inputs[i] = xValues[i];
for (int j = 0; j < numHidden; ++j)
for (int i = 0; i < numInput; ++i)
hSums[j] += this.inputs[i] * this.ihWeights[i][j];
for (int i = 0; i < numHidden; ++i)
hSums[i] += this.hBiases[i];
for (int i = 0; i < numHidden; ++i)
this.hOutputs[i] = HyperTan(hSums[i]);
for (int j = 0; j < numOutput; ++j)
for (int i = 0; i < numHidden; ++i)
oSums[j] += hOutputs[i] * hoWeights[i][j];
for (int i = 0; i < numOutput; ++i)
double[] softOut = Softmax(oSums);
Array.Copy(softOut, outputs, softOut.Length);
double[] retResult = new double[numOutput];
Array.Copy(this.outputs, retResult, retResult.Length);
private static double HyperTan(double x)
if (x < -20.0) return -1.0;
else if (x > 20.0) return 1.0;
else return Math.Tanh(x);
private static double[] Softmax(double[] oSums)
for (int i = 0; i < oSums.Length; ++i)
sum += Math.Exp(oSums[i]);
double[] result = new double[oSums.Length];
for (int i = 0; i < oSums.Length; ++i)
result[i] = Math.Exp(oSums[i]) / sum;
public double[] Train(double[][] trainData, int maxEpochs,
double learnRate, double momentum)
double[][] hoGrads = MakeMatrix(numHidden, numOutput, 0.0);
double[] obGrads = new double[numOutput];
double[][] ihGrads = MakeMatrix(numInput, numHidden, 0.0);
double[] hbGrads = new double[numHidden];
double[] oSignals = new double[numOutput];
double[] hSignals = new double[numHidden];
double[][] ihPrevWeightsDelta = MakeMatrix(numInput, numHidden, 0.0);
double[] hPrevBiasesDelta = new double[numHidden];
double[][] hoPrevWeightsDelta = MakeMatrix(numHidden, numOutput, 0.0);
double[] oPrevBiasesDelta = new double[numOutput];
double[] xValues = new double[numInput];
double[] tValues = new double[numOutput];
double errorSignal = 0.0;
int[] sequence = new int[trainData.Length];
for (int i = 0; i < sequence.Length; ++i)
int errInterval = maxEpochs / 10;
while (epoch < maxEpochs)
if (epoch % errInterval == 0 && epoch < maxEpochs)
double trainErr = Error(trainData);
Console.WriteLine("epoch = " + epoch + " error = " +
trainErr.ToString("F4"));
for (int ii = 0; ii < trainData.Length; ++ii)
Array.Copy(trainData[idx], xValues, numInput);
Array.Copy(trainData[idx], numInput, tValues, 0, numOutput);
for (int k = 0; k < numOutput; ++k)
errorSignal = tValues[k] - outputs[k];
derivative = (1 - outputs[k]) * outputs[k];
oSignals[k] = errorSignal * derivative;
for (int j = 0; j < numHidden; ++j)
for (int k = 0; k < numOutput; ++k)
hoGrads[j][k] = oSignals[k] * hOutputs[j];
for (int k = 0; k < numOutput; ++k)
obGrads[k] = oSignals[k] * 1.0;
for (int j = 0; j < numHidden; ++j)
derivative = (1 + hOutputs[j]) * (1 - hOutputs[j]);
for (int k = 0; k < numOutput; ++k) {
sum += oSignals[k] * hoWeights[j][k];
hSignals[j] = derivative * sum;
for (int i = 0; i < numInput; ++i)
for (int j = 0; j < numHidden; ++j)
ihGrads[i][j] = hSignals[j] * inputs[i];
for (int j = 0; j < numHidden; ++j)
hbGrads[j] = hSignals[j] * 1.0;
for (int i = 0; i < numInput; ++i)
for (int j = 0; j < numHidden; ++j)
double delta = ihGrads[i][j] * learnRate;
ihWeights[i][j] += delta;
ihWeights[i][j] += ihPrevWeightsDelta[i][j] * momentum;
ihPrevWeightsDelta[i][j] = delta;
for (int j = 0; j < numHidden; ++j)
double delta = hbGrads[j] * learnRate;
hBiases[j] += hPrevBiasesDelta[j] * momentum;
hPrevBiasesDelta[j] = delta;
for (int j = 0; j < numHidden; ++j)
for (int k = 0; k < numOutput; ++k)
double delta = hoGrads[j][k] * learnRate;
hoWeights[j][k] += delta;
hoWeights[j][k] += hoPrevWeightsDelta[j][k] * momentum;
hoPrevWeightsDelta[j][k] = delta;
for (int k = 0; k < numOutput; ++k)
double delta = obGrads[k] * learnRate;
oBiases[k] += oPrevBiasesDelta[k] * momentum;
oPrevBiasesDelta[k] = delta;
double[] bestWts = GetWeights();
private void Shuffle(int[] sequence)
for (int i = 0; i < sequence.Length; ++i)
int r = this.rnd.Next(i, sequence.Length);
sequence[r] = sequence[i];
private double Error(double[][] trainData)
double sumSquaredError = 0.0;
double[] xValues = new double[numInput];
double[] tValues = new double[numOutput];
for (int i = 0; i < trainData.Length; ++i)
Array.Copy(trainData[i], xValues, numInput);
Array.Copy(trainData[i], numInput, tValues, 0, numOutput);
double[] yValues = this.ComputeOutputs(xValues);
for (int j = 0; j < numOutput; ++j)
double err = tValues[j] - yValues[j];
sumSquaredError += err * err;
return sumSquaredError / trainData.Length;
public double Accuracy(double[][] testData)
double[] xValues = new double[numInput];
double[] tValues = new double[numOutput];
for (int i = 0; i < testData.Length; ++i)
Array.Copy(testData[i], xValues, numInput);
Array.Copy(testData[i], numInput, tValues, 0, numOutput);
yValues = this.ComputeOutputs(xValues);
int maxIndex = MaxIndex(yValues);
int tMaxIndex = MaxIndex(tValues);
if (maxIndex == tMaxIndex)
return (numCorrect * 1.0) / (numCorrect + numWrong);
private static int MaxIndex(double[] vector)
double biggestVal = vector[0];
for (int i = 0; i < vector.Length; ++i)
if (vector[i] > biggestVal)