ann: Simple Neural Network - Recognizing Numbers 0-9 | C# Online Compiler

ann: Simple Neural Network - Recognizing Numbers 0-9 by Rodan Sotto

// Code copied from this article https://rubikscode.net/2022/07/04/implementing-simple-neural-network-in-c/
// But the original code can be found here https://github.com/NMZivkovic/SimpleNeuralNetworkInCSharp

// The original code did not come with a concrete problem to be solved by this neural network.
// Had to come up with one myself and in the course of training had to make modifications to the original code:
// 1. To make sure we have random initial weights assigned, made a static instance of Random object instead of being created multiple times
// 2. Bumped the learning rate to 20 just for this problem that we are trying to solve
// 3. Changed the calculation of delta in HandleOutputLayer() and HandleHiddenLayers() to not multiply by -1 as it will not work otherwise

// So the problem I came up with for our neural network to solve is to recognize the numbers 0-9 in an image
// Refer to this blogpost for a more detailed explanation: https://rodansotto.github.io/tech-blog/2022/08/07/how-i-trained-my-csharp-neural-network.html

using System;
using System.Collections.Generic;
using System.Linq;

public interface ISynapse
{
    double Weight { get; set; }
    double PreviousWeight { get; set; }
    double GetOutput();

    bool IsFromNeuron(Guid fromNeuronId);
    void UpdateWeight(double learningRate, double delta);
}

// this will be our synapse, the one connecting neurons
public class Synapse : ISynapse
{
    internal INeuron _fromNeuron;
    internal INeuron _toNeuron;

    /// <summary>
    /// Weight of the connection.
    /// </summary>
    public double Weight { get; set; }

    /// <summary>
    /// Weight that connection had in previous itteration.
    /// Used in training process.
    /// </summary>
    public double PreviousWeight { get; set; }

    public Synapse(INeuron fromNeuraon, INeuron toNeuron, double weight)
    {
        _fromNeuron = fromNeuraon;
        _toNeuron = toNeuron;

        Weight = weight;
        PreviousWeight = 0;
    }

	public static Random rand = new Random();

    public Synapse(INeuron fromNeuraon, INeuron toNeuron)
    {
        _fromNeuron = fromNeuraon;
        _toNeuron = toNeuron;

        // var tmpRandom = new Random();
        // Weight = tmpRandom.NextDouble();
		Weight = rand.NextDouble();
        PreviousWeight = 0;
    }

    /// <summary>
    /// Get output value of the connection.
    /// </summary>
    /// <returns>
    /// Output value of the connection.
    /// </returns>
    public double GetOutput()
    {
        return _fromNeuron.CalculateOutput();
    }

    /// <summary>
    /// Checks if Neuron has a certain number as an input neuron.
    /// </summary>
    /// <param name="fromNeuronId">Neuron Id.</param>
    /// <returns>
    /// True - if the neuron is the input of the connection.
    /// False - if the neuron is not the input of the connection. 
    /// </returns>
    public bool IsFromNeuron(Guid fromNeuronId)
    {
        return _fromNeuron.Id.Equals(fromNeuronId);
    }

    /// <summary>
    /// Update weight.
    /// </summary>
    /// <param name="learningRate">Chossen learning rate.</param>
    /// <param name="delta">Calculated difference for which weight of the connection needs to be modified.</param>
    public void UpdateWeight(double learningRate, double delta)
    {
        PreviousWeight = Weight;
        Weight += learningRate * delta;
    }
}

// these are special connections that are used just for the input layer of the neuron,
// i.e. they are used only for adding input to the entirety of the system
// their weights are always 1 and not updated during the training process
public class InputSynapse : ISynapse
{
    internal INeuron _toNeuron;

    public double Weight { get; set; }
    public double Output { get; set; }
    public double PreviousWeight { get; set; }

    public InputSynapse(INeuron toNeuron)
    {
        _toNeuron = toNeuron;
        Weight = 1;
    }

    public InputSynapse(INeuron toNeuron, double output)
    {
        _toNeuron = toNeuron;
        Output = output;
        Weight = 1;
        PreviousWeight = 1;
    }

    public double GetOutput()
    {
        return Output;
    }

    public bool IsFromNeuron(Guid fromNeuronId)
    {
        return false;
    }

    public void UpdateWeight(double learningRate, double delta)
    {
        throw new InvalidOperationException("It is not allowed to call this method on Input Connecion");
    }
}

public interface IInputFunction
{
    double CalculateInput(List<ISynapse> inputs);
}

// this will be our input function
// as biological neurons have multiple inputs and so the same here
// this input function is usually a weighted sum function, that is input from weighted connections is summarized
public class WeightedSumFunction : IInputFunction
{
    public double CalculateInput(List<ISynapse> inputs)
    {
        return inputs.Select(x => x.Weight * x.GetOutput()).Sum();
    }
}

public interface IActivationFunction
{
    double CalculateOutput(double input);
}

// this will be our output function, the activation function
// you actually have to use the right activation function here to simulate the human biological neural network
// there have been already activation functions developed and some of them are explained here: https://rubikscode.net/2017/11/20/common-neural-network-activation-functions/
// here we are using the Step function, also called as Perceptron, first used back in 1957
public class StepActivationFunction : IActivationFunction
{
    private double _treshold;

    public StepActivationFunction(double treshold)
    {
        _treshold = treshold;
    }

    public double CalculateOutput(double input)
    {
        return Convert.ToDouble(input > _treshold);
    }
}

// Sigmoid activation function implementation
public class SigmoidActivationFunction : IActivationFunction
{
    private double _coeficient;

    public SigmoidActivationFunction(double coeficient)
    {
        _coeficient = coeficient;
    }

    public double CalculateOutput(double input)
    {
        return (1 / (1 + Math.Exp(-input * _coeficient)));
    }
}

// Rectifier activation function implementation
public class RectifiedActivationFuncion : IActivationFunction
{
    public double CalculateOutput(double input)
    {
        return Math.Max(0, input);
    }
}

public interface INeuron
{
	Guid Id { get; }
	double PreviousPartialDerivate { get; set; }

	List<ISynapse> Inputs { get; set; }
	List<ISynapse> Outputs { get; set; }

	void AddInputNeuron(INeuron inputNeuron);
	void AddOutputNeuron(INeuron inputNeuron);
	double CalculateOutput();

	void AddInputSynapse(double inputValue);
	void PushValueOnInput(double inputValue);
}

// this will be our neuron
public class Neuron : INeuron
{
    private IActivationFunction _activationFunction;
    private IInputFunction _inputFunction;

    /// <summary>
    /// Input connections of the neuron.
    /// </summary>
    public List<ISynapse> Inputs { get; set; }

    /// <summary>
    /// Output connections of the neuron.
    /// </summary>
    public List<ISynapse> Outputs { get; set; }

    public Guid Id { get; private set; }

    /// <summary>
    /// Calculated partial derivate in previous iteration of training process.
    /// </summary>
    public double PreviousPartialDerivate { get; set; }

    public Neuron(IActivationFunction activationFunction, IInputFunction inputFunction)
    {
        Id = Guid.NewGuid();
        Inputs = new List<ISynapse>();
        Outputs = new List<ISynapse>();

        _activationFunction = activationFunction;
        _inputFunction = inputFunction;
    }

    /// <summary>
    /// Connect two neurons. 
    /// This neuron is the output neuron of the connection.
    /// </summary>
    /// <param name="inputNeuron">Neuron that will be input neuron of the newly created connection.</param>
    public void AddInputNeuron(INeuron inputNeuron)
    {
        var synapse = new Synapse(inputNeuron, this);
        Inputs.Add(synapse);
        inputNeuron.Outputs.Add(synapse);
    }

    /// <summary>
    /// Connect two neurons. 
    /// This neuron is the input neuron of the connection.
    /// </summary>
    /// <param name="outputNeuron">Neuron that will be output neuron of the newly created connection.</param>
    public void AddOutputNeuron(INeuron outputNeuron)
    {
        var synapse = new Synapse(this, outputNeuron);
        Outputs.Add(synapse);
        outputNeuron.Inputs.Add(synapse);
    }

    /// <summary>
    /// Calculate output value of the neuron.
    /// </summary>
    /// <returns>
    /// Output of the neuron.
    /// </returns>
    public double CalculateOutput()
    {
        return _activationFunction.CalculateOutput(_inputFunction.CalculateInput(this.Inputs));
    }

    /// <summary>
    /// Input Layer neurons just receive input values.
    /// For this they need to have connections.
    /// This function adds this kind of connection to the neuron.
    /// </summary>
    /// <param name="inputValue">
    /// Initial value that will be "pushed" as an input to connection.
    /// </param>
    public void AddInputSynapse(double inputValue)
    {
        var inputSynapse = new InputSynapse(this, inputValue);
        Inputs.Add(inputSynapse);
    }

    /// <summary>
    /// Sets new value on the input connections.
    /// </summary>
    /// <param name="inputValue">
    /// New value that will be "pushed" as an input to connection.
    /// </param>
    public void PushValueOnInput(double inputValue)
    {
        ((InputSynapse)Inputs.First()).Output = inputValue;
    }
}

// this will be the neural layer
// it contains the list of neurons used in the layer and the ConnectLayers method, which is used to glue two layers together.
public class NeuralLayer
{
    public List<INeuron> Neurons;

    public NeuralLayer()
    {
        Neurons = new List<INeuron>();
    }

    /// <summary>
    /// Connecting two layers.
    /// </summary>
    public void ConnectLayers(NeuralLayer inputLayer)
    {
        var combos = Neurons.SelectMany(neuron => inputLayer.Neurons, (neuron, input) => new { neuron, input });
        combos.ToList().ForEach(x => x.neuron.AddInputNeuron(x.input));
    }
}

/// <summary>
/// Factory used to create layers.
/// </summary>
public class NeuralLayerFactory
{
	public NeuralLayer CreateNeuralLayer(int numberOfNeurons, IActivationFunction activationFunction, IInputFunction inputFunction)
	{
		var layer = new NeuralLayer();

		for (int i = 0; i < numberOfNeurons; i++)
		{
			var neuron = new Neuron(activationFunction, inputFunction);
			layer.Neurons.Add(neuron);
		}

		return layer;
	}
}

// This will be the implementation of the simple neural network
// During the construction of the object, the initial input layer is added to the network.
// Other layers are added through the function AddLayer, which adds a passed layer on top of the current layer list.
// The GetOutput method will activate the output layer of the network, thus initiating a chain reaction through the network.
// The most important method of this class is the Train method. It receives the training set and the number of epochs. 
// For each epoch, it runs the whole training set through the network.
// Then, the output is compared with desired output and the functions HandleOutputLayer and HandleHiddenLayer are called. These functions implement the backpropagation algorithm.	
public class SimpleNeuralNetwork
{
    private NeuralLayerFactory _layerFactory;

    internal List<NeuralLayer> _layers;
    internal double _learningRate;
    internal double[][] _expectedResult;

    /// <summary>
    /// Constructor of the Neural Network.
    /// Note:
    /// Initialy input layer with defined number of inputs will be created.
    /// </summary>
    /// <param name="numberOfInputNeurons">
    /// Number of neurons in input layer.
    /// </param>
    public SimpleNeuralNetwork(int numberOfInputNeurons)
    {
        _layers = new List<NeuralLayer>();
        _layerFactory = new NeuralLayerFactory();

        // Create input layer that will collect inputs.
        CreateInputLayer(numberOfInputNeurons);

        _learningRate = 20; //2.95;
    }

    /// <summary>
    /// Add layer to the neural network.
    /// Layer will automatically be added as the output layer to the last layer in the neural network.
    /// </summary>
    public void AddLayer(NeuralLayer newLayer)
    {
        if (_layers.Any())
        {
            var lastLayer = _layers.Last();
            newLayer.ConnectLayers(lastLayer);
        }

        _layers.Add(newLayer);
    }

    /// <summary>
    /// Push input values to the neural network.
    /// </summary>
    public void PushInputValues(double[] inputs)
    {
        _layers.First().Neurons.ForEach(x => x.PushValueOnInput(inputs[_layers.First().Neurons.IndexOf(x)]));
    }

    /// <summary>
    /// Set expected values for the outputs.
    /// </summary>
    public void PushExpectedValues(double[][] expectedOutputs)
    {
        _expectedResult = expectedOutputs;
    }

    /// <summary>
    /// Calculate output of the neural network.
    /// </summary>
    /// <returns></returns>
    public List<double> GetOutput()
    {
        var returnValue = new List<double>();

        _layers.Last().Neurons.ForEach(neuron =>
        {
             returnValue.Add(neuron.CalculateOutput());
        });

        return returnValue;
    }

    /// <summary>
    /// Train neural network.
    /// </summary>
    /// <param name="inputs">Input values.</param>
    /// <param name="numberOfEpochs">Number of epochs.</param>
    public void Train(double[][] inputs, int numberOfEpochs)
    {
        double totalError = 0;

        for(int i = 0; i < numberOfEpochs; i++)
        {
            for(int j = 0; j < inputs.GetLength(0); j ++)
            {
                PushInputValues(inputs[j]);

                var outputs = new List<double>();

                // Get outputs.
                _layers.Last().Neurons.ForEach(x =>
                {
                    outputs.Add(x.CalculateOutput());
                });

                // Calculate error by summing errors on all output neurons.
                totalError = CalculateTotalError(outputs, j);
                HandleOutputLayer(j);
                HandleHiddenLayers();
            }
        }
    }

    /// <summary>
    /// Hellper function that creates input layer of the neural network.
    /// </summary>
    private void CreateInputLayer(int numberOfInputNeurons)
    {
        var inputLayer = _layerFactory.CreateNeuralLayer(numberOfInputNeurons, new RectifiedActivationFuncion(), new WeightedSumFunction());
        inputLayer.Neurons.ForEach(x => x.AddInputSynapse(0));
        this.AddLayer(inputLayer);
    }

    /// <summary>
    /// Hellper function that calculates total error of the neural network.
    /// </summary>
    private double CalculateTotalError(List<double> outputs, int row)
    {
        double totalError = 0;

        outputs.ForEach(output =>
        {
            var error = Math.Pow(output - _expectedResult[row][outputs.IndexOf(output)], 2);
            totalError += error;
        });

        return totalError;
    }

    /// <summary>
    /// Hellper function that runs backpropagation algorithm on the output layer of the network.
    /// </summary>
    /// <param name="row">
    /// Input/Expected output row.
    /// </param>
    private void HandleOutputLayer(int row)
    {
        _layers.Last().Neurons.ForEach(neuron =>
        {
            neuron.Inputs.ForEach(connection =>
            {
                var output = neuron.CalculateOutput();
                var netInput = connection.GetOutput();

                var expectedOutput = _expectedResult[row][_layers.Last().Neurons.IndexOf(neuron)];

                var nodeDelta = (expectedOutput - output) * output * (1 - output);
                //var delta = -1 * netInput * nodeDelta;
                var delta = netInput * nodeDelta;

                connection.UpdateWeight(_learningRate, delta);

                neuron.PreviousPartialDerivate = nodeDelta;
            });
        });
    }

    /// <summary>
    /// Hellper function that runs backpropagation algorithm on the hidden layer of the network.
    /// </summary>
    /// <param name="row">
    /// Input/Expected output row.
    /// </param>
    private void HandleHiddenLayers()
    {
        for (int k = _layers.Count - 2; k > 0; k--)
        {
            _layers[k].Neurons.ForEach(neuron =>
            {
                neuron.Inputs.ForEach(connection =>
                {
                    var output = neuron.CalculateOutput();
                    var netInput = connection.GetOutput();
                    double sumPartial = 0;

                    _layers[k + 1].Neurons
                    .ForEach(outputNeuron =>
                    {
                        outputNeuron.Inputs.Where(i => i.IsFromNeuron(neuron.Id))
                        .ToList()
                        .ForEach(outConnection =>
                        {
                            sumPartial += outConnection.PreviousWeight * outputNeuron.PreviousPartialDerivate;
                        });
                    });

                    //var delta = -1 * netInput * sumPartial * output * (1 - output);
                    var delta = netInput * sumPartial * output * (1 - output);
                    connection.UpdateWeight(_learningRate, delta);
                });
            });
        }
    }

	public double[] DivideDoubleArray(double[] arr, double divideBy)
	{
		for (int i = 0; i < arr.Length; i++)
		{
			arr[i] /= divideBy;
		}
		return arr;
	}
}

public class Program
{
	public static void Main()
	{
		// So our input data represents a 5x7 1-bit image that contains a number from 0 to 9
		// To save space, the input data is an integer array of 7 corresponding to the 7 rows of 5 pixels in that 5x7 image
		// Each row contains the decimal value of the 5 pixels in that row
		//  e.g. if row contains these 5 pixels [0 1 1 1 0], then the decimal value will be 14
		
		// create neural network with 7 neurons in the input layer
		var network = new SimpleNeuralNetwork(7); // <- the input layer will be created here

		// add 2 layers, each with a number of neurons and activation and input functions to use
		var layerFactory = new NeuralLayerFactory();
		network.AddLayer(layerFactory.CreateNeuralLayer(4, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the hidden layer
		network.AddLayer(layerFactory.CreateNeuralLayer(1, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the output layer

		// define expected outputs
		// if you notice I'm dividing the values by 10 because apparently this neural network does not work if they are not between 0 and 1 (including 0 and 1)
		network.PushExpectedValues(
			new double[][] {
				network.DivideDoubleArray(new double[] { 1 }, 10),
				network.DivideDoubleArray(new double[] { 2 }, 10),
				network.DivideDoubleArray(new double[] { 3 }, 10),
				network.DivideDoubleArray(new double[] { 4 }, 10),
				network.DivideDoubleArray(new double[] { 5 }, 10),
				network.DivideDoubleArray(new double[] { 6 }, 10),
				network.DivideDoubleArray(new double[] { 7 }, 10),
				network.DivideDoubleArray(new double[] { 8 }, 10),
				network.DivideDoubleArray(new double[] { 9 }, 10),
				network.DivideDoubleArray(new double[] { 0 }, 10),
			});
			
		// train with the input training set and number of epochs (or times) it will run the training set through the network
		// again I'm dividing the values so they stay within 0 and 1, otherwise it will not work
		// I'm dividing by 31 as the maximum value a row of pixel in our 5x7 image can have is [1 1 1 1 1] = 31 in decimal
		Console.WriteLine("Training...");
		network.Train(
			new double[][] {
				network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31),
				network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31),
				network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31),
				network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31),
				network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31),
			}, 10000);

		// and now to test how good the neural network was trained :)
		Console.WriteLine("Testing...");
		network.PushInputValues(network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31));
		var outputs = network.GetOutput();
		Console.WriteLine($"Input: 1; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 2; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 3; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 4; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 5; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 6; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 7; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 8; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 9; Output: {string.Join(',', outputs)}");

		network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31));
		outputs = network.GetOutput();
		Console.WriteLine($"Input: 0; Output: {string.Join(',', outputs)}");
	}
}

​x
 
// Code copied from this article https://rubikscode.net/2022/07/04/implementing-simple-neural-network-in-c/
// But the original code can be found here https://github.com/NMZivkovic/SimpleNeuralNetworkInCSharp
​
// The original code did not come with a concrete problem to be solved by this neural network.
// Had to come up with one myself and in the course of training had to make modifications to the original code:
// 1. To make sure we have random initial weights assigned, made a static instance of Random object instead of being created multiple times
// 2. Bumped the learning rate to 20 just for this problem that we are trying to solve
// 3. Changed the calculation of delta in HandleOutputLayer() and HandleHiddenLayers() to not multiply by -1 as it will not work otherwise
​
// So the problem I came up with for our neural network to solve is to recognize the numbers 0-9 in an image
// Refer to this blogpost for a more detailed explanation: https://rodansotto.github.io/tech-blog/2022/08/07/how-i-trained-my-csharp-neural-network.html
​
using System;
using System.Collections.Generic;
using System.Linq;
​
public interface ISynapse
{
    double Weight { get; set; }
    double PreviousWeight { get; set; }
    double GetOutput();
​
    bool IsFromNeuron(Guid fromNeuronId);
    void UpdateWeight(double learningRate, double delta);
}
​
// this will be our synapse, the one connecting neurons
public class Synapse : ISynapse
{
    internal INeuron _fromNeuron;
    internal INeuron _toNeuron;
​
    /// <summary>
    /// Weight of the connection.
    /// </summary>
    public double Weight { get; set; }
​
    /// <summary>
    /// Weight that connection had in previous itteration.
    /// Used in training process.
    /// </summary>
    public double PreviousWeight { get; set; }
​
    public Synapse(INeuron fromNeuraon, INeuron toNeuron, double weight)
    {
        _fromNeuron = fromNeuraon;
        _toNeuron = toNeuron;
​
        Weight = weight;
        PreviousWeight = 0;
    }
​
    public static Random rand = new Random();
​
    public Synapse(INeuron fromNeuraon, INeuron toNeuron)
    {
        _fromNeuron = fromNeuraon;
        _toNeuron = toNeuron;
​
        // var tmpRandom = new Random();
        // Weight = tmpRandom.NextDouble();
        Weight = rand.NextDouble();
        PreviousWeight = 0;
    }
​
    /// <summary>
    /// Get output value of the connection.
    /// </summary>
    /// <returns>
    /// Output value of the connection.
    /// </returns>
    public double GetOutput()
    {
        return _fromNeuron.CalculateOutput();
    }
​
    /// <summary>
    /// Checks if Neuron has a certain number as an input neuron.
    /// </summary>
    /// <param name="fromNeuronId">Neuron Id.</param>
    /// <returns>
    /// True - if the neuron is the input of the connection.
    /// False - if the neuron is not the input of the connection. 
    /// </returns>
    public bool IsFromNeuron(Guid fromNeuronId)
    {
        return _fromNeuron.Id.Equals(fromNeuronId);
    }
​
    /// <summary>
    /// Update weight.
    /// </summary>
    /// <param name="learningRate">Chossen learning rate.</param>
    /// <param name="delta">Calculated difference for which weight of the connection needs to be modified.</param>
    public void UpdateWeight(double learningRate, double delta)
    {
        PreviousWeight = Weight;
        Weight += learningRate * delta;
    }
}
​
// these are special connections that are used just for the input layer of the neuron,
// i.e. they are used only for adding input to the entirety of the system
// their weights are always 1 and not updated during the training process
public class InputSynapse : ISynapse
{
    internal INeuron _toNeuron;
​
    public double Weight { get; set; }
    public double Output { get; set; }
    public double PreviousWeight { get; set; }
​
    public InputSynapse(INeuron toNeuron)
    {
        _toNeuron = toNeuron;
        Weight = 1;
    }
​
    public InputSynapse(INeuron toNeuron, double output)
    {
        _toNeuron = toNeuron;
        Output = output;
        Weight = 1;
        PreviousWeight = 1;
    }
​
    public double GetOutput()
    {
        return Output;
    }
​
    public bool IsFromNeuron(Guid fromNeuronId)
    {
        return false;
    }
​
    public void UpdateWeight(double learningRate, double delta)
    {
        throw new InvalidOperationException("It is not allowed to call this method on Input Connecion");
    }
}
​
public interface IInputFunction
{
    double CalculateInput(List<ISynapse> inputs);
}
​
// this will be our input function
// as biological neurons have multiple inputs and so the same here
// this input function is usually a weighted sum function, that is input from weighted connections is summarized
public class WeightedSumFunction : IInputFunction
{
    public double CalculateInput(List<ISynapse> inputs)
    {
        return inputs.Select(x => x.Weight * x.GetOutput()).Sum();
    }
}
​
public interface IActivationFunction
{
    double CalculateOutput(double input);
}
​
// this will be our output function, the activation function
// you actually have to use the right activation function here to simulate the human biological neural network
// there have been already activation functions developed and some of them are explained here: https://rubikscode.net/2017/11/20/common-neural-network-activation-functions/
// here we are using the Step function, also called as Perceptron, first used back in 1957
public class StepActivationFunction : IActivationFunction
{
    private double _treshold;
​
    public StepActivationFunction(double treshold)
    {
        _treshold = treshold;
    }
​
    public double CalculateOutput(double input)
    {
        return Convert.ToDouble(input > _treshold);
    }
}
​
// Sigmoid activation function implementation
public class SigmoidActivationFunction : IActivationFunction
{
    private double _coeficient;
​
    public SigmoidActivationFunction(double coeficient)
    {
        _coeficient = coeficient;
    }
​
    public double CalculateOutput(double input)
    {
        return (1 / (1 + Math.Exp(-input * _coeficient)));
    }
}
​
// Rectifier activation function implementation
public class RectifiedActivationFuncion : IActivationFunction
{
    public double CalculateOutput(double input)
    {
        return Math.Max(0, input);
    }
}
​
public interface INeuron
{
    Guid Id { get; }
    double PreviousPartialDerivate { get; set; }
​
    List<ISynapse> Inputs { get; set; }
    List<ISynapse> Outputs { get; set; }
​
    void AddInputNeuron(INeuron inputNeuron);
    void AddOutputNeuron(INeuron inputNeuron);
    double CalculateOutput();
​
    void AddInputSynapse(double inputValue);
    void PushValueOnInput(double inputValue);
}
​
// this will be our neuron
public class Neuron : INeuron
{
    private IActivationFunction _activationFunction;
    private IInputFunction _inputFunction;
​
    /// <summary>
    /// Input connections of the neuron.
    /// </summary>
    public List<ISynapse> Inputs { get; set; }
​
    /// <summary>
    /// Output connections of the neuron.
    /// </summary>
    public List<ISynapse> Outputs { get; set; }
​
    public Guid Id { get; private set; }
​
    /// <summary>
    /// Calculated partial derivate in previous iteration of training process.
    /// </summary>
    public double PreviousPartialDerivate { get; set; }
​
    public Neuron(IActivationFunction activationFunction, IInputFunction inputFunction)
    {
        Id = Guid.NewGuid();
        Inputs = new List<ISynapse>();
        Outputs = new List<ISynapse>();
​
        _activationFunction = activationFunction;
        _inputFunction = inputFunction;
    }
​
    /// <summary>
    /// Connect two neurons. 
    /// This neuron is the output neuron of the connection.
    /// </summary>
    /// <param name="inputNeuron">Neuron that will be input neuron of the newly created connection.</param>
    public void AddInputNeuron(INeuron inputNeuron)
    {
        var synapse = new Synapse(inputNeuron, this);
        Inputs.Add(synapse);
        inputNeuron.Outputs.Add(synapse);
    }
​
    /// <summary>
    /// Connect two neurons. 
    /// This neuron is the input neuron of the connection.
    /// </summary>
    /// <param name="outputNeuron">Neuron that will be output neuron of the newly created connection.</param>
    public void AddOutputNeuron(INeuron outputNeuron)
    {
        var synapse = new Synapse(this, outputNeuron);
        Outputs.Add(synapse);
        outputNeuron.Inputs.Add(synapse);
    }
​
    /// <summary>
    /// Calculate output value of the neuron.
    /// </summary>
    /// <returns>
    /// Output of the neuron.
    /// </returns>
    public double CalculateOutput()
    {
        return _activationFunction.CalculateOutput(_inputFunction.CalculateInput(this.Inputs));
    }
​
    /// <summary>
    /// Input Layer neurons just receive input values.
    /// For this they need to have connections.
    /// This function adds this kind of connection to the neuron.
    /// </summary>
    /// <param name="inputValue">
    /// Initial value that will be "pushed" as an input to connection.
    /// </param>
    public void AddInputSynapse(double inputValue)
    {
        var inputSynapse = new InputSynapse(this, inputValue);
        Inputs.Add(inputSynapse);
    }
​
    /// <summary>
    /// Sets new value on the input connections.
    /// </summary>
    /// <param name="inputValue">
    /// New value that will be "pushed" as an input to connection.
    /// </param>
    public void PushValueOnInput(double inputValue)
    {
        ((InputSynapse)Inputs.First()).Output = inputValue;
    }
}
​
// this will be the neural layer
// it contains the list of neurons used in the layer and the ConnectLayers method, which is used to glue two layers together.
public class NeuralLayer
{
    public List<INeuron> Neurons;
​
    public NeuralLayer()
    {
        Neurons = new List<INeuron>();
    }
​
    /// <summary>
    /// Connecting two layers.
    /// </summary>
    public void ConnectLayers(NeuralLayer inputLayer)
    {
        var combos = Neurons.SelectMany(neuron => inputLayer.Neurons, (neuron, input) => new { neuron, input });
        combos.ToList().ForEach(x => x.neuron.AddInputNeuron(x.input));
    }
}
​
/// <summary>
/// Factory used to create layers.
/// </summary>
public class NeuralLayerFactory
{
    public NeuralLayer CreateNeuralLayer(int numberOfNeurons, IActivationFunction activationFunction, IInputFunction inputFunction)
    {
        var layer = new NeuralLayer();
​
        for (int i = 0; i < numberOfNeurons; i++)
        {
            var neuron = new Neuron(activationFunction, inputFunction);
            layer.Neurons.Add(neuron);
        }
​
        return layer;
    }
}
​
// This will be the implementation of the simple neural network
// During the construction of the object, the initial input layer is added to the network.
// Other layers are added through the function AddLayer, which adds a passed layer on top of the current layer list.
// The GetOutput method will activate the output layer of the network, thus initiating a chain reaction through the network.
// The most important method of this class is the Train method. It receives the training set and the number of epochs. 
// For each epoch, it runs the whole training set through the network.
// Then, the output is compared with desired output and the functions HandleOutputLayer and HandleHiddenLayer are called. These functions implement the backpropagation algorithm.  
public class SimpleNeuralNetwork
{
    private NeuralLayerFactory _layerFactory;
​
    internal List<NeuralLayer> _layers;
    internal double _learningRate;
    internal double[][] _expectedResult;
​
    /// <summary>
    /// Constructor of the Neural Network.
    /// Note:
    /// Initialy input layer with defined number of inputs will be created.
    /// </summary>
    /// <param name="numberOfInputNeurons">
    /// Number of neurons in input layer.
    /// </param>
    public SimpleNeuralNetwork(int numberOfInputNeurons)
    {
        _layers = new List<NeuralLayer>();
        _layerFactory = new NeuralLayerFactory();
​
        // Create input layer that will collect inputs.
        CreateInputLayer(numberOfInputNeurons);
​
        _learningRate = 20; //2.95;
    }
​
    /// <summary>
    /// Add layer to the neural network.
    /// Layer will automatically be added as the output layer to the last layer in the neural network.
    /// </summary>
    public void AddLayer(NeuralLayer newLayer)
    {
        if (_layers.Any())
        {
            var lastLayer = _layers.Last();
            newLayer.ConnectLayers(lastLayer);
        }
​
        _layers.Add(newLayer);
    }
​
    /// <summary>
    /// Push input values to the neural network.
    /// </summary>
    public void PushInputValues(double[] inputs)
    {
        _layers.First().Neurons.ForEach(x => x.PushValueOnInput(inputs[_layers.First().Neurons.IndexOf(x)]));
    }
​
    /// <summary>
    /// Set expected values for the outputs.
    /// </summary>
    public void PushExpectedValues(double[][] expectedOutputs)
    {
        _expectedResult = expectedOutputs;
    }
​
    /// <summary>
    /// Calculate output of the neural network.
    /// </summary>
    /// <returns></returns>
    public List<double> GetOutput()
    {
        var returnValue = new List<double>();
​
        _layers.Last().Neurons.ForEach(neuron =>
        {
             returnValue.Add(neuron.CalculateOutput());
        });
​
        return returnValue;
    }
​
    /// <summary>
    /// Train neural network.
    /// </summary>
    /// <param name="inputs">Input values.</param>
    /// <param name="numberOfEpochs">Number of epochs.</param>
    public void Train(double[][] inputs, int numberOfEpochs)
    {
        double totalError = 0;
​
        for(int i = 0; i < numberOfEpochs; i++)
        {
            for(int j = 0; j < inputs.GetLength(0); j ++)
            {
                PushInputValues(inputs[j]);
​
                var outputs = new List<double>();
​
                // Get outputs.
                _layers.Last().Neurons.ForEach(x =>
                {
                    outputs.Add(x.CalculateOutput());
                });
​
                // Calculate error by summing errors on all output neurons.
                totalError = CalculateTotalError(outputs, j);
                HandleOutputLayer(j);
                HandleHiddenLayers();
            }
        }
    }
​
    /// <summary>
    /// Hellper function that creates input layer of the neural network.
    /// </summary>
    private void CreateInputLayer(int numberOfInputNeurons)
    {
        var inputLayer = _layerFactory.CreateNeuralLayer(numberOfInputNeurons, new RectifiedActivationFuncion(), new WeightedSumFunction());
        inputLayer.Neurons.ForEach(x => x.AddInputSynapse(0));
        this.AddLayer(inputLayer);
    }
​
    /// <summary>
    /// Hellper function that calculates total error of the neural network.
    /// </summary>
    private double CalculateTotalError(List<double> outputs, int row)
    {
        double totalError = 0;
​
        outputs.ForEach(output =>
        {
            var error = Math.Pow(output - _expectedResult[row][outputs.IndexOf(output)], 2);
            totalError += error;
        });
​
        return totalError;
    }
​
    /// <summary>
    /// Hellper function that runs backpropagation algorithm on the output layer of the network.
    /// </summary>
    /// <param name="row">
    /// Input/Expected output row.
    /// </param>
    private void HandleOutputLayer(int row)
    {
        _layers.Last().Neurons.ForEach(neuron =>
        {
            neuron.Inputs.ForEach(connection =>
            {
                var output = neuron.CalculateOutput();
                var netInput = connection.GetOutput();
​
                var expectedOutput = _expectedResult[row][_layers.Last().Neurons.IndexOf(neuron)];
​
                var nodeDelta = (expectedOutput - output) * output * (1 - output);
                //var delta = -1 * netInput * nodeDelta;
                var delta = netInput * nodeDelta;
​
                connection.UpdateWeight(_learningRate, delta);
​
                neuron.PreviousPartialDerivate = nodeDelta;
            });
        });
    }
​
    /// <summary>
    /// Hellper function that runs backpropagation algorithm on the hidden layer of the network.
    /// </summary>
    /// <param name="row">
    /// Input/Expected output row.
    /// </param>
    private void HandleHiddenLayers()
    {
        for (int k = _layers.Count - 2; k > 0; k--)
        {
            _layers[k].Neurons.ForEach(neuron =>
            {
                neuron.Inputs.ForEach(connection =>
                {
                    var output = neuron.CalculateOutput();
                    var netInput = connection.GetOutput();
                    double sumPartial = 0;
​
                    _layers[k + 1].Neurons
                    .ForEach(outputNeuron =>
                    {
                        outputNeuron.Inputs.Where(i => i.IsFromNeuron(neuron.Id))
                        .ToList()
                        .ForEach(outConnection =>
                        {
                            sumPartial += outConnection.PreviousWeight * outputNeuron.PreviousPartialDerivate;
                        });
                    });
​
                    //var delta = -1 * netInput * sumPartial * output * (1 - output);
                    var delta = netInput * sumPartial * output * (1 - output);
                    connection.UpdateWeight(_learningRate, delta);
                });
            });
        }
    }
​
    public double[] DivideDoubleArray(double[] arr, double divideBy)
    {
        for (int i = 0; i < arr.Length; i++)
        {
            arr[i] /= divideBy;
        }
        return arr;
    }
}
​
public class Program
{
    public static void Main()
    {
        // So our input data represents a 5x7 1-bit image that contains a number from 0 to 9
        // To save space, the input data is an integer array of 7 corresponding to the 7 rows of 5 pixels in that 5x7 image
        // Each row contains the decimal value of the 5 pixels in that row
        //  e.g. if row contains these 5 pixels [0 1 1 1 0], then the decimal value will be 14
        
        // create neural network with 7 neurons in the input layer
        var network = new SimpleNeuralNetwork(7); // <- the input layer will be created here
​
        // add 2 layers, each with a number of neurons and activation and input functions to use
        var layerFactory = new NeuralLayerFactory();
        network.AddLayer(layerFactory.CreateNeuralLayer(4, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the hidden layer
        network.AddLayer(layerFactory.CreateNeuralLayer(1, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the output layer
​
        // define expected outputs
        // if you notice I'm dividing the values by 10 because apparently this neural network does not work if they are not between 0 and 1 (including 0 and 1)
        network.PushExpectedValues(
            new double[][] {
                network.DivideDoubleArray(new double[] { 1 }, 10),
                network.DivideDoubleArray(new double[] { 2 }, 10),
                network.DivideDoubleArray(new double[] { 3 }, 10),
                network.DivideDoubleArray(new double[] { 4 }, 10),
                network.DivideDoubleArray(new double[] { 5 }, 10),
                network.DivideDoubleArray(new double[] { 6 }, 10),
                network.DivideDoubleArray(new double[] { 7 }, 10),
                network.DivideDoubleArray(new double[] { 8 }, 10),
                network.DivideDoubleArray(new double[] { 9 }, 10),
                network.DivideDoubleArray(new double[] { 0 }, 10),
            });
            
        // train with the input training set and number of epochs (or times) it will run the training set through the network
        // again I'm dividing the values so they stay within 0 and 1, otherwise it will not work
        // I'm dividing by 31 as the maximum value a row of pixel in our 5x7 image can have is [1 1 1 1 1] = 31 in decimal
        Console.WriteLine("Training...");
        network.Train(
            new double[][] {
                network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31),
                network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31),
                network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31),
                network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31),
                network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31),
            }, 10000);
​
        // and now to test how good the neural network was trained :)
        Console.WriteLine("Testing...");
        network.PushInputValues(network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31));
        var outputs = network.GetOutput();
        Console.WriteLine($"Input: 1; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 2; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 3; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 4; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 5; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 6; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 7; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 8; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 9; Output: {string.Join(',', outputs)}");
​
        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31));
        outputs = network.GetOutput();
        Console.WriteLine($"Input: 0; Output: {string.Join(',', outputs)}");
    }
}
​

View IL Code