// Code copied from this article https://rubikscode.net/2022/07/04/implementing-simple-neural-network-in-c/

// But the original code can be found here https://github.com/NMZivkovic/SimpleNeuralNetworkInCSharp

​

// The original code did not come with a concrete problem to be solved by this neural network.

// Had to come up with one myself and in the course of training had to make modifications to the original code:

// 1. To make sure we have random initial weights assigned, made a static instance of Random object instead of being created multiple times

// 2. Bumped the learning rate to 20 just for this problem that we are trying to solve

// 3. Changed the calculation of delta in HandleOutputLayer() and HandleHiddenLayers() to not multiply by -1 as it will not work otherwise

​

// So the problem I came up with for our neural network to solve is to recognize the numbers 0-9 in an image

// Refer to this blogpost for a more detailed explanation: https://rodansotto.github.io/tech-blog/2022/08/07/how-i-trained-my-csharp-neural-network.html

​

using System;

using System.Collections.Generic;

using System.Linq;

​

public interface ISynapse

{

    double Weight { get; set; }

    double PreviousWeight { get; set; }

    double GetOutput();

​

    bool IsFromNeuron(Guid fromNeuronId);

    void UpdateWeight(double learningRate, double delta);

}

​

// this will be our synapse, the one connecting neurons

public class Synapse : ISynapse

{

    internal INeuron _fromNeuron;

    internal INeuron _toNeuron;

​

    /// <summary>

    /// Weight of the connection.

    /// </summary>

    public double Weight { get; set; }

​

    /// <summary>

    /// Weight that connection had in previous itteration.

    /// Used in training process.

    /// </summary>

    public double PreviousWeight { get; set; }

​

    public Synapse(INeuron fromNeuraon, INeuron toNeuron, double weight)

    {

        _fromNeuron = fromNeuraon;

        _toNeuron = toNeuron;

​

        Weight = weight;

        PreviousWeight = 0;

    }

​

    public static Random rand = new Random();

​

    public Synapse(INeuron fromNeuraon, INeuron toNeuron)

    {

        _fromNeuron = fromNeuraon;

        _toNeuron = toNeuron;

​

        // var tmpRandom = new Random();

        // Weight = tmpRandom.NextDouble();

        Weight = rand.NextDouble();

        PreviousWeight = 0;

    }

​

    /// <summary>

    /// Get output value of the connection.

    /// </summary>

    /// <returns>

    /// Output value of the connection.

    /// </returns>

    public double GetOutput()

    {

        return _fromNeuron.CalculateOutput();

    }

​

    /// <summary>

    /// Checks if Neuron has a certain number as an input neuron.

    /// </summary>

    /// <param name="fromNeuronId">Neuron Id.</param>

    /// <returns>

    /// True - if the neuron is the input of the connection.

    /// False - if the neuron is not the input of the connection. 

    /// </returns>

    public bool IsFromNeuron(Guid fromNeuronId)

    {

        return _fromNeuron.Id.Equals(fromNeuronId);

    }

​

    /// <summary>

    /// Update weight.

    /// </summary>

    /// <param name="learningRate">Chossen learning rate.</param>

    /// <param name="delta">Calculated difference for which weight of the connection needs to be modified.</param>

    public void UpdateWeight(double learningRate, double delta)

    {

        PreviousWeight = Weight;

        Weight += learningRate * delta;

    }

}

​

// these are special connections that are used just for the input layer of the neuron,

// i.e. they are used only for adding input to the entirety of the system

// their weights are always 1 and not updated during the training process

public class InputSynapse : ISynapse

{

    internal INeuron _toNeuron;

​

    public double Weight { get; set; }

    public double Output { get; set; }

    public double PreviousWeight { get; set; }

​

    public InputSynapse(INeuron toNeuron)

    {

        _toNeuron = toNeuron;

        Weight = 1;

    }

​

    public InputSynapse(INeuron toNeuron, double output)

    {

        _toNeuron = toNeuron;

        Output = output;

        Weight = 1;

        PreviousWeight = 1;

    }

​

    public double GetOutput()

    {

        return Output;

    }

​

    public bool IsFromNeuron(Guid fromNeuronId)

    {

        return false;

    }

​

    public void UpdateWeight(double learningRate, double delta)

    {

        throw new InvalidOperationException("It is not allowed to call this method on Input Connecion");

    }

}

​

public interface IInputFunction

{

    double CalculateInput(List<ISynapse> inputs);

}

​

// this will be our input function

// as biological neurons have multiple inputs and so the same here

// this input function is usually a weighted sum function, that is input from weighted connections is summarized

public class WeightedSumFunction : IInputFunction

{

    public double CalculateInput(List<ISynapse> inputs)

    {

        return inputs.Select(x => x.Weight * x.GetOutput()).Sum();

    }

}

​

public interface IActivationFunction

{

    double CalculateOutput(double input);

}

​

// this will be our output function, the activation function

// you actually have to use the right activation function here to simulate the human biological neural network

// there have been already activation functions developed and some of them are explained here: https://rubikscode.net/2017/11/20/common-neural-network-activation-functions/

// here we are using the Step function, also called as Perceptron, first used back in 1957

public class StepActivationFunction : IActivationFunction

{

    private double _treshold;

​

    public StepActivationFunction(double treshold)

    {

        _treshold = treshold;

    }

​

    public double CalculateOutput(double input)

    {

        return Convert.ToDouble(input > _treshold);

    }

}

​

// Sigmoid activation function implementation

public class SigmoidActivationFunction : IActivationFunction

{

    private double _coeficient;

​

    public SigmoidActivationFunction(double coeficient)

    {

        _coeficient = coeficient;

    }

​

    public double CalculateOutput(double input)

    {

        return (1 / (1 + Math.Exp(-input * _coeficient)));

    }

}

​

// Rectifier activation function implementation

public class RectifiedActivationFuncion : IActivationFunction

{

    public double CalculateOutput(double input)

    {

        return Math.Max(0, input);

    }

}

​

public interface INeuron

{

    Guid Id { get; }

    double PreviousPartialDerivate { get; set; }

​

    List<ISynapse> Inputs { get; set; }

    List<ISynapse> Outputs { get; set; }

​

    void AddInputNeuron(INeuron inputNeuron);

    void AddOutputNeuron(INeuron inputNeuron);

    double CalculateOutput();

​

    void AddInputSynapse(double inputValue);

    void PushValueOnInput(double inputValue);

}

​

// this will be our neuron

public class Neuron : INeuron

{

    private IActivationFunction _activationFunction;

    private IInputFunction _inputFunction;

​

    /// <summary>

    /// Input connections of the neuron.

    /// </summary>

    public List<ISynapse> Inputs { get; set; }

​

    /// <summary>

    /// Output connections of the neuron.

    /// </summary>

    public List<ISynapse> Outputs { get; set; }

​

    public Guid Id { get; private set; }

​

    /// <summary>

    /// Calculated partial derivate in previous iteration of training process.

    /// </summary>

    public double PreviousPartialDerivate { get; set; }

​

    public Neuron(IActivationFunction activationFunction, IInputFunction inputFunction)

    {

        Id = Guid.NewGuid();

        Inputs = new List<ISynapse>();

        Outputs = new List<ISynapse>();

​

        _activationFunction = activationFunction;

        _inputFunction = inputFunction;

    }

​

    /// <summary>

    /// Connect two neurons. 

    /// This neuron is the output neuron of the connection.

    /// </summary>

    /// <param name="inputNeuron">Neuron that will be input neuron of the newly created connection.</param>

    public void AddInputNeuron(INeuron inputNeuron)

    {

        var synapse = new Synapse(inputNeuron, this);

        Inputs.Add(synapse);

        inputNeuron.Outputs.Add(synapse);

    }

​

    /// <summary>

    /// Connect two neurons. 

    /// This neuron is the input neuron of the connection.

    /// </summary>

    /// <param name="outputNeuron">Neuron that will be output neuron of the newly created connection.</param>

    public void AddOutputNeuron(INeuron outputNeuron)

    {

        var synapse = new Synapse(this, outputNeuron);

        Outputs.Add(synapse);

        outputNeuron.Inputs.Add(synapse);

    }

​

    /// <summary>

    /// Calculate output value of the neuron.

    /// </summary>

    /// <returns>

    /// Output of the neuron.

    /// </returns>

    public double CalculateOutput()

    {

        return _activationFunction.CalculateOutput(_inputFunction.CalculateInput(this.Inputs));

    }

​

    /// <summary>

    /// Input Layer neurons just receive input values.

    /// For this they need to have connections.

    /// This function adds this kind of connection to the neuron.

    /// </summary>

    /// <param name="inputValue">

    /// Initial value that will be "pushed" as an input to connection.

    /// </param>

    public void AddInputSynapse(double inputValue)

    {

        var inputSynapse = new InputSynapse(this, inputValue);

        Inputs.Add(inputSynapse);

    }

​

    /// <summary>

    /// Sets new value on the input connections.

    /// </summary>

    /// <param name="inputValue">

    /// New value that will be "pushed" as an input to connection.

    /// </param>

    public void PushValueOnInput(double inputValue)

    {

        ((InputSynapse)Inputs.First()).Output = inputValue;

    }

}

​

// this will be the neural layer

// it contains the list of neurons used in the layer and the ConnectLayers method, which is used to glue two layers together.

public class NeuralLayer

{

    public List<INeuron> Neurons;

​

    public NeuralLayer()

    {

        Neurons = new List<INeuron>();

    }

​

    /// <summary>

    /// Connecting two layers.

    /// </summary>

    public void ConnectLayers(NeuralLayer inputLayer)

    {

        var combos = Neurons.SelectMany(neuron => inputLayer.Neurons, (neuron, input) => new { neuron, input });

        combos.ToList().ForEach(x => x.neuron.AddInputNeuron(x.input));

    }

}

​

/// <summary>

/// Factory used to create layers.

/// </summary>

public class NeuralLayerFactory

{

    public NeuralLayer CreateNeuralLayer(int numberOfNeurons, IActivationFunction activationFunction, IInputFunction inputFunction)

    {

        var layer = new NeuralLayer();

​

        for (int i = 0; i < numberOfNeurons; i++)

        {

            var neuron = new Neuron(activationFunction, inputFunction);

            layer.Neurons.Add(neuron);

        }

​

        return layer;

    }

}

​

// This will be the implementation of the simple neural network

// During the construction of the object, the initial input layer is added to the network.

// Other layers are added through the function AddLayer, which adds a passed layer on top of the current layer list.

// The GetOutput method will activate the output layer of the network, thus initiating a chain reaction through the network.

// The most important method of this class is the Train method. It receives the training set and the number of epochs. 

// For each epoch, it runs the whole training set through the network.

// Then, the output is compared with desired output and the functions HandleOutputLayer and HandleHiddenLayer are called. These functions implement the backpropagation algorithm.  

public class SimpleNeuralNetwork

{

    private NeuralLayerFactory _layerFactory;

​

    internal List<NeuralLayer> _layers;

    internal double _learningRate;

    internal double[][] _expectedResult;

​

    /// <summary>

    /// Constructor of the Neural Network.

    /// Note:

    /// Initialy input layer with defined number of inputs will be created.

    /// </summary>

    /// <param name="numberOfInputNeurons">

    /// Number of neurons in input layer.

    /// </param>

    public SimpleNeuralNetwork(int numberOfInputNeurons)

    {

        _layers = new List<NeuralLayer>();

        _layerFactory = new NeuralLayerFactory();

​

        // Create input layer that will collect inputs.

        CreateInputLayer(numberOfInputNeurons);

​

        _learningRate = 20; //2.95;

    }

​

    /// <summary>

    /// Add layer to the neural network.

    /// Layer will automatically be added as the output layer to the last layer in the neural network.

    /// </summary>

    public void AddLayer(NeuralLayer newLayer)

    {

        if (_layers.Any())

        {

            var lastLayer = _layers.Last();

            newLayer.ConnectLayers(lastLayer);

        }

​

        _layers.Add(newLayer);

    }

​

    /// <summary>

    /// Push input values to the neural network.

    /// </summary>

    public void PushInputValues(double[] inputs)

    {

        _layers.First().Neurons.ForEach(x => x.PushValueOnInput(inputs[_layers.First().Neurons.IndexOf(x)]));

    }

​

    /// <summary>

    /// Set expected values for the outputs.

    /// </summary>

    public void PushExpectedValues(double[][] expectedOutputs)

    {

        _expectedResult = expectedOutputs;

    }

​

    /// <summary>

    /// Calculate output of the neural network.

    /// </summary>

    /// <returns></returns>

    public List<double> GetOutput()

    {

        var returnValue = new List<double>();

​

        _layers.Last().Neurons.ForEach(neuron =>

        {

             returnValue.Add(neuron.CalculateOutput());

        });

​

        return returnValue;

    }

​

    /// <summary>

    /// Train neural network.

    /// </summary>

    /// <param name="inputs">Input values.</param>

    /// <param name="numberOfEpochs">Number of epochs.</param>

    public void Train(double[][] inputs, int numberOfEpochs)

    {

        double totalError = 0;

​

        for(int i = 0; i < numberOfEpochs; i++)

        {

            for(int j = 0; j < inputs.GetLength(0); j ++)

            {

                PushInputValues(inputs[j]);

​

                var outputs = new List<double>();

​

                // Get outputs.

                _layers.Last().Neurons.ForEach(x =>

                {

                    outputs.Add(x.CalculateOutput());

                });

​

                // Calculate error by summing errors on all output neurons.

                totalError = CalculateTotalError(outputs, j);

                HandleOutputLayer(j);

                HandleHiddenLayers();

            }

        }

    }

​

    /// <summary>

    /// Hellper function that creates input layer of the neural network.

    /// </summary>

    private void CreateInputLayer(int numberOfInputNeurons)

    {

        var inputLayer = _layerFactory.CreateNeuralLayer(numberOfInputNeurons, new RectifiedActivationFuncion(), new WeightedSumFunction());

        inputLayer.Neurons.ForEach(x => x.AddInputSynapse(0));

        this.AddLayer(inputLayer);

    }

​

    /// <summary>

    /// Hellper function that calculates total error of the neural network.

    /// </summary>

    private double CalculateTotalError(List<double> outputs, int row)

    {

        double totalError = 0;

​

        outputs.ForEach(output =>

        {

            var error = Math.Pow(output - _expectedResult[row][outputs.IndexOf(output)], 2);

            totalError += error;

        });

​

        return totalError;

    }

​

    /// <summary>

    /// Hellper function that runs backpropagation algorithm on the output layer of the network.

    /// </summary>

    /// <param name="row">

    /// Input/Expected output row.

    /// </param>

    private void HandleOutputLayer(int row)

    {

        _layers.Last().Neurons.ForEach(neuron =>

        {

            neuron.Inputs.ForEach(connection =>

            {

                var output = neuron.CalculateOutput();

                var netInput = connection.GetOutput();

​

                var expectedOutput = _expectedResult[row][_layers.Last().Neurons.IndexOf(neuron)];

​

                var nodeDelta = (expectedOutput - output) * output * (1 - output);

                //var delta = -1 * netInput * nodeDelta;

                var delta = netInput * nodeDelta;

​

                connection.UpdateWeight(_learningRate, delta);

​

                neuron.PreviousPartialDerivate = nodeDelta;

            });

        });

    }

​

    /// <summary>

    /// Hellper function that runs backpropagation algorithm on the hidden layer of the network.

    /// </summary>

    /// <param name="row">

    /// Input/Expected output row.

    /// </param>

    private void HandleHiddenLayers()

    {

        for (int k = _layers.Count - 2; k > 0; k--)

        {

            _layers[k].Neurons.ForEach(neuron =>

            {

                neuron.Inputs.ForEach(connection =>

                {

                    var output = neuron.CalculateOutput();

                    var netInput = connection.GetOutput();

                    double sumPartial = 0;

​

                    _layers[k + 1].Neurons

                    .ForEach(outputNeuron =>

                    {

                        outputNeuron.Inputs.Where(i => i.IsFromNeuron(neuron.Id))

                        .ToList()

                        .ForEach(outConnection =>

                        {

                            sumPartial += outConnection.PreviousWeight * outputNeuron.PreviousPartialDerivate;

                        });

                    });

​

                    //var delta = -1 * netInput * sumPartial * output * (1 - output);

                    var delta = netInput * sumPartial * output * (1 - output);

                    connection.UpdateWeight(_learningRate, delta);

                });

            });

        }

    }

​

    public double[] DivideDoubleArray(double[] arr, double divideBy)

    {

        for (int i = 0; i < arr.Length; i++)

        {

            arr[i] /= divideBy;

        }

        return arr;

    }

}

​

public class Program

{

    public static void Main()

    {

        // So our input data represents a 5x7 1-bit image that contains a number from 0 to 9

        // To save space, the input data is an integer array of 7 corresponding to the 7 rows of 5 pixels in that 5x7 image

        // Each row contains the decimal value of the 5 pixels in that row

        //  e.g. if row contains these 5 pixels [0 1 1 1 0], then the decimal value will be 14

        // create neural network with 7 neurons in the input layer

        var network = new SimpleNeuralNetwork(7); // <- the input layer will be created here

​

        // add 2 layers, each with a number of neurons and activation and input functions to use

        var layerFactory = new NeuralLayerFactory();

        network.AddLayer(layerFactory.CreateNeuralLayer(4, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the hidden layer

        network.AddLayer(layerFactory.CreateNeuralLayer(1, new SigmoidActivationFunction(1), new WeightedSumFunction())); // <- this is the output layer

​

        // define expected outputs

        // if you notice I'm dividing the values by 10 because apparently this neural network does not work if they are not between 0 and 1 (including 0 and 1)

        network.PushExpectedValues(

            new double[][] {

                network.DivideDoubleArray(new double[] { 1 }, 10),

                network.DivideDoubleArray(new double[] { 2 }, 10),

                network.DivideDoubleArray(new double[] { 3 }, 10),

                network.DivideDoubleArray(new double[] { 4 }, 10),

                network.DivideDoubleArray(new double[] { 5 }, 10),

                network.DivideDoubleArray(new double[] { 6 }, 10),

                network.DivideDoubleArray(new double[] { 7 }, 10),

                network.DivideDoubleArray(new double[] { 8 }, 10),

                network.DivideDoubleArray(new double[] { 9 }, 10),

                network.DivideDoubleArray(new double[] { 0 }, 10),

            });

        // train with the input training set and number of epochs (or times) it will run the training set through the network

        // again I'm dividing the values so they stay within 0 and 1, otherwise it will not work

        // I'm dividing by 31 as the maximum value a row of pixel in our 5x7 image can have is [1 1 1 1 1] = 31 in decimal

        Console.WriteLine("Training...");

        network.Train(

            new double[][] {

                network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31),

                network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31),

                network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31),

                network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31),

                network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31),

            }, 10000);

​

        // and now to test how good the neural network was trained :)

        Console.WriteLine("Testing...");

        network.PushInputValues(network.DivideDoubleArray(new double[] { 4, 12, 4, 4, 4, 4, 4 }, 31));

        var outputs = network.GetOutput();

        Console.WriteLine($"Input: 1; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 2, 4, 8, 15 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 2; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 1, 6, 1, 9, 6 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 3; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 3, 5, 5, 9, 15, 1, 1 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 4; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 8, 8, 14, 1, 1, 14 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 5; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 16, 14, 17, 17, 14 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 6; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 14, 1, 1, 2, 2, 4, 4 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 7; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 6, 9, 9, 6 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 8; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 7, 1, 6 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 9; Output: {string.Join(',', outputs)}");

​

        network.PushInputValues(network.DivideDoubleArray(new double[] { 6, 9, 9, 9, 9, 9, 6 }, 31));

        outputs = network.GetOutput();

        Console.WriteLine($"Input: 0; Output: {string.Join(',', outputs)}");

    }

}

​