Java实现的简单神经网络(基于Sigmoid激活函数)

主体代码

NeuronNetwork.java

package com.rockbb.math.nnetwork;
 
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
 
public class NeutonNetwork {
    private List<NeuronLayer> layers;
 
    public NeuronNetwork(int[] sizes, double bpFactor, Activator activator) {
        layers = new ArrayList<>(sizes.length - 1);
        int inputSize = sizes[0];
        for (int i = 1; i < sizes.length; i++) {
            NeuronLayer layer = new NeuronLayer(inputSize, sizes[i], activator, bpFactor);
            layers.add(layer);
            inputSize = sizes[i];
        }
        for (int i = 0; i < layers.size() - 1; i++) {
            layers.get(i).setNext(layers.get(i + 1));
        }
    }
 
    public List<NeuronLayer> getLayers() {return layers;}
    public void setLayers(List<NeuronLayer> layers) {this.layers = layers;}
 
    public double getError() {
        return layers.get(layers.size() - 1).getError();
    }
 
    public List<Double> predict(List<Double> inputs) {
        List<Double> middle = inputs;
        for (int i = 0; i < layers.size(); i++) {
            middle = layers.get(i).forward(middle);
        }
        return middle;
    }
 
    public void backward() {
        for (int j= layers.size() - 1; j >=0; j--) {
            layers.get(j).backward();
        }
    }
 
    public void fillTargets(List<Double> targets) {
        layers.get(layers.size() - 1).fillTargets(targets);
    }
 
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        for (int j = 0; j < layers.size(); j++) {
            sb.append(layers.get(j).toString());
        }
        return sb.toString();
    }
 
    public static String listToString(List<Double> list) {
        StringBuilder sb = new StringBuilder();
        for (Double t : list) {
            sb.append(String.format("% 10.8f ", t));
        }
        return sb.toString();
    }
 
    public static void main(String[] args) {
        int[] sz = new int[]{2, 4, 1};
        double[][] trainData = {{0d, 0d},{0d, 1d},{1d, 0d},{1d, 1d}};
        double[][] targetDate = {{0d},{1d},{1d},{0d}};
 
        NeuronNetwork nn = new NeuronNetwork(sz, 0.5d, new SigmoidActivator());
        for (int kk = 0; kk < 20000; kk++) {
            double totalError = 0d;
            for (int i = 0; i < trainData.length; i++) {
                List<Double> inputs = Arrays.asList(trainData[i][0], trainData[i][1]);
                List<Double> targets = Arrays.asList(targetDate[i][0]);
                nn.fillTargets(targets);
                nn.predict(inputs);
                //System.out.print(nn);
                System.out.println(String.format("kk:%5d, i:%d, error: %.8f\n", kk, i, nn.getError()));
                totalError += nn.getError();
                nn.backward();
            }
            System.out.println(String.format("kk:%5d, Total Error: %.8f\n\n", kk, totalError));
            if (totalError < 0.0001) {
                System.out.println(nn);
                break;
            }
        }
        System.out.println(nn);
    }
}

NeuronLayer.java

package com.rockbb.math.nnetwork;
 
import java.util.ArrayList;
import java.util.List;
 
public class NeuronLayer {
    private int inputSize;
    private List<Neuron> neurons;
    private double bias;
    private Activator activator;
    private NeuronLayer next;
    private double bpFactor;
    private List<Double> inputs;
 
    public NeuronLayer(int inputSize, int size, Activator activator, double bpFactor) {
        this.inputSize = inputSize;
        this.activator = activator;
        this.bpFactor = bpFactor;
        this.bias = Math.random() - 0.5;
 
        this.neutrons = new ArrayList<>(size);
        for (int i = 0; i < size; i++) {
            Neuron neuron = new Neuron(this, inputSize);
            neurons.add(neuron);
        }
    }
 
    public int getInputSize() {return inputSize;}
    public void setInputSize(int inputSize) {this.inputSize = inputSize;}
    public List<Neuron> getNeurons() {return neurons;}
    public void setNeurons(List<Neuron> neurons) {this.neurons = neurons;}
    public double getBias() {return bias;}
    public void setBias(double bias) {this.bias = bias;}
    public Activator getActivator() {return activator;}
    public void setActivator(Activator activator) {this.activator = activator;}
    public NeutronLayer getNext() {return next;}
    public void setNext(NeutronLayer next) {this.next = next;}
 
    public List<Double> forward(List<Double> inputs) {
        this.inputs = inputs;
        List<Double> outputs = new ArrayList<Double>(neurons.size());
        for (int i = 0; i < neurons.size(); i++) {
            outputs.add(0d);
        }
        for (int i = 0; i < neurons.size(); i++) {
            double output = neurons.get(i).forward(inputs);
            outputs.set(i, output);
        }
        return outputs;
    }
 
    public void backward() {
        if (this.next == null) {
            // If this is the output layer, calculate delta for each neutron
            double totalDelta = 0d;
            for (int i = 0; i < neurons.size(); i++) {
                Neutron n = neurons.get(i);
                double delta = -(n.getTarget() - n.getOutput()) * activator.backwardDelta(n.getOutput());
                n.setBpDelta(delta);
                totalDelta += delta;
                // Reflect to each weight under this neuron
                for (int j = 0; j < n.getWeights().size(); j++) {
                    n.getWeights().set(j, n.getWeights().get(j) - bpFactor * delta * inputs.get(j));
                }
            }
            // Relfect to bias
            this.bias = this.bias - bpFactor * totalDelta / neutrons.size();
        } else {
            // if this is the hidden layer
            double totalDelta = 0d;
            for (int i = 0; i < neurons.size(); i++) {
                Neuron n = neurons.get(i);
                List<Neuron> downNeurons = next.getNeurons();
                double delta = 0;
                for (int j = 0; j < downNeurons.size(); j++) {
                    delta += downNeurons.get(j).getBpDelta() * downNeurons.get(j).getWeights().get(i);
                }
                delta = delta * activator.backwardDelta(n.getOutput());
                n.setBpDelta(delta);
                totalDelta += delta;
                // Reflect to each weight under this neuron
                for (int j = 0; j < n.getWeights().size(); j++) {
                    n.getWeights().set(j, n.getWeights().get(j) - bpFactor * delta * inputs.get(j));
                }
            }
            // Relfect to bias
            this.bias = this.bias - bpFactor * totalDelta / neutrons.size();
        }
    }
 
    public double getError() {
        double totalError = 0d;
        for (int i = 0; i < neurons.size(); i++) {
            totalError += Math.pow(neurons.get(i).getError(), 2);
        }
        return totalError / (2 * neurons.size());
    }
 
    public void fillTargets(List<Double> targets) {
        for (int i = 0; i < neurons.size(); i++) {
            neurons.get(i).setTarget(targets.get(i));
        }
    }
 
    public double filter(double netInput) {
        return activator.forward(netInput + bias);
    }
 
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append(String.format("Input size: %d, bias: %.8f\n", inputSize, bias));
        for (int i = 0; i < neurons.size(); i++) {
            sb.append(String.format("%3d: %s\n", i, neurons.get(i).toString()));
        }
        return sb.toString();
    }
}

Neuron.java

package com.rockbb.math.nnetwork;
 
import java.util.ArrayList;
import java.util.List;
 
public class Neuron {
    private NeuronLayer layer;
    private List<Double> weights;
    private double output;
    private double target;
    private double bpDelta;
 
    public Neuron(NeuronLayer layer, int inputSize) {
        this.layer = layer;
        this.weights = new ArrayList<>(inputSize);
        for (int i = 0; i < inputSize; i++) {
            // Initialize each weight with value [0.1, 1)
            weights.add(Math.random() * 0.9 + 0.1);
        }
        this.bpDelta = 0d;
    }
 
    public NeuronLayer getLayer() {return layer;}
    public void setLayer(NeuronLayer layer) {this.layer = layer;}
    public List<Double> getWeights() {return weights;}
    public void setWeights(List<Double> weights) {this.weights = weights;}
    public double getOutput() {return output;}
    public void setOutput(double output) {this.output = output;}
    public double getTarget() {return target;}
    public void setTarget(double target) {this.target = target;}
    public double getBpDelta() {return bpDelta;}
    public void setBpDelta(double bpDelta) {this.bpDelta = bpDelta;}
 
    public double calcNetInput(List<Double> inputs) {
        double netOutput = 0f;
        for (int i = 0; i < weights.size(); i++) {
            netOutput += inputs.get(i) * weights.get(i);
        }
        return netOutput;
    }
 
    public double forward(List<Double> inputs) {
        double netInput = calcNetInput(inputs);
        this.output = layer.filter(netInput);
        return this.output;
    }
 
    public double getError() {
        return target - output;
    }
 
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append(String.format("O:% 10.8f T:% 10.8f D:% 10.8f w:{", output, target, bpDelta));
        for (int i = 0; i < weights.size(); i++) {
            sb.append(String.format("% 10.8f ", weights.get(i)));
        }
        sb.append('}');
        return sb.toString();
    }
}

激活函数

Activator.java

package com.rockbb.math.nnetwork;
 
public interface Activator {
 
    double forward(double input);
 
    double backwardDelta(double output);
}

SigmoidActivator.java

package com.rockbb.math.nnetwork;
 
public class SigmoidActivator implements Activator {
 
    public double forward(double input) {
        return 1 / (1 + Math.exp(-input));
    }
 
    public double backwardDelta(double output) {
        return output * (1 - output);
    }
}

在同样的训练数据和误差目标下, 比 http://www.emergentmind.com/neural-network 使用更少的训练次数.

使用Sigmoid激活函数工作正常.

使用ReLu激活函数时总会使某个Neuron冻结, 不能收敛, 待检查