Welcome, guest | Sign In | My Account | Store | Cart
```"""
Artificial Neural Network
"""

import numpy
import copy

def sigmoid(x):
return 1.0/(1.0 + numpy.exp(-x))

class NeuralNet:

def __init__(self, numUnitsPerLayer):
"""
Constructor
@param numUnitsPerLayer numbers of units per layer, excluding bias
"""

numLayers = len(numUnitsPerLayer)
if numLayers < 2:
raise RuntimeError, 'ERROR number of layers must be >= 2! (Got %d)' % numLayers

# total number of layers incl. input and output
self.numLayers = numLayers

# activations
self.activations = [ numpy.zeros( (numUnitsPerLayer[i] + 1,), numpy.float64 ) for i in range(numLayers)]

# set the biases
for el in range(numLayers):
self.activations[el][0] = 1.0

# weights
self.weights = []
self.bigDeltas = []
for el in range(numLayers - 1):
shp = (numUnitsPerLayer[el+1], numUnitsPerLayer[el] + 1)
self.weights.append( numpy.zeros(shp, numpy.float64) )
self.bigDeltas.append( numpy.zeros(shp, numpy.float64) )

# back propagating errors, no error for layer 0
self.deltas = [ numpy.zeros( (len(a)-1,), numpy.float64 ) for a in self.activations]
self.deltas[0][:] = 0.0 # by definition

def randomlyInitializeWeights(self, magnitude = 0.1):
"""
Randomly initialize the weights to values between -magnitude ... +magnitude
@param magnitude
"""
numpy.random.seed(1234)
for w in self.weights:
w[:] = magnitude * (numpy.random.rand(w.shape[0], w.shape[1]) - 0.5)

def forward(self, inputData):
"""
Compute activation by propagating input forward
@param inputData input (excl. bias)
"""
self.activations[0][1:] = inputData # copy input data
for el in range(1, self.numLayers):
z = numpy.dot(self.weights[el-1], self.activations[el-1])
self.activations[el][1:] = sigmoid(z)
self.activations[el][0] = 1.0

return self.getOutput()

def backward(self, targetOutputData):
"""
Propagate error backward
@param targetOutputData target output data
"""
weightsTranspose = [ numpy.transpose(w) for w in self.weights ]
self.deltas[self.numLayers - 1][:] = self.activations[-1][1:] - targetOutputData
for el in range(self.numLayers - 2, 0, -1):
a = self.activations[el]
gprime = a*(1.0 - a)
d = numpy.dot(weightsTranspose[el], self.deltas[el + 1]) * a*(1.0 - a)
self.deltas[el][:] = d[1:]

def getOutput(self):
"""
Get the network output (excl. bias)
@return array
"""
return self.activations[-1][1:]

def getInput(self):
"""
Get the network input (excl. bias)
@return array
"""
return self.activations[0][1:]

def getCost(self, inputOutputList, lam=0.0):
"""
Compute cost function associated with input/output training data
@param inputOutputList list of [(input, output), ...] values
@param lam >= 0 regularization parameter (lam = 0 means no regularization)
"""
res = 0.0

# standard term
for x, y in inputOutputList:

# output from inputs and weights
out = self.forward(x)

# error
res -= numpy.sum( y*numpy.log(out) + (1.0-y)*numpy.log(1.0-out) )

# regularization term
for w in self.weights:
res += (lam/2.0) * numpy.sum(w[:, 1:]**2)

res /= float(len(inputOutputList))

return res

def train(self, inputOutputList, lam=0.0, alpha=1.0):
"""
Update the weights using training set
@param inputOutputList list of [(input, output), ...] values
@param lam >= 0 regularization parameter (lam = 0 means no regularization)
@param alpha > 0 gradient descent step
@return cost before completion of step, cost after completion of step
"""

numTraining = len(inputOutputList)

# accumulate the error
for x, y in inputOutputList:

# compute the activations at each level
self.forward(x)

# compute the errors at each level
self.backward(y)

for el in range(self.numLayers-1):
# d J /d Theta
self.bigDeltas[el] += numpy.outer(self.deltas[el+1], self.activations[el])

cost = self.getCost(inputOutputList, lam)

# update the weights across all layers
for el in range(self.numLayers-1):
self.weights[el][:, :] -= alpha*self.bigDeltas[el][:, :] / numTraining
# regularization term
self.weights[el][:, 1:] -= alpha*lam*self.weights[el][:, 1:]

newCost = self.getCost(inputOutputList, lam)

return cost, newCost
```