Simple Neural Networks in Python

© Dr. Yves J. Hilpisch

The Python Quants GmbH

In [1]:
import numpy as np
from pylab import plt
plt.style.use('seaborn')
%matplotlib inline

Neural Network for Regression

Regression & Neural Networks

Linear OLS Regression

In [2]:
x = np.linspace(0, 10, 5)
y = 3 * x + 2.5 + np.random.standard_normal(len(x)) * 2.5
In [3]:
plt.plot(x, y, 'ro')
Out[3]:
[<matplotlib.lines.Line2D at 0x118699898>]
In [4]:
reg = np.polyfit(x, y, deg=1)
In [5]:
reg
Out[5]:
array([ 2.70872676,  5.2841577 ])
In [6]:
yr = np.polyval(reg, x)
plt.plot(x, y, 'ro')
plt.plot(x, yr, 'b')
Out[6]:
[<matplotlib.lines.Line2D at 0x118777860>]
In [7]:
((y - yr) ** 2).mean()
Out[7]:
3.546067894358238

Network Training — Single Step

In [8]:
l0 = np.array((x, len(x) * [1])).T
In [9]:
l0
Out[9]:
array([[  0. ,   1. ],
       [  2.5,   1. ],
       [  5. ,   1. ],
       [  7.5,   1. ],
       [ 10. ,   1. ]])
In [10]:
weights = np.array(((2., 2.)))
In [11]:
l1 = np.dot(l0, weights)
l1
Out[11]:
array([  2.,   7.,  12.,  17.,  22.])
In [12]:
y
Out[12]:
array([  7.24892495,   8.84079526,  18.82127318,  27.39911404,  31.82885007])
In [13]:
d = y - l1
d
Out[13]:
array([  5.24892495,   1.84079526,   6.82127318,  10.39911404,   9.82885007])
In [14]:
(d ** 2).mean()  # MSE
Out[14]:
56.44347496461436
In [15]:
alpha = 0.01  # learning rate
In [16]:
update = alpha * np.dot(d, l0)
update
Out[16]:
array([ 2.1499021 ,  0.34138958])
In [17]:
weights += update  # updating weights
In [18]:
weights
Out[18]:
array([ 4.1499021 ,  2.34138958])
In [19]:
l1 = np.dot(l0, weights)
d = y - l1
(d ** 2).mean()  # new MSE
Out[19]:
47.682492163492263

Network Training — Multi Step

In [20]:
weights = np.array(((1., 100.)))
In [33]:
for _ in range(51):
    # layer 1
    l1 = np.dot(l0, weights)

    # deltas of layer 1
    d = y - l1
    
    # print MSE
    if _ % 5 == 0:
        print('MSE after %4d iterations: %6.2f' % (_, (d ** 2).mean()))

    # update weights based on deltas
    weights += alpha * np.dot(d, l0)

MSE after    0 iterations:   3.55
MSE after    5 iterations:   3.55
MSE after   10 iterations:   3.55
MSE after   15 iterations:   3.55
MSE after   20 iterations:   3.55
MSE after   25 iterations:   3.55
MSE after   30 iterations:   3.55
MSE after   35 iterations:   3.55
MSE after   40 iterations:   3.55
MSE after   45 iterations:   3.55
MSE after   50 iterations:   3.55
In [34]:
yr = np.polyval(reg, x)
plt.plot(x, y, 'ro')
plt.plot(x, yr, 'b')
plt.plot(x, l1, 'm--')
Out[34]:
[<matplotlib.lines.Line2D at 0x118c2ef60>]

Neural Network for Classification

https://iamtrask.github.io/2015/07/12/basic-python-network/

Sigmoid Function

https://en.wikipedia.org/wiki/Sigmoid_function

In [35]:
# sigmoid function
def sigmoid(x, deriv=False):
    if deriv == True:
        return sigmoid(x) * (1 - sigmoid(x))
    return 1 / (1 + np.exp(-x))
In [36]:
x = np.linspace(-10, 10, 250)
y = sigmoid(x)
d = sigmoid(x, deriv=True)
In [37]:
s = np.where(x > 0, 1, 0)
In [38]:
fig, ax = plt.subplots(2, sharex=True, figsize=(10, 8))
ax[0].plot(x, y, 'b')
ax[0].plot(x, s, 'm--')

ax[1].plot(x, d, 'g');

The Data

In [39]:
# input dataset (features)
# layer 0
l0 = np.array([[0, 0, 1],
               [0, 1, 1],
               [1, 0, 1],
               [1, 1, 1] ])
In [40]:
# output dataset (labels)          
y = np.array([[0,
               0,
               1,
               1]]).T

Single Step

In [41]:
# initialize weights randomly with mean 0
np.random.seed(1)
weights = 2 * np.random.random((3, 1)) - 1
weights
Out[41]:
array([[-0.16595599],
       [ 0.44064899],
       [-0.99977125]])
In [42]:
np.dot(l0, weights)
Out[42]:
array([[-0.99977125],
       [-0.55912226],
       [-1.16572724],
       [-0.72507825]])
In [43]:
l1 = sigmoid(np.dot(l0, weights))
l1
Out[43]:
array([[ 0.2689864 ],
       [ 0.36375058],
       [ 0.23762817],
       [ 0.3262757 ]])
In [44]:
e = y - l1
e
Out[44]:
array([[-0.2689864 ],
       [-0.36375058],
       [ 0.76237183],
       [ 0.6737243 ]])
In [45]:
(e ** 2).mean()
Out[45]:
0.30994584990928159
In [46]:
sigmoid(l1, True)
Out[46]:
array([[ 0.24553187],
       [ 0.24190935],
       [ 0.24650375],
       [ 0.24346281]])
In [47]:
d = e * sigmoid(l1, True)
d
Out[47]:
array([[-0.06604473],
       [-0.08799467],
       [ 0.18792752],
       [ 0.16402681]])
In [48]:
u = np.dot(l0.T, d)
u
Out[48]:
array([[ 0.35195432],
       [ 0.07603214],
       [ 0.19791493]])
In [49]:
weights += u
weights
Out[49]:
array([[ 0.18599833],
       [ 0.51668113],
       [-0.80185633]])
In [50]:
l1 = sigmoid(np.dot(l0, weights))
e = y - l1
(e ** 2).mean()
Out[50]:
0.24425422705654065

Multiple Steps

In [51]:
# initialize weights randomly with mean 0
np.random.seed(1)
weights = 2 * np.random.random((3, 1)) - 1
weights
Out[51]:
array([[-0.16595599],
       [ 0.44064899],
       [-0.99977125]])
In [52]:
for _ in range(1001):
    # forward propagation
    # layer 1
    l1 = sigmoid(np.dot(l0, weights))

    # errors of layer 1
    e = y - l1
    if _ % 200 == 0:
        print('\nafter %d iterations' % _)
        print('layer 1:', l1.T)
        print('errors: ', e.T)
        print('MSE:    ', (e ** 2).mean())

    # multiply errors by the slope of the 
    # sigmoid at the values in l1
    d = e * sigmoid(l1, True)

    # update weights
    weights += np.dot(l0.T, d)

after 0 iterations
layer 1: [[ 0.2689864   0.36375058  0.23762817  0.3262757 ]]
errors:  [[-0.2689864  -0.36375058  0.76237183  0.6737243 ]]
MSE:     0.309945849909

after 200 iterations
layer 1: [[ 0.03581881  0.02486184  0.97910131  0.96983694]]
errors:  [[-0.03581881 -0.02486184  0.02089869  0.03016306]]
MSE:     0.000811915861218

after 400 iterations
layer 1: [[ 0.01812805  0.01228468  0.98963848  0.98469571]]
errors:  [[-0.01812805 -0.01228468  0.01036152  0.01530429]]
MSE:     0.000205280470873

after 600 iterations
layer 1: [[ 0.01210241  0.00814395  0.99312159  0.98977191]]
errors:  [[-0.01210241 -0.00814395  0.00687841  0.01022809]]
MSE:     9.11796267574e-05

after 800 iterations
layer 1: [[ 0.00907573  0.00608783  0.99485433  0.99232527]]
errors:  [[-0.00907573 -0.00608783  0.00514567  0.00767473]]
MSE:     5.1202496537e-05

after 1000 iterations
layer 1: [[ 0.00725744  0.00485959  0.99589051  0.9938605 ]]
errors:  [[-0.00725744 -0.00485959  0.00410949  0.0061395 ]]
MSE:     3.27168767611e-05