# Simple Neural Networks in Python¶

The Python Quants GmbH

In [1]:
import numpy as np
from pylab import plt
plt.style.use('seaborn')
%matplotlib inline


## Neural Network for Regression¶

### Linear OLS Regression¶

In [2]:
x = np.linspace(0, 10, 5)
y = 3 * x + 2.5 + np.random.standard_normal(len(x)) * 2.5

In [3]:
plt.plot(x, y, 'ro')

Out[3]:
[<matplotlib.lines.Line2D at 0x118699898>]
In [4]:
reg = np.polyfit(x, y, deg=1)

In [5]:
reg

Out[5]:
array([ 2.70872676,  5.2841577 ])
In [6]:
yr = np.polyval(reg, x)
plt.plot(x, y, 'ro')
plt.plot(x, yr, 'b')

Out[6]:
[<matplotlib.lines.Line2D at 0x118777860>]
In [7]:
((y - yr) ** 2).mean()

Out[7]:
3.546067894358238

### Network Training — Single Step

In [8]:
l0 = np.array((x, len(x) * [1])).T

In [9]:
l0

Out[9]:
array([[  0. ,   1. ],
[  2.5,   1. ],
[  5. ,   1. ],
[  7.5,   1. ],
[ 10. ,   1. ]])
In [10]:
weights = np.array(((2., 2.)))

In [11]:
l1 = np.dot(l0, weights)
l1

Out[11]:
array([  2.,   7.,  12.,  17.,  22.])
In [12]:
y

Out[12]:
array([  7.24892495,   8.84079526,  18.82127318,  27.39911404,  31.82885007])
In [13]:
d = y - l1
d

Out[13]:
array([  5.24892495,   1.84079526,   6.82127318,  10.39911404,   9.82885007])
In [14]:
(d ** 2).mean()  # MSE

Out[14]:
56.44347496461436
In [15]:
alpha = 0.01  # learning rate

In [16]:
update = alpha * np.dot(d, l0)
update

Out[16]:
array([ 2.1499021 ,  0.34138958])
In [17]:
weights += update  # updating weights

In [18]:
weights

Out[18]:
array([ 4.1499021 ,  2.34138958])
In [19]:
l1 = np.dot(l0, weights)
d = y - l1
(d ** 2).mean()  # new MSE

Out[19]:
47.682492163492263

### Network Training — Multi Step

In [20]:
weights = np.array(((1., 100.)))

In [33]:
for _ in range(51):
# layer 1
l1 = np.dot(l0, weights)

# deltas of layer 1
d = y - l1

# print MSE
if _ % 5 == 0:
print('MSE after %4d iterations: %6.2f' % (_, (d ** 2).mean()))

# update weights based on deltas
weights += alpha * np.dot(d, l0)

MSE after    0 iterations:   3.55
MSE after    5 iterations:   3.55
MSE after   10 iterations:   3.55
MSE after   15 iterations:   3.55
MSE after   20 iterations:   3.55
MSE after   25 iterations:   3.55
MSE after   30 iterations:   3.55
MSE after   35 iterations:   3.55
MSE after   40 iterations:   3.55
MSE after   45 iterations:   3.55
MSE after   50 iterations:   3.55

In [34]:
yr = np.polyval(reg, x)
plt.plot(x, y, 'ro')
plt.plot(x, yr, 'b')
plt.plot(x, l1, 'm--')

Out[34]:
[<matplotlib.lines.Line2D at 0x118c2ef60>]

## Neural Network for Classification¶

### Sigmoid Function¶

In [35]:
# sigmoid function
def sigmoid(x, deriv=False):
if deriv == True:
return sigmoid(x) * (1 - sigmoid(x))
return 1 / (1 + np.exp(-x))

In [36]:
x = np.linspace(-10, 10, 250)
y = sigmoid(x)
d = sigmoid(x, deriv=True)

In [37]:
s = np.where(x > 0, 1, 0)

In [38]:
fig, ax = plt.subplots(2, sharex=True, figsize=(10, 8))
ax[0].plot(x, y, 'b')
ax[0].plot(x, s, 'm--')

ax[1].plot(x, d, 'g');


### The Data¶

In [39]:
# input dataset (features)
# layer 0
l0 = np.array([[0, 0, 1],
[0, 1, 1],
[1, 0, 1],
[1, 1, 1] ])

In [40]:
# output dataset (labels)
y = np.array([[0,
0,
1,
1]]).T


### Single Step¶

In [41]:
# initialize weights randomly with mean 0
np.random.seed(1)
weights = 2 * np.random.random((3, 1)) - 1
weights

Out[41]:
array([[-0.16595599],
[ 0.44064899],
[-0.99977125]])
In [42]:
np.dot(l0, weights)

Out[42]:
array([[-0.99977125],
[-0.55912226],
[-1.16572724],
[-0.72507825]])
In [43]:
l1 = sigmoid(np.dot(l0, weights))
l1

Out[43]:
array([[ 0.2689864 ],
[ 0.36375058],
[ 0.23762817],
[ 0.3262757 ]])
In [44]:
e = y - l1
e

Out[44]:
array([[-0.2689864 ],
[-0.36375058],
[ 0.76237183],
[ 0.6737243 ]])
In [45]:
(e ** 2).mean()

Out[45]:
0.30994584990928159
In [46]:
sigmoid(l1, True)

Out[46]:
array([[ 0.24553187],
[ 0.24190935],
[ 0.24650375],
[ 0.24346281]])
In [47]:
d = e * sigmoid(l1, True)
d

Out[47]:
array([[-0.06604473],
[-0.08799467],
[ 0.18792752],
[ 0.16402681]])
In [48]:
u = np.dot(l0.T, d)
u

Out[48]:
array([[ 0.35195432],
[ 0.07603214],
[ 0.19791493]])
In [49]:
weights += u
weights

Out[49]:
array([[ 0.18599833],
[ 0.51668113],
[-0.80185633]])
In [50]:
l1 = sigmoid(np.dot(l0, weights))
e = y - l1
(e ** 2).mean()

Out[50]:
0.24425422705654065

### Multiple Steps¶

In [51]:
# initialize weights randomly with mean 0
np.random.seed(1)
weights = 2 * np.random.random((3, 1)) - 1
weights

Out[51]:
array([[-0.16595599],
[ 0.44064899],
[-0.99977125]])
In [52]:
for _ in range(1001):
# forward propagation
# layer 1
l1 = sigmoid(np.dot(l0, weights))

# errors of layer 1
e = y - l1
if _ % 200 == 0:
print('\nafter %d iterations' % _)
print('layer 1:', l1.T)
print('errors: ', e.T)
print('MSE:    ', (e ** 2).mean())

# multiply errors by the slope of the
# sigmoid at the values in l1
d = e * sigmoid(l1, True)

# update weights
weights += np.dot(l0.T, d)

after 0 iterations
layer 1: [[ 0.2689864   0.36375058  0.23762817  0.3262757 ]]
errors:  [[-0.2689864  -0.36375058  0.76237183  0.6737243 ]]
MSE:     0.309945849909

after 200 iterations
layer 1: [[ 0.03581881  0.02486184  0.97910131  0.96983694]]
errors:  [[-0.03581881 -0.02486184  0.02089869  0.03016306]]
MSE:     0.000811915861218

after 400 iterations
layer 1: [[ 0.01812805  0.01228468  0.98963848  0.98469571]]
errors:  [[-0.01812805 -0.01228468  0.01036152  0.01530429]]
MSE:     0.000205280470873

after 600 iterations
layer 1: [[ 0.01210241  0.00814395  0.99312159  0.98977191]]
errors:  [[-0.01210241 -0.00814395  0.00687841  0.01022809]]
MSE:     9.11796267574e-05

after 800 iterations
layer 1: [[ 0.00907573  0.00608783  0.99485433  0.99232527]]
errors:  [[-0.00907573 -0.00608783  0.00514567  0.00767473]]
MSE:     5.1202496537e-05

after 1000 iterations
layer 1: [[ 0.00725744  0.00485959  0.99589051  0.9938605 ]]
errors:  [[-0.00725744 -0.00485959  0.00410949  0.0061395 ]]
MSE:     3.27168767611e-05