229 lines
8.1 KiB
Python
Executable File
229 lines
8.1 KiB
Python
Executable File
import sys
|
|
import numpy as np
|
|
from matplotlib import pyplot
|
|
|
|
sys.path.append('..')
|
|
from submission import SubmissionBase
|
|
|
|
|
|
def displayData(X, example_width=None, figsize=(10, 10)):
|
|
"""
|
|
Displays 2D data stored in X in a nice grid.
|
|
"""
|
|
# Compute rows, cols
|
|
if X.ndim == 2:
|
|
m, n = X.shape
|
|
elif X.ndim == 1:
|
|
n = X.size
|
|
m = 1
|
|
X = X[None] # Promote to a 2 dimensional array
|
|
else:
|
|
raise IndexError('Input X should be 1 or 2 dimensional.')
|
|
|
|
example_width = example_width or int(np.round(np.sqrt(n)))
|
|
example_height = n / example_width
|
|
|
|
# Compute number of items to display
|
|
display_rows = int(np.floor(np.sqrt(m)))
|
|
display_cols = int(np.ceil(m / display_rows))
|
|
|
|
fig, ax_array = pyplot.subplots(display_rows, display_cols, figsize=figsize)
|
|
fig.subplots_adjust(wspace=0.025, hspace=0.025)
|
|
|
|
ax_array = [ax_array] if m == 1 else ax_array.ravel()
|
|
|
|
for i, ax in enumerate(ax_array):
|
|
# Display Image
|
|
h = ax.imshow(X[i].reshape(example_width, example_width, order='F'),
|
|
cmap='Greys', extent=[0, 1, 0, 1])
|
|
ax.axis('off')
|
|
|
|
|
|
def predict(Theta1, Theta2, X):
|
|
"""
|
|
Predict the label of an input given a trained neural network
|
|
Outputs the predicted label of X given the trained weights of a neural
|
|
network(Theta1, Theta2)
|
|
"""
|
|
# Useful values
|
|
m = X.shape[0]
|
|
num_labels = Theta2.shape[0]
|
|
|
|
# You need to return the following variables correctly
|
|
p = np.zeros(m)
|
|
h1 = sigmoid(np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), Theta1.T))
|
|
h2 = sigmoid(np.dot(np.concatenate([np.ones((m, 1)), h1], axis=1), Theta2.T))
|
|
p = np.argmax(h2, axis=1)
|
|
return p
|
|
|
|
|
|
def debugInitializeWeights(fan_out, fan_in):
|
|
"""
|
|
Initialize the weights of a layer with fan_in incoming connections and fan_out outgoings
|
|
connections using a fixed strategy. This will help you later in debugging.
|
|
|
|
Note that W should be set a matrix of size (1+fan_in, fan_out) as the first row of W handles
|
|
the "bias" terms.
|
|
|
|
Parameters
|
|
----------
|
|
fan_out : int
|
|
The number of outgoing connections.
|
|
|
|
fan_in : int
|
|
The number of incoming connections.
|
|
|
|
Returns
|
|
-------
|
|
W : array_like (1+fan_in, fan_out)
|
|
The initialized weights array given the dimensions.
|
|
"""
|
|
# Initialize W using "sin". This ensures that W is always of the same values and will be
|
|
# useful for debugging
|
|
W = np.sin(np.arange(1, 1 + (1+fan_in)*fan_out))/10.0
|
|
W = W.reshape(fan_out, 1+fan_in, order='F')
|
|
return W
|
|
|
|
|
|
def computeNumericalGradient(J, theta, e=1e-4):
|
|
"""
|
|
Computes the gradient using "finite differences" and gives us a numerical estimate of the
|
|
gradient.
|
|
|
|
Parameters
|
|
----------
|
|
J : func
|
|
The cost function which will be used to estimate its numerical gradient.
|
|
|
|
theta : array_like
|
|
The one dimensional unrolled network parameters. The numerical gradient is computed at
|
|
those given parameters.
|
|
|
|
e : float (optional)
|
|
The value to use for epsilon for computing the finite difference.
|
|
|
|
Notes
|
|
-----
|
|
The following code implements numerical gradient checking, and
|
|
returns the numerical gradient. It sets `numgrad[i]` to (a numerical
|
|
approximation of) the partial derivative of J with respect to the
|
|
i-th input argument, evaluated at theta. (i.e., `numgrad[i]` should
|
|
be the (approximately) the partial derivative of J with respect
|
|
to theta[i].)
|
|
"""
|
|
numgrad = np.zeros(theta.shape)
|
|
perturb = np.diag(e * np.ones(theta.shape))
|
|
for i in range(theta.size):
|
|
loss1, _ = J(theta - perturb[:, i])
|
|
loss2, _ = J(theta + perturb[:, i])
|
|
numgrad[i] = (loss2 - loss1)/(2*e)
|
|
return numgrad
|
|
|
|
|
|
def checkNNGradients(nnCostFunction, lambda_=0):
|
|
"""
|
|
Creates a small neural network to check the backpropagation gradients. It will output the
|
|
analytical gradients produced by your backprop code and the numerical gradients
|
|
(computed using computeNumericalGradient). These two gradient computations should result in
|
|
very similar values.
|
|
|
|
Parameters
|
|
----------
|
|
nnCostFunction : func
|
|
A reference to the cost function implemented by the student.
|
|
|
|
lambda_ : float (optional)
|
|
The regularization parameter value.
|
|
"""
|
|
input_layer_size = 3
|
|
hidden_layer_size = 5
|
|
num_labels = 3
|
|
m = 5
|
|
|
|
# We generate some 'random' test data
|
|
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
|
|
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
|
|
|
|
# Reusing debugInitializeWeights to generate X
|
|
X = debugInitializeWeights(m, input_layer_size - 1)
|
|
y = np.arange(1, 1+m) % num_labels
|
|
# print(y)
|
|
# Unroll parameters
|
|
nn_params = np.concatenate([Theta1.ravel(), Theta2.ravel()])
|
|
|
|
# short hand for cost function
|
|
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
|
|
num_labels, X, y, lambda_)
|
|
cost, grad = costFunc(nn_params)
|
|
numgrad = computeNumericalGradient(costFunc, nn_params)
|
|
|
|
# Visually examine the two gradient computations.The two columns you get should be very similar.
|
|
print(np.stack([numgrad, grad], axis=1))
|
|
print('The above two columns you get should be very similar.')
|
|
print('(Left-Your Numerical Gradient, Right-Analytical Gradient)\n')
|
|
|
|
# Evaluate the norm of the difference between two the solutions. If you have a correct
|
|
# implementation, and assuming you used e = 0.0001 in computeNumericalGradient, then diff
|
|
# should be less than 1e-9.
|
|
diff = np.linalg.norm(numgrad - grad)/np.linalg.norm(numgrad + grad)
|
|
|
|
print('If your backpropagation implementation is correct, then \n'
|
|
'the relative difference will be small (less than 1e-9). \n'
|
|
'Relative Difference: %g' % diff)
|
|
|
|
|
|
def sigmoid(z):
|
|
"""
|
|
Computes the sigmoid of z.
|
|
"""
|
|
return 1.0 / (1.0 + np.exp(-z))
|
|
|
|
|
|
class Grader(SubmissionBase):
|
|
X = np.reshape(3 * np.sin(np.arange(1, 31)), (3, 10), order='F')
|
|
Xm = np.reshape(np.sin(np.arange(1, 33)), (16, 2), order='F') / 5
|
|
ym = np.arange(1, 17) % 4
|
|
t1 = np.sin(np.reshape(np.arange(1, 25, 2), (4, 3), order='F'))
|
|
t2 = np.cos(np.reshape(np.arange(1, 41, 2), (4, 5), order='F'))
|
|
t = np.concatenate([t1.ravel(), t2.ravel()], axis=0)
|
|
|
|
def __init__(self):
|
|
part_names = ['Feedforward and Cost Function',
|
|
'Regularized Cost Function',
|
|
'Sigmoid Gradient',
|
|
'Neural Network Gradient (Backpropagation)',
|
|
'Regularized Gradient']
|
|
part_names_key = ['aAiP2', '8ajiz', 'rXsEO', 'TvZch', 'pfIYT']
|
|
assignment_key = 'xolSVXukR72JH37bfzo0pg'
|
|
super().__init__('neural-network-learning', assignment_key, part_names, part_names_key)
|
|
|
|
def __iter__(self):
|
|
for part_id in range(1, 6):
|
|
try:
|
|
func = self.functions[part_id]
|
|
|
|
# Each part has different expected arguments/different function
|
|
if part_id == 1:
|
|
res = func(self.t, 2, 4, 4, self.Xm, self.ym, 0)[0]
|
|
elif part_id == 2:
|
|
res = func(self.t, 2, 4, 4, self.Xm, self.ym, 1.5)
|
|
elif part_id == 3:
|
|
res = func(self.X, )
|
|
elif part_id == 4:
|
|
J, grad = func(self.t, 2, 4, 4, self.Xm, self.ym, 0)
|
|
grad1 = np.reshape(grad[:12], (4, 3))
|
|
grad2 = np.reshape(grad[12:], (4, 5))
|
|
grad = np.concatenate([grad1.ravel('F'), grad2.ravel('F')])
|
|
res = np.hstack([J, grad]).tolist()
|
|
elif part_id == 5:
|
|
J, grad = func(self.t, 2, 4, 4, self.Xm, self.ym, 1.5)
|
|
grad1 = np.reshape(grad[:12], (4, 3))
|
|
grad2 = np.reshape(grad[12:], (4, 5))
|
|
grad = np.concatenate([grad1.ravel('F'), grad2.ravel('F')])
|
|
res = np.hstack([J, grad]).tolist()
|
|
else:
|
|
raise KeyError
|
|
yield part_id, res
|
|
except KeyError:
|
|
yield part_id, 0
|