In [1]:
import random
import numpy as np
from cs231n.data_utils import load_CIFAR10
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# for auto-reloading extenrnal modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2
def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, num_dev=500):
"""
Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
it for the linear classifier. These are the same steps as we used for the
SVM, but condensed to a single function.
"""
# Load the raw CIFAR-10 data
cifar10_dir = 'cs231n/datasets/cifar-10-batches-py'
# Cleaning up variables to prevent loading data multiple times (which may cause memory issue)
try:
del X_train, y_train
del X_test, y_test
print('Clear previously loaded data.')
except:
pass
X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)
# subsample the data
mask = list(range(num_training, num_training + num_validation))
X_val = X_train[mask]
y_val = y_train[mask]
mask = list(range(num_training))
X_train = X_train[mask]
y_train = y_train[mask]
mask = list(range(num_test))
X_test = X_test[mask]
y_test = y_test[mask]
mask = np.random.choice(num_training, num_dev, replace=False)
X_dev = X_train[mask]
y_dev = y_train[mask]
# Preprocessing: reshape the image data into rows
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))
X_dev = np.reshape(X_dev, (X_dev.shape[0], -1))
# Normalize the data: subtract the mean image
mean_image = np.mean(X_train, axis = 0)
X_train -= mean_image
X_val -= mean_image
X_test -= mean_image
X_dev -= mean_image
# add bias dimension and transform into columns
X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])
X_val = np.hstack([X_val, np.ones((X_val.shape[0], 1))])
X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])
X_dev = np.hstack([X_dev, np.ones((X_dev.shape[0], 1))])
return X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev = get_CIFAR10_data()
print('Train data shape: ', X_train.shape)
print('Train labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
print('dev data shape: ', X_dev.shape)
print('dev labels shape: ', y_dev.shape)
1. Sigmoid_loss and its Derivative¶
2. Implemantation¶
In [2]:
def sigmoid (x):
return 1./(1+np.exp(-x))
In [3]:
from builtins import range
import numpy as np
from random import shuffle
from past.builtins import xrange
def sigmoid_loss(W, X, y, reg):
# Initialize the loss and gradient to zero.
loss = 0.0
dW = np.zeros_like(W)
num_examples=X.shape[0] # X_dev(500, 3073) --> 500
num_class=W.shape[1] # W(3073, 10) --> 10
fs = np.dot(X,W)# 500, 10
ps = sigmoid(fs)
loss = np.sum( .5 * (ps-fs)**2 )
dp = np.copy(ps)
dp[np.arange(num_examples), y] -= 1 # 500, 10
df = ps*(1-ps)*dp # 500, 10
dW += np.dot(X.T, df) #3073 500 x 500 10 => 3073 10
loss /= num_examples
dW /= num_examples
loss += 0.5 * reg * np.sum(W * W)
dW += reg * W
return loss, dW
In [4]:
from cs231n.classifiers.linear_classifier import LinearClassifier
class Sigmoid(LinearClassifier):
""" A subclass that uses the Sigmoid + L2 loss function """
def loss(self, X_batch, y_batch, reg):
return sigmoid_loss(self.W, X_batch, y_batch, reg)
Hyperparameter Tuning with Validation Set!¶
In [5]:
# Use the validation set to tune hyperparameters (regularization strength and
# learning rate). You should experiment with different ranges for the learning
# rates and regularization strengths; if you are careful you should be able to
# get a classification accuracy of over 0.35 on the validation set.
results = {}
best_val = -1
best_sigmoid = None
# Provided as a reference. You may or may not want to change these hyperparameters
learning_rates = [1e-7, 5e-7]
regularization_strengths = [2.5e4, 5e4]
# *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
for num_iters in [1500]:
for lr in learning_rates:
for reg in regularization_strengths:
sigmoid_machine= Sigmoid()
sigmoid_machine.train(X_train, y_train, learning_rate=lr, reg=reg, num_iters=num_iters, batch_size=200, verbose=False)
y_train_predict = sigmoid_machine.predict(X_train)
y_val_predict = sigmoid_machine.predict(X_val)
train_acc = sum(y_train_predict ==y_train)/len(y_train)
val_acc = sum(y_val_predict ==y_val)/len(y_val)
results[(lr, reg)] = (train_acc, val_acc)
if(best_val < val_acc):
best_val = val_acc
best_sigmoid = sigmoid_machine
# *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
# Print out results.
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' % (
lr, reg, train_accuracy, val_accuracy))
print('best validation accuracy achieved during cross-validation: %f' % best_val)
Let's evaluate our best model!¶
In [6]:
# evaluate on test set
# Evaluate the best sigmoid on test set
y_test_pred = best_sigmoid.predict(X_test)
test_accuracy = np.mean(y_test == y_test_pred)
print('sigmoid on raw pixels final test set accuracy: %f' % (test_accuracy, ))
Visualization¶
In [7]:
# Visualize the learned weights for each class
w = best_sigmoid.W[:-1,:] # strip out the bias
w = w.reshape(32, 32, 3, 10)
w_min, w_max = np.min(w), np.max(w)
classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
for i in range(10):
plt.subplot(2, 5, i + 1)
# Rescale the weights to be between 0 and 255
wimg = 255.0 * (w[:, :, :, i].squeeze() - w_min) / (w_max - w_min)
plt.imshow(wimg.astype('uint8'))
plt.axis('off')
plt.title(classes[i])
