import os
import gzip
import math
import operator
import sklearn.model_selection
import random
import matplotlib.pyplot as matplot
import matplotlib
%matplotlib inline
import pandas as pd
import numpy as np
import pickle as cPickle
from time import time
from itertools import chain
from collections import Counter
from PIL import Image
from scipy.stats import mode
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
1NN¶
Importing the data from tensorflow package and checking the shape of the training and testing data set¶
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/')
train = mnist.train.images
validation = mnist.validation.images
test = mnist.test.images
trlab = mnist.train.labels
vallab = mnist.validation.labels
tslab = mnist.test.labels
train = np.concatenate((train, validation), axis=0)
trlab = np.concatenate((trlab, vallab), axis=0)
train.shape
trlab.shape
test.shape
Example image (28 x 28 pixels)¶
x = np.reshape(train[0], [28,28])
matplot.imshow(x, cmap='Greys_r')
Checking the distribution of color (greyscale) in the image¶
0 = Black ; 1 = White
matplot.subplots(figsize=(12, 8))
matplot.hist(train[0])
matplot.show()
Some example images with their corresponding labels¶
classes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
num_classes = len(classes)
samples = 8
matplot.subplots(figsize=(15, 10))
for y, cls in enumerate(classes):
idxs = np.nonzero([i == y for i in trlab])
idxs = np.random.choice(idxs[0], samples, replace=False)
for i , idx in enumerate(idxs):
plt_idx = i * num_classes + y + 1
matplot.subplot(samples, num_classes, plt_idx)
matplot.imshow(train[idx].reshape((28, 28)))
matplot.axis("off")
if i == 0:
matplot.title(cls)
matplot.show()
We can see that the machine can confuse 1 with 7 similarly, 9 is very close to 4, 0 to 8 and 2 is very close to 7 in some instances. We might encounter error in prediction in these cases
KNN function¶
def knn(train, train_label, test, test_label, k):
pred = []
for w in range(len(test)):
test_1 = test[w]
diff = (train - test_1)
dist = np.einsum('ij, ij->i', diff, diff) #distance measure
nearest_lbs = train_label[np.argsort(dist)[:k]]
major = mode(nearest_lbs)[0][0]
pred.append(major)
cm = pd.DataFrame(confusion_matrix(test_label, pred))
err = 1-accuracy_score(test_label, pred)
return [err,cm,pred]
st = time()
Q1 = knn(train,trlab,test,tslab,1)
t = time() - st
Results of 1NN¶
print(Q1[0]) #Error Rate
print(t) #time taken (in seconds) for classifying 10000 test images
print(t/10000) #Query time (in seconds) for 1 image classification
print(1-Q1[0]) #Accuracy
print(Q1[1]) #Confusion Matrix: Original Label VS Predicted Value
Error rate for individual digits¶
0 - 0.007173
1 - 0.005286
2 - 0.038760
3 - 0.039604
4 - 0.038697
5 - 0.035874
6 - 0.014614
7 - 0.035019
8 - 0.055441
9 - 0.041625
- After implementing the 1NN we get a very good error rate of 3.09 %
- Error is particularly high in 2,3,4,5,7,8 and 9 highest among them is 8
- there is misprediction between 7 with 1; 8 and 5 with 3; 9 with 4; and 7 with 2
KNN with Leave one out Cross validation¶
def knncv(data, label, klist):
df = pd.DataFrame(index=range(len(label)), columns=range(len(klist)))
for p in range(len(label)):
te = data[p]
te_lb = label[p]
tr = np.delete(data, p, 0)
train_label = np.delete(label, p)
diff = (tr - te)
dis = np.einsum('ij, ij->i', diff, diff)
for i, k in enumerate(klist):
near = train_label[np.argsort(dis)[:k]]
pick = mode(near)[0][0]
if pick == te_lb:
df.iloc[p][i] = 0
else:
df.iloc[p][i] = 1
return df
LOOCV kNN for k values 1 to 20¶
Q2 = knncv(train, trlab, range(1,21))
# print(np.mean(Q2, axis = 0)) # Error rate for K values (1 to 20)
matplot.subplots(figsize=(20, 10))
matplot.plot(np.mean(Q2, axis = 0))
matplot.xticks(range(0,20), range(1,21))
matplot.grid(True)
matplot.title('Error rate vs K value')
matplot.show()
K = 3 seems to give the best result¶
st = time()
Q2a = knn(train,trlab,test,tslab,3)
t = time()-st
print(t/10000) #Query time (in seconds) for 1 image classification for 3NN
Results for 3NN¶
print(Q2a[0]*100) #Error Rate
print(Q2a[1]) #Confusion Matrix
Error rate for individual digits¶
0 - 0.006122
1 - 0.001762
2 - 0.034884
3 - 0.033663
4 - 0.032587
5 - 0.036996
6 - 0.014614
7 - 0.035992
8 - 0.061602
9 - 0.040634
- We improved a bit
- After implementing the 3NN we get a better error rate of 2.95 %
- Error is decresed little in 2,3,4,5,7,8 and 9 highest among them is still 8, where error increased
- there is still some misprediction between 7 with 1; 8 and 5 with 3; 9 with 4; and 7 with 2
