SL5: Human Tumor Microarray dataset - clustering with k-means

# data analysis and wrangling
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
# import random as rnd

# visualization
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

# machine learning
from sklearn.cluster import KMeans
from sklearn.cluster import MiniBatchKMeans

from IPython.display import Image # to visualize images
from tabulate import tabulate # to create tables

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# Load the Cancer Microarray dataset (already splitted in train and test)
xtrain = pd.read_csv('/kaggle/input/14cancer/14cancer.xtrain.txt', sep='\s+',header=None)
ytrain = pd.read_csv('/kaggle/input/14cancer/14cancer.ytrain.txt',sep='\s+',header=None)

xtrain = xtrain.transpose() # The columns represent the genes, and the rows are the different samples
ytrain = ytrain.transpose() # for each sample I have a label

(n_samples, n_genes), n_labels = xtrain.shape, np.unique(ytrain).size
print(f"#genes: {n_genes}, #samples: {n_samples}, #labels {n_labels}")

xtrain

# K-means with k from 2 to 10

n_clusters = range(2,11)
alg = 'k-means++' # Method for initialization
niter = 10 # Number of time the k-means algorithm will be run with different centroid seeds.
wc_km_sos=[] # within_cluster_km_sos

print('k\tInertia\t\t\tdecrease %')
print(50 * '-')
formatter_result = ("{:d}\t{:f}\t{:f}")

for k in n_clusters:
    
    results = []
    results.append(k)
    
    km = KMeans(init=alg, n_clusters=k, n_init=niter).fit(xtrain)
    
    # inertia = Sum of squared distances of samples to their closest cluster center  
    wcv = km.inertia_
    wc_km_sos.append(wcv)
    results.append(wcv)
    
    # variations in %
    if len(wc_km_sos)>1:
        results.append(
            (wc_km_sos[k-2] - wc_km_sos[k-3])*100/wc_km_sos[k-2]
        )
    else:
        results.append(0)
        
    print(formatter_result.format(*results))    

# fig 
width, height = 8, 4
fig, ax = plt.subplots(figsize=(width,height))

ax.plot(n_clusters, wc_km_sos,  marker='o', color="darkblue")
ax.grid(color='grey', linestyle='-', linewidth=0.5);
#ax.yaxis.set_major_formatter(FormatStrFormatter('%.f'))

ax.set_xlabel("Number of Clusters K", fontsize=12)
ax.set_ylabel("within-cluster sum of squares", fontsize=12)

plt.suptitle("Total within-cluster sum of squares\n for K-means clustering",fontsize=20)
plt.subplots_adjust(top=0.825) # change title position

plt.show()

# We can compare the above chart with the one in the book:
scale = 80
Image("../input/14cancer/chart.png", width = width*scale, height = height*scale)

n_clusters = range(2,11)
niter = 10 # Number of time the k-means algorithm will be run with different centroid seeds.
wc_kmpp, wc_rnd = [], []

print('k\tK-means\t\t\trandom')
print(60 * '-')
formatter_result = ("{:d}\t{:f}\t{:f}")

for k in n_clusters:
    
    results = []
    results.append(k)
    
    kmpp = KMeans(init="k-means++", n_clusters=k, n_init=niter).fit(xtrain)
    rnd = KMeans(init="random", n_clusters=k, n_init=niter).fit(xtrain)

    results.append(kmpp.inertia_)
    results.append(rnd.inertia_)
   
    wc_kmpp.append(kmpp.inertia_)
    wc_rnd.append(rnd.inertia_)

    print(formatter_result.format(*results))    

# fig 
width, height = 8, 4
fig, ax = plt.subplots(figsize=(width,height))

ax.plot(n_clusters, wc_kmpp,  marker='*', color="darkblue", label = "k-means++")
ax.plot(n_clusters, wc_rnd,  marker='o', color="orange", label = "random")

ax.grid(color='grey', linestyle='-', linewidth=0.5);
#ax.yaxis.set_major_formatter(FormatStrFormatter('%.f'))
ax.legend()
ax.set_xlabel("Number of Clusters K", fontsize=12)
ax.set_ylabel("within-cluster sum of squares", fontsize=12)

plt.suptitle("Comparison with different method of initialization",fontsize=20)
plt.subplots_adjust(top=0.825) # change title position

plt.show()

n_clusters = range(2,11)
wc_ten_seeds, wc_twenty_seeds = [], []

print('k\tn_iter=10\t\tn_iter=20')
print(70 * '-')
formatter_result = ("{:d}\t{:f}\t{:f}")

for k in n_clusters:
    
    results = []
    results.append(k)
    
    ten_seeds = KMeans(init="k-means++", n_clusters=k, n_init=10).fit(xtrain)
    twenty_seeds = KMeans(init="k-means++", n_clusters=k, n_init=20).fit(xtrain)

    results.append(ten_seeds.inertia_)
    results.append(twenty_seeds.inertia_)
   
    wc_ten_seeds.append(ten_seeds.inertia_)
    wc_twenty_seeds.append(twenty_seeds.inertia_)

    print(formatter_result.format(*results))    

# fig 
width, height = 8, 4
fig, ax = plt.subplots(figsize=(width,height))

ax.plot(n_clusters, wc_ten_seeds,  marker='*', color="darkblue", label = "n_iter=10")
ax.plot(n_clusters, wc_twenty_seeds,  marker='o', color="orange", label = "n_iter=20")

ax.grid(color='grey', linestyle='-', linewidth=0.5);
#ax.yaxis.set_major_formatter(FormatStrFormatter('%.f'))
ax.legend()
ax.set_xlabel("Number of Clusters K", fontsize=12)
ax.set_ylabel("within-cluster sum of squares", fontsize=12)

plt.suptitle("Comparison between different n_iter",fontsize=20)
plt.subplots_adjust(top=0.825) # change title position

plt.show()

# K-means with k from 2 to 10

n_clusters = range(2,11)
alg = 'k-means++' # Method for initialization
niter = 10 # Number of time the k-means algorithm will be run with different centroid seeds.
wc_mbkm_sos=[]

print('k\tInertia\t\t\tdecrease %')
print(50 * '-')
formatter_result = ("{:d}\t{:f}\t{:f}")

for k in n_clusters:
    
    results = []
    results.append(k)
    
    mbkm = MiniBatchKMeans(init=alg, n_clusters=k, n_init=niter).fit(xtrain)
    
    # inertia = Sum of squared distances of samples to their closest cluster center  
    wcv = mbkm.inertia_
    wc_mbkm_sos.append(wcv)
    results.append(wcv)
    
    # variations in %
    if len(wc_mbkm_sos)>1:
        results.append(
            (wc_mbkm_sos[k-2] - wc_mbkm_sos[k-3])*100/wc_mbkm_sos[k-2]
        )
    else:
        results.append(0)
        
    print(formatter_result.format(*results))    

# fig 
width, height = 8, 4
fig, ax = plt.subplots(figsize=(width,height))

ax.plot(n_clusters, wc_mbkm_sos,  marker='o', color="darkblue", label = "k-means")
ax.plot(n_clusters, wc_km_sos,  marker='o', color="orange",  label = "Mini batch k-means")
ax.grid(color='grey', linestyle='-', linewidth=0.5);
#ax.yaxis.set_major_formatter(FormatStrFormatter('%.f'))

ax.legend()
ax.set_xlabel("Number of Clusters K", fontsize=12)
ax.set_ylabel("within-cluster sum of squares", fontsize=12)

plt.suptitle("Total within-cluster sum of squares\ncomparison",fontsize=20)
plt.subplots_adjust(top=0.825) # change title position

plt.show()

rows = KMeans(init="k-means++", n_clusters=3).fit(xtrain).labels_ # labels of each sample after clustering
columns = ytrain.to_numpy().flatten() # make the df into an iterable list

# Collect info in a table
tab = np.zeros(3*n_labels).reshape(3,n_labels) # rows: clusters, columns: cancer labels

# Update table
for i in range(n_samples):
    tab[rows[i],columns[i]-1]+=1 # column-1 because we range over 14 clusters (0,13)
    
# Better formatting of the table into a DataFrame
table = pd.DataFrame(tab.astype(int))
table.columns = ["breast", "prostate", "lung", "collerectal", "lymphoma", "bladder",
                 "melanoma", "uterus", "leukemia", "renal", "pancreas", "ovary", "meso", "cns"]

table