Dealing with Missing Data (NaN)
from sklearn.preprocessing import Imputer
# mean imputation (other options: median, most_frequent)
imr = Imputer(missing_values='NaN', strategy='mean', axis=0)
imr = imr.fit(df)
imputed_data = imr.transform(df.values)# return the number of missing values per column
df.isnull().sum()
# drop rows with missing values
df.dropna()
# drop columns with NaN in any row
df.dropna(axis=1)
# only drop rows where all columns are NaN
df.dropna(how='all')
# drop rows that have not at least 4 non-NaN values
df.dropna(thresh=4)
# only drop rows where NaN appear in specific columns (here: 'C')
df.dropna(subset=['C'])