# Import all required libraries
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import scipy.stats as stats
from scipy.stats import skewnorm
from scipy.stats import kurtosis, norm
from scipy.stats import gamma
import missingno as msno
import random
import statsmodels.api as sm
# Increase font size of all Seaborn plot elements
sns.set(font_scale = 1.25)
# Load in UK Smoking Data
births14 = pd.read_csv("data/births14.csv")
# Set seed
random.seed(123)Exploratory Data Analysis + Data Visualization
Lecture 3
What is exploratory data analysis?
Outliers
Outliers = 1.5 * Interquartile range
Assess outliers visually
Remove outliers
# Select numerical columns
numerical_cols = births14.select_dtypes(include = ['number']).columns
for col in numerical_cols:
# Find Q1, Q3, and interquartile range (IQR) for each column
Q1 = births14[col].quantile(0.25)
Q3 = births14[col].quantile(0.75)
IQR = Q3 - Q1
# Upper and lower bounds for each column
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
# Filter out the outliers from the DataFrame
births14_clean = births14[(births14[col] >= lower_bound) & (births14[col] <= upper_bound)]
Why are there still outliers?
Missing values (NaN) visually
Conditions of normality
Histogram: bell-shaped curve
Skewness: Close to 0 for symmetry; Kurtosis: Close to 3 for normal “tailedness.”
Sample Size: Larger samples are less sensitive to non-normality.
Empirical Rule: 68-95-99.7% rule (data within 1, 2, and 3 st dev. of the mean).
Skewness
- Several definitions
- Sensitive to outliers
- Designed for one peak (unimodal)
- Mean and median are inconclusive from skew alone
Kurtosis
- Several definitions…
- Sensitive to outliers
- Designed for one peak (unimodal)
Q-Q plot
Testing normality: data shape
Code
# Change theme to "white"
sns.set_style("white")
# Make a copy of the data
dataCopy = births14.copy()
# Remove NAs
dataCopyFin = dataCopy.dropna()
# Specify desired column
i_col = dataCopyFin.weight
# Subplots
fig, (ax1, ax2) = plt.subplots(ncols = 2, nrows = 1)
# Density plot
sns.kdeplot(i_col, linewidth = 5, ax = ax1)
ax1.set_title('Newborn Weight Density plot')
# Q-Q plot
sm.qqplot(i_col, line='s', ax = ax2)
ax2.set_title('Newborn Weight Q-Q plot')
plt.tight_layout()
plt.show()
Positive-skew (left-tailed)
Code
# Change theme to "white"
sns.set_style("white")
# Make a copy of the data
dataCopy = births14.copy()
# Select only numerical columns
dataRed = dataCopyFin.select_dtypes(include=np.number)
# Fill the subplots
for k in dataRed.columns:
# Create a figure with two subplots
fig, (ax1, ax2) = plt.subplots(ncols = 2, nrows = 1)
# Density plot
sns.kdeplot(dataRed[k], linewidth = 5, ax = ax1)
ax1.set_title(f'{k} Density Plot')
# Q-Q plot
sm.qqplot(dataRed[k], line = 's', ax = ax2)
ax2.set_title(f'{k} QQ Plot')
plt.tight_layout()
plt.show()
Why storytelling?
Epic of Gilgamesh (c. 2100 BC)
First known “hero’s journey”
Goal: Apply storytelling to your visuals
Aircraft-Wildlife Collisions
{seaborn}
Seaborn is a Python data visualization library based on matplotlib. It provides a high-level interface for drawing attractive and informative statistical graphics.
One Numeric Variable
Histogram
- Frequency Distribution
- Easy to Interpret
- Identifies Patterns
Density Plot
- Smooth Distribution Curve
- Highlights Density
- Comparative Analysis
Histograms
Histograms: bins
Density Plot
Two Numeric Variables
Scatterplot
- Relationship Visualization
- Outlier Identification
- Pattern Recognition
2D Density Plot
- Density Distributions
- Combine Contour and Color
- Complex Data Interpretation
Scatterplots
Scatterplots - color
Scatterplots - size + color
Scatterplots - linear relationships
Scatterplots - grouped relationships
2D Density Plots
2D Density plots: contours
2D Density plots: filled contours
Two Ordered Numeric Variables
Line Plot
- Trend Identification
- Simple and Clear
- Comparative Analysis
Area Plot
- Cumulative Representation
- Emphasizes Volume
- Layered Comparisons
Line Plot
Line Plot: grouped lines
One Categorical
Barplot
- Categorical Comparison
- Clear Visualization
- Versatile Use
Pie Chart
- Proportional Representation
- Simple Interpretation
- Visual Appeal
Barplot
Pie Chart
Can’t use {seaborn}
One Numerical + One Categorical
Boxplot
- Displays Quartiles
- Identifies Outliers
- Comparative Analysis
Violin chart
- Density Representation
- Richer Data Insight
- Visualizes Data Spread
Boxplots
Trim axes
Violin Plots
Violin Plots: paired
Violin Plots: quartiles + split
Cleaning up our plots
My minimum expectation:
Aside: Correlations
Code
sns.set_theme(style = "white")
birds_num = birds.select_dtypes(include = 'number')
corr = birds_num.corr()
mask = np.triu(np.ones_like(corr, dtype = bool))
f, ax = plt.subplots(figsize = (8, 6))
cmap = sns.diverging_palette(230, 20, as_cmap = True)
sns.heatmap(corr, mask = mask, cmap = cmap, vmax = 0.5, center = 0,
square = True, linewidths = .5, cbar_kws = {"shrink": 0.5})
plt.show()
Lastly: Pairgrids
Livecoding: Diwali sales data
diwali = pd.read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2023/2023-11-14/diwali_sales_data.csv', encoding = 'iso-8859-1')
diwali.head()| User_ID | Cust_name | Product_ID | Gender | Age Group | Age | Marital_Status | State | Zone | Occupation | Product_Category | Orders | Amount | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1002903 | Sanskriti | P00125942 | F | 26-35 | 28 | 0 | Maharashtra | Western | Healthcare | Auto | 1 | 23952.0 |
| 1 | 1000732 | Kartik | P00110942 | F | 26-35 | 35 | 1 | Andhra Pradesh | Southern | Govt | Auto | 3 | 23934.0 |
| 2 | 1001990 | Bindu | P00118542 | F | 26-35 | 35 | 1 | Uttar Pradesh | Central | Automobile | Auto | 3 | 23924.0 |
| 3 | 1001425 | Sudevi | P00237842 | M | 0-17 | 16 | 0 | Karnataka | Southern | Construction | Auto | 2 | 23912.0 |
| 4 | 1000588 | Joni | P00057942 | M | 26-35 | 28 | 1 | Gujarat | Western | Food Processing | Auto | 2 | 23877.0 |
Code
# Examine data
diwali.info()
# Data types
diwali.dtypes
# Describe numerical columns
diwali.describe()
# Describe categories
diwali.describe(exclude = [np.number])
# Unique levels
categorical_cols = diwali.select_dtypes(include = ['object', 'category']).columns
unique_levels = diwali[col].unique()
# Outliers
# Make a copy of the diwali data
dataCopy = diwali.copy()
# Select only numerical columns
dataRed = dataCopy.select_dtypes(include = np.number)
# List of numerical columns
dataRedColsList = dataRed.columns[...]
# For all values in the numerical column list from above
for i_col in dataRedColsList:
# List of the values in i_col
dataRed_i = dataRed.loc[:,i_col]
# Define the 25th and 75th percentiles
q25, q75 = round((dataRed_i.quantile(q = 0.25)), 3), round((dataRed_i.quantile(q = 0.75)), 3)
# Define the interquartile range from the 25th and 75th percentiles defined above
IQR = round((q75 - q25), 3)
# Calculate the outlier cutoff
cut_off = IQR * 1.5
# Define lower and upper cut-offs
lower, upper = round((q25 - cut_off), 3), round((q75 + cut_off), 3)
# Print the values
print(' ')
# For each value of i_col, print the 25th and 75th percentiles and IQR
print(i_col, 'q25 =', q25, 'q75 =', q75, 'IQR =', IQR)
# Print the lower and upper cut-offs
print('lower, upper:', lower, upper)
# Count the number of outliers outside the (lower, upper) limits, print that value
print('Number of Outliers: ', dataRed_i[(dataRed_i < lower) | (dataRed_i > upper)].count())
# Missing values
diwali.isnull().sum()
# Normality - qq plot
# Change theme to "white"
sns.set_style("white")
# Make a copy of the data
dataCopy = diwali.copy()
# Remove NAs
dataCopyFin = dataCopy.dropna()
# Specify desired column
i_col = dataCopyFin.Amount
# Subplots
fig, (ax1, ax2) = plt.subplots(ncols = 2, nrows = 1)
# Density plot
sns.kdeplot(i_col, linewidth = 5, ax = ax1)
ax1.set_title('Amount spent (₹)')
# Q-Q plot
sm.qqplot(i_col, line = 's', ax = ax2)
ax2.set_title('Amount spent Q-Q plot')
plt.tight_layout()
plt.show()