Project: Analyse a Dataset & Write a Report

import pandas as pd

raw = pd.read_csv("students.csv", na_values=["AB"])
print("Rows:", len(raw), " duplicate rows:", raw.duplicated().sum())
print("gender:", raw["gender"].value_counts().to_dict())
print("school_type:", raw["school_type"].value_counts().to_dict())
marks = ["maths", "science", "english", "hindi", "social_science"]
print("Marks above 100:", {c: int((raw[c] > 100).sum()) for c in marks if (raw[c] > 100).any()})
print("Attendance above 100:", int((raw["attendance"] > 100).sum()))
print("Missing:", raw.isna().sum()[raw.isna().sum() > 0].to_dict())

import pandas as pd

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)          # impossible marks become missing
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

students = load_students()
print("Rows after cleaning:", len(students))
print("gender:", students["gender"].value_counts().to_dict())
print("school_type:", students["school_type"].value_counts().to_dict())
print("Missing:", students.isna().sum()[students.isna().sum() > 0].to_dict())

import pandas as pd
import matplotlib.pyplot as plt

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

students = load_students()
maths = students["maths"].dropna()
q1, q3 = maths.quantile([0.25, 0.75])
print(f"n = {len(maths)}, mean {maths.mean():.1f}, median {maths.median():.1f}, "
      f"SD {maths.std():.1f}, IQR {q3 - q1:.1f}, skew {maths.skew():.2f}")

bands = pd.cut(maths, [0, 33, 60, 75, 101], right=False,
               labels=["0-32 (fail)", "33-59", "60-74", "75-100"])
table = bands.value_counts(sort=False).to_frame("students")
table["percent"] = (table["students"] / len(maths) * 100).round(1)
print(table)

fig, axes = plt.subplots(1, 2, figsize=(10, 3.5))
axes[0].hist(maths, bins=range(15, 105, 5), edgecolor="white", color="#7c3aed")
axes[0].set_title("Maths marks")
groups = [students.loc[students["school_type"] == s, "maths"].dropna() for s in ["Government", "Private"]]
axes[1].boxplot(groups, tick_labels=["Government", "Private"])
axes[1].set_title("Maths marks by school type")
plt.show()

import numpy as np
import pandas as pd
from scipy import stats

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

students = load_students()
maths = students["maths"].dropna()
n = len(maths)
ci = stats.t.interval(0.95, df=n - 1, loc=maths.mean(), scale=stats.sem(maths))
print(f"Mean maths mark: {maths.mean():.2f}, 95% CI {ci[0]:.2f} to {ci[1]:.2f}")

passed = int((maths >= 33).sum())
pci = stats.binomtest(passed, n).proportion_ci(method="wilson")
print(f"Pass rate: {passed}/{n} = {passed / n:.1%}, 95% CI {pci.low:.1%} to {pci.high:.1%}")

import numpy as np
import pandas as pd
from scipy import stats

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

def compare(df, column, a, b):
    x = df.loc[df[column] == a, "maths"].dropna()
    y = df.loc[df[column] == b, "maths"].dropna()
    res = stats.ttest_ind(x, y, equal_var=False)
    ci = res.confidence_interval(0.95)
    d = (x.mean() - y.mean()) / np.sqrt((x.var() + y.var()) / 2)
    print(f"{a} ({len(x)}) {x.mean():.2f} vs {b} ({len(y)}) {y.mean():.2f}: "
          f"diff {x.mean() - y.mean():+.2f}, 95% CI {ci.low:.2f} to {ci.high:.2f}, "
          f"p = {res.pvalue:.4f}, d = {d:.2f}")

students = load_students()
compare(students, "tuition", "Yes", "No")
compare(students, "school_type", "Private", "Government")
compare(students, "gender", "F", "M")

import pandas as pd
from scipy import stats

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

students = load_students().dropna(subset=["maths"])
cities = [g["maths"] for _, g in students.groupby("city")]
anova = stats.f_oneway(*cities)
print(f"Cities: ANOVA F = {anova.statistic:.2f}, p = {anova.pvalue:.3f}; "
      f"Kruskal-Wallis p = {stats.kruskal(*cities).pvalue:.3f}")

students["result"] = students["maths"].ge(33).map({True: "Pass", False: "Fail"})
table = pd.crosstab(students["school_type"], students["result"])
print(table)
chi = stats.chi2_contingency(table)
print(f"Chi-square = {chi.statistic:.2f}, p = {chi.pvalue:.4f}, smallest expected count {chi.expected_freq.min():.1f}")
print(pd.crosstab(students["school_type"], students["result"], normalize="index").round(3))

import numpy as np
import pandas as pd
from scipy import stats

def load_students():
    df = pd.read_csv("students.csv", na_values=["AB"]).drop_duplicates()
    df["gender"] = df["gender"].str.strip().str.upper()
    school = df["school_type"].str.strip().str.lower().str.rstrip(".")
    df["school_type"] = school.replace({"govt": "government"}).str.title()
    for col in ["maths", "science", "english", "hindi", "social_science"]:
        df[col] = df[col].where(df[col] <= 100)
    df["attendance"] = df["attendance"].where(df["attendance"] <= 100)
    return df

data = load_students().dropna(subset=["maths", "study_hours", "attendance"])
for col in ["study_hours", "attendance"]:
    r = stats.pearsonr(data[col], data["maths"])
    print(f"r(maths, {col}) = {r.statistic:.3f}, p = {r.pvalue:.1e}")

names = ["study_hours", "attendance", "tuition_yes", "private"]
data = data.assign(tuition_yes=(data["tuition"] == "Yes").astype(int),
                   private=(data["school_type"] == "Private").astype(int))
X = np.column_stack([np.ones(len(data)), data[names]])
y = data["maths"].to_numpy()
coef, *_ = np.linalg.lstsq(X, y, rcond=None)
resid = y - X @ coef
df = len(y) - X.shape[1]
se = np.sqrt(np.diag((resid ** 2).sum() / df * np.linalg.inv(X.T @ X)))
p = 2 * stats.t.sf(np.abs(coef / se), df)
r2 = 1 - (resid ** 2).sum() / ((y - y.mean()) ** 2).sum()
table = pd.DataFrame({"coef": coef, "std_err": se, "p_value": p}, index=["intercept"] + names)
print(table.round(4))
print(f"n = {len(y)}, R^2 = {r2:.3f}")

MATHS MARKS IN THE DISTRICT SAMPLE: A SHORT REPORT (practice data)

1. Summary
   478 students with a maths mark were analysed after cleaning (3 duplicate rows removed,
   8 spellings of school type unified, 2 impossible values set to missing).
   The average mark is 58.1 (95% CI 56.5 to 59.7) and 92.1% of students pass
   (95% CI 89.3% to 94.2%). Marks are symmetric with a wide spread (SD 17.9).

2. Group differences (all three comparisons planned in advance)
   - Private vs government schools: +4.6 marks (95% CI 1.4 to 7.7, p = 0.005), a small effect.
     Pass rates (93.3% vs 90.8%) do not differ beyond chance (p = 0.41).
   - Tuition vs no tuition: +1.2 marks (95% CI -2.0 to 4.4, p = 0.46): no clear difference.
   - Girls vs boys: +2.0 marks (95% CI -1.2 to 5.3, p = 0.22): no clear difference.
   - Cities: no evidence of different averages (ANOVA p = 0.78).

3. Factors linked with higher marks (regression, n = 471, R^2 = 0.39)
   Each extra hour of daily study: about +8 marks; each attendance point: about +0.9 marks,
   holding other factors fixed. Private schooling keeps a +3.7 association; tuition does not.

4. Limits
   Observational data: the associations are not proof of cause. Students were not randomly
   assigned to schools or study habits, and unmeasured factors (family background, teacher
   quality) may explain part of every difference. The file is synthetic practice data.

5. Recommendation
   Study time and attendance are the factors most strongly linked with marks and the ones a
   school can influence; a pilot programme with a control group would test whether
   encouraging them actually raises marks.