DATA_DIR = Path.cwd()
FILEPATH = DATA_DIR / "titanic_sub.csv"

df_raw: pd.DataFrame = pd.read_csv(
    FILEPATH,
    low_memory=False,
    na_values=["", "NA", "n/a"]
)
print(f"Loaded: {df_raw.shape[0]:,} righe × {df_raw.shape[1]} colonne")

df_raw.head(10)

# Datatypes & non-null values
df_raw.info(verbose=True)

# Descriptive statistics (numerical and categorical)
df_raw.describe(include="all").T

print("Duplicated rows:", df_clean.duplicated().sum())

missing_counts = report_missing_values(df_clean,  top_n=None, show_heatmap=True)

# distribuzione complessiva
class_counts = df_clean['Survived'].value_counts()
class_pct    = df_clean['Survived'].value_counts(normalize=True).round(3) * 100
print(class_counts)
print(class_pct)

num_cols = df_clean.select_dtypes(include="number").columns.drop(["PassengerId"])
df_clean[num_cols].hist(figsize=(10,6), bins=30, layout=(2,3))
plt.suptitle("Numeric feature distributions")
plt.tight_layout()

corr_matrix = df_clean[num_cols].corr()
plt.figure(figsize=(6,5))
sns.heatmap(corr_matrix, annot=True, fmt=".2f", cmap="coolwarm", vmin=-1, vmax=1)
plt.title("Pearson correlation – numeric"); plt.tight_layout()

from sklearn.model_selection import train_test_split

# Separo target e features

y = df_clean['Survived'] # target
X = df_clean.drop(columns=['Survived', 'PassengerId'])  # drop PassengerId (non predittiva)

# Train-test split (75% train, 25% test) ----------------------------------
X_train_full, X_test, y_train_full, y_test = train_test_split(
    X, 
    y, 
    test_size=0.25, 
    random_state=SEED, 
    stratify=y
)

# Train-Validation split (75% train, 25% validation) ----------------------------------
X_train, X_val, y_train, y_val = train_test_split(
    X_train_full, 
    y_train_full, 
    test_size=0.25, 
    random_state=SEED, 
    stratify=y_train_full
)

print("X_train: ", X_train.shape)
print("X_val  : ", X_val.shape)
print("X_test : ", X_test.shape)
print("y_train: ", y_train.shape)
print("y_val  : ", y_val.shape)
print("y_test : ", y_test.shape)

def class_balance(label, name):
    pct = label.value_counts(normalize=True).round(3) * 100
    return f"{name:<6} 0 → {pct[0]:>5.1f}% | 1 → {pct[1]:>5.1f}%"

print(class_balance(y_train, "train"))
print(class_balance(y_val,   "valid"))
print(class_balance(y_test,  "test "))

missing_counts = report_missing_values(X_train,  top_n=None, show_heatmap=True)

import pandas as pd
import matplotlib.pyplot as plt

# Calcola mediana Age stratificata per Pclass e Sex
group_medians = (
    X_train.groupby(['Pclass', 'Sex'])['Age']
      .median()
      .unstack()
      .round(1)  # round for nicer display
)

print("Mediana Età per (Pclass, Sex)", group_medians)

# Plot
ax = group_medians.plot(kind='bar')
ax.set_title('Mediana Età per Classe di Viaggio e Sesso')
ax.set_xlabel('Classe di Viaggio (Pclass)')
ax.set_ylabel('Età (anni)')
plt.tight_layout()
plt.show()

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, f1_score

# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------

def build_pipeline(include_age: bool = True, include_embarked: bool = True) -> Pipeline:
    """Restituisce una pipeline di pre-elaborazione + regressione logistica.

    * Utilizza l'imputazione mediana per le caratteristiche numeriche;
    * Utilizza l'imputazione più frequente + One-Hot (handle_unknown=‘ignore’) per le categorie;
    * Scala le caratteristiche numeriche in modo che i coefficienti LR siano più comparabili.
    """
    
    numeric_features = ["Pclass"]
    categorical_features = ["Sex"]

    if include_age:
        numeric_features.append("Age")
    if include_embarked:
        categorical_features.append("Embarked")

    numeric_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="most_frequent")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    model = Pipeline(
        steps=[
            ("preprocess", preprocessor),
            ("classifier", LogisticRegression(max_iter=1000, random_state=SEED)),
        ]
    )

    return model


def run_experiment(
    name: str,
    X_train: pd.DataFrame,
    y_train: pd.Series,
    X_val: pd.DataFrame,
    y_val: pd.Series,
    include_age: bool = True,
    include_embarked: bool = True,
    drop_na: bool = False,
):
    """Fit + evaluate a pipeline and return metrics + sample sizes."""
    if drop_na:
        keep_idx = X_train.dropna(subset=["Age", "Embarked"]).index
        X_train = X_train.loc[keep_idx]
        y_train = y_train.loc[keep_idx]

        keep_idx = X_val.dropna(subset=["Age", "Embarked"]).index
        X_val = X_val.loc[keep_idx]
        y_val = y_val.loc[keep_idx]

    pipe = build_pipeline(include_age=include_age, include_embarked=include_embarked)
    pipe.fit(X_train, y_train)
    y_pred = pipe.predict(X_val)

    acc = accuracy_score(y_val, y_pred)
    f1 = f1_score(y_val, y_pred)

    return (name, acc, f1, len(X_train), len(X_val))


# ---------------------------------------------------------------------------
# Experiments (A, B, C) mirroring the original rapid tests
# ---------------------------------------------------------------------------

results = []

# A) Drop the problematic columns entirely
cols_A = X_train.columns.difference(["Age", "Embarked"])
results.append(
    run_experiment(
        "A_drop_cols",
        X_train[cols_A],
        y_train,
        X_val[cols_A],
        y_val,
        include_age=False,
        include_embarked=False,
    )
)

# B) Drop rows with missing Age or Embarked
results.append(
    run_experiment(
        "B_drop_rows",
        X_train,
        y_train,
        X_val,
        y_val,
        include_age=True,
        include_embarked=True,
        drop_na=True,
    )
)

# C) Imputation handled inside the pipeline (recommended)
results.append(
    run_experiment(
        "C_impute",
        X_train,
        y_train,
        X_val,
        y_val,
        include_age=True,
        include_embarked=True,
        drop_na=False,
    )
)

# Pretty‑print the outcomes
for name, acc, f1, n_tr, n_val in results:
    print(f"{name:12s} | acc = {acc:.3f} • f1 = {f1:.3f}   (n_tr = {n_tr}, n_val = {n_val})")

import pandas as pd
import numpy as np
import scipy.stats as st

def chi2_cramer(col: pd.Series, target: pd.Series):
    """Restituisce χ², p-value e Cramér V per una variabile categorica vs target."""
    contingency = pd.crosstab(col, target)
    chi2, p, _, _ = st.chi2_contingency(contingency, correction=False)
    n = contingency.values.sum()
    k = min(contingency.shape)
    V = np.sqrt(chi2 / (n * (k - 1)))
    return chi2, p, V

# Esempio solo sul training-set (dopo lo split!)
for feat in ["Sex", "Embarked", "Pclass"]:
    chi2, p, V = chi2_cramer(X_train[feat], y_train)
    print(f"{feat:<8}  χ²={chi2:6.1f}  p={p:.3e}  V={V:.3f}")

def drop_nan_rows(df: pd.DataFrame, y: pd.Series) -> tuple[pd.DataFrame, pd.Series]:
    mask = ~df[["Age", "Embarked"]].isna().any(axis=1)
    return df.loc[mask].copy(), y.loc[mask]

X_train, y_train = drop_nan_rows(X_train, y_train)
X_val,   y_val   = drop_nan_rows(X_val,   y_val)
X_test,  y_test  = drop_nan_rows(X_test,  y_test)

print(f"Dropped rows → train {len(y_train)}, val {len(y_val)}, test {len(y_test)}")

X_train

import pandas as pd
from sklearn.preprocessing import OneHotEncoder

# ------------------------------------------------------------------
# 1) Preparo il OneHotEncoder
# ------------------------------------------------------------------
cat_cols = ["Sex", "Embarked"]

ohe = OneHotEncoder(
    handle_unknown="ignore",   
    sparse_output=False,       
    dtype="int8"               
)

ohe.fit(X_train[cat_cols])     # <<-- solo TRAIN, niente leakage!

def encode_with_ohe(df: pd.DataFrame) -> pd.DataFrame:
    """Ritorna un DataFrame con le variabili categoriche una-hot-encodate
    e le numeriche intatte, mantenendo l’indice originale."""
    df_num = df.drop(columns=cat_cols)  # tutte le feature NON categoriche
    encoded = ohe.transform(df[cat_cols])

    # ricava nomi colonne (es. Sex_male, Embarked_S, …) dalla versione 1.2+
    new_cols = ohe.get_feature_names_out(cat_cols)

    df_cat = pd.DataFrame(encoded, columns=new_cols, index=df.index)
    return pd.concat([df_num, df_cat], axis=1)

# ------------------------------------------------------------------
# 2) Applico l’encoder a tutti i set (train, validation, test)
# ------------------------------------------------------------------
X_train_enc = encode_with_ohe(X_train)
X_val_enc   = encode_with_ohe(X_val)
X_test_enc  = encode_with_ohe(X_test)

print(X_train_enc.columns.tolist())

X_train_enc

import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold, RandomizedSearchCV, GridSearchCV
from sklearn.metrics import make_scorer, f1_score

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
f1_scorer = make_scorer(f1_score)

# ------------------------------------------------------------------
# 1) ricerca grossolana con RandomizedSearchCV
# ------------------------------------------------------------------
param_dist = {
    "max_depth":        [None] + list(np.arange(2, 51)),     # 2…50
    "min_samples_leaf": np.arange(1, 51),                    # 1…50
    "max_features":     [None, "sqrt", "log2"]
}

rnd_search = RandomizedSearchCV(
    estimator=DecisionTreeClassifier(
        random_state=SEED,
        class_weight="balanced"
    ),
    param_distributions=param_dist,
    n_iter=100,                  # 100 campioni casuali >> 3*4*5 = 60 del tuo grid originale
    scoring=f1_scorer,
    n_jobs=-1,
    cv=cv,
    random_state=SEED
).fit(X_train_enc, y_train)

best_params = rnd_search.best_params_
best_f1     = rnd_search.best_score_
print(f"[1] RandomizedSearch  → best F1={best_f1:.3f}  params={best_params}")

# ------------------------------------------------------------------
# 2) se un iperparametro cade sul bordo, raffinamento locale con GridSearchCV
# ------------------------------------------------------------------
def around(x: int, width: int = 4, low: int = 1) -> list[int]:
    """Restituisce [x-w, …, x, …, x+w] entro i limiti."""
    return [k for k in range(x - width, x + width + 1) if k >= low]

leaf_star = best_params["min_samples_leaf"]
depth_star = best_params["max_depth"] if best_params["max_depth"] else 0  # None → 0 fittizio

# controlla se siamo ai bordi dell’intervallo iniziale
border_leaf  = leaf_star in (param_dist["min_samples_leaf"][0], param_dist["min_samples_leaf"][-1])
border_depth = depth_star in (0, param_dist["max_depth"][-1])

if border_leaf or border_depth:
    grid = {
        "max_depth":        [None] if depth_star == 0 else around(depth_star, width=3, low=2),
        "min_samples_leaf": around(leaf_star, width=3, low=1),
        "max_features":     [best_params["max_features"]],  # già ben determinato
    }

    grid_search = GridSearchCV(
        estimator=DecisionTreeClassifier(
            random_state=SEED,
            class_weight="balanced"
        ),
        param_grid=grid,
        scoring=f1_scorer,
        cv=cv,
        n_jobs=-1
    ).fit(X_train_enc, y_train)

    best_params = grid_search.best_params_
    best_f1     = grid_search.best_score_
    print(f"[2] GridSearch      → best F1={best_f1:.3f}  params={best_params}")

# ------------------------------------------------------------------
# 3) addestramento finale sul train completo e valutazione sul validation
# ------------------------------------------------------------------
dt_best = DecisionTreeClassifier(
    random_state=SEED,
    class_weight="balanced",
    **best_params
).fit(X_train_enc, y_train)

val_f1  = f1_score(y_val, dt_best.predict(X_val_enc))
val_acc = accuracy_score(y_val, dt_best.predict(X_val_enc))
print(f"[VAL] acc={val_acc:.3f}  F1={val_f1:.3f}  con params={best_params}")

# fit su train + val con i migliori hyper-parametri
X_fin = pd.concat([X_train_enc, X_val_enc], axis=0)
y_fin = pd.concat([y_train, y_val], axis=0)

dt_final = DecisionTreeClassifier(
    random_state=SEED,
    class_weight="balanced",
    **best_params
).fit(X_fin, y_fin)

y_test_pred = dt_final.predict(X_test_enc)
print(f"TEST  –  acc {accuracy_score(y_test, y_test_pred):.3f} "
      f"F1 {f1_score(y_test, y_test_pred):.3f}")

dt_final.get_params()

from sklearn.metrics import ConfusionMatrixDisplay

# a) Confusion matrix
ConfusionMatrixDisplay.from_estimator(dt_final, X_test_enc, y_test, normalize='true')
plt.title("Decision Tree – confusion matrix (norm.)")
plt.tight_layout() 
plt.show()

# b) Feature importance
imp = (pd.Series(dt_final.feature_importances_, index=X_fin.columns)
         .sort_values(ascending=False))
sns.barplot(x=imp.head(10), y=imp.head(10).index)
plt.title("Top-10 feature importance (DT)")
plt.xlabel("Gini importance") 
plt.ylabel("Features") 
plt.tight_layout() 
plt.show()

from sklearn.tree import plot_tree

# Plot Albero
plt.figure(figsize=(24, 16))
plot_tree(dt_final,
          feature_names=X_test_enc.columns,
          class_names=['Not Survived', 'Survived'],
          filled=True,
          rounded=True)
plt.title('Decision Tree')
plt.savefig("DT5")
plt.show()

# ----------------  PERMUTATION IMPORTANCE  ----------------
from sklearn.inspection import permutation_importance
import pandas as pd
import matplotlib.pyplot as plt

# 1) calcolo
perm = permutation_importance(
    estimator      = dt_final,
    X              = X_test_enc,
    y              = y_test,
    n_repeats      = 30,      # >10 = stima più stabile
    random_state   = SEED,
    n_jobs         = -1
)

# 2) pandas Series ordinata
imp_series = pd.Series(
    perm.importances_mean,
    index=X_test_enc.columns
).sort_values(ascending=False)

display(imp_series.head(10).to_frame("perm_importance"))

# 3) grafico
plt.figure(figsize=(8, 4))
imp_series.head(10).plot(kind="barh")
plt.gca().invert_yaxis()           # feature più importante in alto
plt.title("Permutation importance – top 10")
plt.xlabel("Δ accuracy media dopo permutazione")
plt.tight_layout()
plt.show()