{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "9bnf434dH6rj" }, "outputs": [], "source": [] }, { "cell_type": "markdown", "source": [ "**Max Voting**" ], "metadata": { "id": "KRypFHdLIQvo" } }, { "cell_type": "code", "source": [ "# Import libraries\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.metrics import accuracy_score, confusion_matrix, ConfusionMatrixDisplay\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n", "from xgboost import XGBClassifier\n", "\n", "df = pd.read_csv(\"heart.csv\")\n", "\n", "X = df.drop('target', axis=1)\n", "y = df['target']\n", "\n", "# Train-test split\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42, stratify=y\n", ")\n", "\n", "scaler = StandardScaler()\n", "X_train = scaler.fit_transform(X_train)\n", "X_test = scaler.transform(X_test)\n", "\n", "# Initialize base classifiers\n", "log_reg = LogisticRegression(max_iter=300, random_state=42)\n", "dt_clf = DecisionTreeClassifier(random_state=42)\n", "rf_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n", "xgb_clf = XGBClassifier(use_label_encoder=False, eval_metric='logloss', random_state=42)\n", "\n", "# Hard Voting Classifier\n", "hard_voting = VotingClassifier(\n", " estimators=[('lr', log_reg), ('dt', dt_clf), ('rf', rf_clf), ('xgb', xgb_clf)],\n", " voting='hard'\n", ")\n", "hard_voting.fit(X_train, y_train)\n", "y_pred_hard = hard_voting.predict(X_test)\n", "print(\"Hard Voting Accuracy:\", accuracy_score(y_test, y_pred_hard))\n", "\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "_3jzsP04IQKo", "outputId": "d0dcd420-feeb-46fe-905c-3a999dff9ea2" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.12/dist-packages/xgboost/training.py:200: UserWarning: [09:36:02] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: \n", "Parameters: { \"use_label_encoder\" } are not used.\n", "\n", " bst.update(dtrain, iteration=i, fobj=obj)\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Hard Voting Accuracy: 1.0\n" ] } ] }, { "cell_type": "code", "source": [ "# Soft Voting Classifier\n", "soft_voting = VotingClassifier(\n", " estimators=[('lr', log_reg), ('dt', dt_clf), ('rf', rf_clf), ('xgb', xgb_clf)],\n", " voting='soft'\n", ")\n", "soft_voting.fit(X_train, y_train)\n", "y_pred_soft = soft_voting.predict(X_test)\n", "print(\"Soft Voting Accuracy:\", accuracy_score(y_test, y_pred_soft))\n", "\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "9hkPx_J4IQNB", "outputId": "652bb555-9a0c-404b-db28-f44db3bf9a2f" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.12/dist-packages/xgboost/training.py:200: UserWarning: [09:36:14] WARNING: /__w/xgboost/xgboost/src/learner.cc:782: \n", "Parameters: { \"use_label_encoder\" } are not used.\n", "\n", " bst.update(dtrain, iteration=i, fobj=obj)\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Soft Voting Accuracy: 1.0\n" ] } ] }, { "cell_type": "markdown", "source": [ "**Averaging Method**" ], "metadata": { "id": "X_j3QBYtMO74" } }, { "cell_type": "code", "source": [ "\n", "\n", "# Import Libraries\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", "from sklearn.datasets import fetch_openml\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.tree import DecisionTreeRegressor\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n", "\n", "# ------------------------------\n", "# Load Dataset\n", "# ------------------------------\n", "boston = fetch_openml(name=\"boston\", version=1, as_frame=True)\n", "\n", "# Convert to numeric (important)\n", "X = boston.data.apply(pd.to_numeric)\n", "y = pd.to_numeric(boston.target)\n", "\n", "# ------------------------------\n", "# Train-Test Split\n", "# ------------------------------\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42\n", ")\n", "\n", "# ------------------------------\n", "# Create Base Models\n", "# ------------------------------\n", "model1 = LinearRegression()\n", "model2 = DecisionTreeRegressor(random_state=42)\n", "model3 = RandomForestRegressor(n_estimators=100, random_state=42)\n", "\n", "# ------------------------------\n", "# Train Models\n", "# ------------------------------\n", "model1.fit(X_train, y_train)\n", "model2.fit(X_train, y_train)\n", "model3.fit(X_train, y_train)\n", "\n", "# ------------------------------\n", "# Individual Predictions\n", "# ------------------------------\n", "pred1 = model1.predict(X_test)\n", "pred2 = model2.predict(X_test)\n", "pred3 = model3.predict(X_test)\n", "\n", "# ------------------------------\n", "# Averaging Ensemble Prediction\n", "# ------------------------------\n", "y_pred = (pred1 + pred2 + pred3) / 3\n", "\n", "\n", "# ------------------------------\n", "r2 = r2_score(y_test, y_pred)\n", "\n", "\n", "print(\"R2 Score :\", r2)\n", "\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "PZKEEjmFWmmp", "outputId": "4ed8d179-fd23-4cf5-e2dd-04b9c58083ec" }, "execution_count": 9, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "R2 Score : 0.8872852109557785\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "PGCtLVNbXpWK" }, "execution_count": 7, "outputs": [] }, { "cell_type": "markdown", "source": [ "**Stacking**" ], "metadata": { "id": "Q10OsWkKRc7r" } }, { "cell_type": "code", "source": [ "!pip install vecstack" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "yGdHdzSmSB1O", "outputId": "11c7efde-8b28-4470-8e17-76f136e7ae9b" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting vecstack\n", " Downloading vecstack-0.5.2-py3-none-any.whl.metadata (2.0 kB)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (from vecstack) (2.0.2)\n", "Requirement already satisfied: scipy in /usr/local/lib/python3.12/dist-packages (from vecstack) (1.16.3)\n", "Requirement already satisfied: scikit-learn>=1.6.0 in /usr/local/lib/python3.12/dist-packages (from vecstack) (1.6.1)\n", "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=1.6.0->vecstack) (1.5.3)\n", "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=1.6.0->vecstack) (3.6.0)\n", "Downloading vecstack-0.5.2-py3-none-any.whl (22 kB)\n", "Installing collected packages: vecstack\n", "Successfully installed vecstack-0.5.2\n" ] } ] }, { "cell_type": "code", "source": [ "# Import libraries\n", "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "import xgboost as xgb\n", "from vecstack import stacking\n", "df = pd.read_csv(\"/content/heart.csv\")\n", "X = df.drop(\"target\", axis=1)\n", "y = df[\"target\"]\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42\n", ")\n", "\n", "model_1 = LinearRegression()\n", "model_2 = xgb.XGBRegressor(eval_metric='rmse', random_state=42)\n", "model_3 = RandomForestRegressor(n_estimators=100, random_state=42)\n", "\n", "all_models = [model_1, model_2, model_3]\n", "\n", "s_train, s_test = stacking(\n", " all_models, X_train, y_train, X_test,\n", " regression=True, n_folds=4, shuffle=True, random_state=42\n", ")\n", "\n", "meta_model = LinearRegression()\n", "meta_model.fit(s_train, y_train)\n", "\n", "\n", "pred_final = meta_model.predict(s_test)\n", "\n", "mse = mean_squared_error(y_test, pred_final)\n", "print(\"Mean Squared Error (Stacking):\", mse)\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "ntEiJV15IQSQ", "outputId": "8e093c73-a6af-4e34-b842-7e33ab57f6b4" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mean Squared Error (Stacking): 0.020857985206334067\n" ] } ] }, { "cell_type": "markdown", "source": [ "**Blending**" ], "metadata": { "id": "WMkCX3NRS5uK" } }, { "cell_type": "code", "source": [ "# Import libraries\n", "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "import xgboost as xgb\n", "\n", "df = pd.read_csv(\"/content/heart.csv\")\n", "X = df.drop(\"target\", axis=1)\n", "y = df[\"target\"]\n", "\n", "# Train-validation-test split (70%-20%-10%)\n", "X_train_full, X_test, y_train_full, y_test = train_test_split(X, y, test_size=0.10, random_state=42)\n", "X_train, X_val, y_train, y_val = train_test_split(X_train_full, y_train_full, test_size=0.2222, random_state=42)\n", "# 0.2222 of 90% = 20% validation\n", "\n", "# Initialize base models\n", "model_1 = LinearRegression()\n", "model_2 = xgb.XGBRegressor(eval_metric='rmse', random_state=42)\n", "model_3 = RandomForestRegressor(n_estimators=100, random_state=42)\n", "base_models = [model_1, model_2, model_3]\n", "\n", "# Train base models and make predictions for validation and test sets\n", "val_preds = []\n", "test_preds = []\n", "\n", "for model in base_models:\n", " model.fit(X_train, y_train)\n", " val_preds.append(pd.DataFrame(model.predict(X_val)))\n", " test_preds.append(pd.DataFrame(model.predict(X_test)))\n", "\n", "# Combine validation predictions as meta-features\n", "meta_X_val = pd.concat(val_preds, axis=1)\n", "meta_X_test = pd.concat(test_preds, axis=1)\n", "\n", "# Train meta-learner on validation meta-features\n", "meta_model = LinearRegression()\n", "meta_model.fit(meta_X_val, y_val)\n", "\n", "# Final predictions on test set\n", "final_pred = meta_model.predict(meta_X_test)\n", "\n", "# Evaluate performance\n", "mse = mean_squared_error(y_test, final_pred)\n", "print(\"Mean Squared Error (Blending):\", mse)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "u8pt8Su7IQUw", "outputId": "a1fd7cd3-e16a-4fe1-ede5-7e76ad5dda42" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mean Squared Error (Blending): 0.027088923263424304\n" ] } ] }, { "cell_type": "markdown", "source": [ "**Bagging**" ], "metadata": { "id": "k6KuuEqXVFzj" } }, { "cell_type": "code", "source": [ "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.ensemble import BaggingClassifier\n", "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.datasets import load_iris\n", "\n", "iris = load_iris()\n", "X = iris.data\n", "y = iris.target\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42\n", ")\n", "\n", "bagging_model = BaggingClassifier(\n", " estimator=DecisionTreeClassifier(random_state=42),\n", " n_estimators=10,\n", " random_state=42\n", ")\n", "\n", "bagging_model.fit(X_train, y_train)\n", "\n", "pred_final = bagging_model.predict(X_test)\n", "\n", "\n", "accuracy = accuracy_score(y_test, pred_final)\n", "print(\"Accuracy (Bagging on Iris):\", accuracy)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Jmp0TuYPIQXf", "outputId": "2de36393-5286-4d60-f56d-e8d3c8ad3883" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Accuracy (Bagging on Iris): 1.0\n" ] } ] }, { "cell_type": "markdown", "source": [ "**Boosting**" ], "metadata": { "id": "_Teff69BXEZv" } }, { "cell_type": "code", "source": [ "# Import necessary libraries\n", "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.ensemble import GradientBoostingRegressor\n", "\n", "\n", "df = pd.read_csv(\"/content/heart.csv\")\n", "\n", "\n", "X = df.drop(\"target\", axis=1)\n", "y = df[\"target\"]\n", "\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42\n", ")\n", "\n", "boosting_model = GradientBoostingRegressor(\n", " n_estimators=100,\n", " learning_rate=0.1,\n", " max_depth=3,\n", " random_state=42\n", ")\n", "\n", "\n", "boosting_model.fit(X_train, y_train)\n", "\n", "\n", "pred_final = boosting_model.predict(X_test)\n", "\n", "\n", "mse = mean_squared_error(y_test, pred_final)\n", "print(\"Mean Squared Error (Boosting):\", mse)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3BoK_4SMXEib", "outputId": "83e0c29f-b76a-4782-fcd2-c2afff82135c" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mean Squared Error (Boosting): 0.07407866489977881\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "TXZmGwEfXElB" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "OnwLhTQUXEop" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "xjA0SfNGIQaI" }, "execution_count": null, "outputs": [] } ] }