From 6d93095d5d0c1705f45ce7037ff35e7c3df726be Mon Sep 17 00:00:00 2001
From: Toren Lev Fronsdal <toren@stanford.edu>
Date: Thu, 25 Feb 2021 20:45:50 -0800
Subject: [PATCH] Update model
---
baseline_model.py | 72 +++++++++++++++++++++++++++--------------------
1 file changed, 41 insertions(+), 31 deletions(-)
diff --git a/baseline_model.py b/baseline_model.py
index d824440..5c0e0ba 100644
--- a/baseline_model.py
+++ b/baseline_model.py
@@ -21,19 +21,16 @@ import torch.optim as optim
# sklearn
import sklearn.base
-from sklearn.decomposition import PCA
import joblib
from sklearn.base import TransformerMixin, BaseEstimator
-from sklearn.feature_selection import VarianceThreshold
-from sklearn.preprocessing import OneHotEncoder
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import make_union, make_pipeline
from sklearn.compose import make_column_transformer
-
-from scipy.stats import kurtosis, skew
-
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import log_loss
########################
### Global variables ###
@@ -58,7 +55,7 @@ class Preprocessor(TransformerMixin):
num_pc_genes = 80,
num_pc_cells = 10,
seed = 2021
- ):
+ ):
self.variance_threshold = variance_threshold,
self.num_pc_genes = num_pc_genes
@@ -68,14 +65,15 @@ class Preprocessor(TransformerMixin):
def fit(self, X, y = None, X_test = None):
if X_test is not None:
X = pd.concat([X, X_test], axis = 0, ignore_index = True)
-
+
gene_feats = [col for col in X.columns if col.startswith('g-')]
cell_feats = [col for col in X.columns if col.startswith('c-')]
- numeric_feats = gene_feats+cell_feats
+ numeric_feats = gene_feats + cell_feats
categorical_feats = ['cp_time', 'cp_dose']
-
+
self._transformer = make_column_transformer(
- (OneHotEncoder(), categorical_feats)
+ (OneHotEncoder(), categorical_feats),
+ (StandardScaler(), numeric_feats)
)
self._transformer.fit(X)
return self
@@ -433,8 +431,8 @@ class Network(sklearn.base.BaseEstimator):
preds_proba = 1 / (1 + np.exp(-preds))
return preds_proba.astype("float32")
-
# %% [code]
+# FOR FUTURE USE: if use label smoothing
class SmoothCrossEntropyLoss(nn.modules.loss._WeightedLoss):
"""
Computes smoothed cross entropy (log) loss.
@@ -486,7 +484,7 @@ class SmoothCrossEntropyLoss(nn.modules.loss._WeightedLoss):
return loss
-
+# FOR FUTURE USE: if use clipping
class ClippedCrossEntropyLoss(nn.modules.loss._WeightedLoss):
"""
Computes clipped cross entropy (log) loss.
@@ -532,66 +530,78 @@ class ClippedCrossEntropyLoss(nn.modules.loss._WeightedLoss):
train_drug = pd.read_csv("../input/lish-moa/train_drug.csv")
X = pd.read_csv("../input/lish-moa/train_features.csv")
y = pd.read_csv("../input/lish-moa/train_targets_scored.csv")
-X_test = pd.read_csv("../input/lish-moa/test_features.csv")
-submission = pd.read_csv("../input/lish-moa/sample_submission.csv")
# Remove control observations
y = y.loc[X["cp_type"]=="trt_cp"].reset_index(drop=True)
X = X.loc[X["cp_type"]=="trt_cp"].reset_index(drop=True)
-# used to set control obs. to zero for preds
-X_test_copy = X_test.copy()
# %% [code]
+##########################
+### Data Preprocessing ###
+##########################
+
+X, X_test, y, y_test = train_test_split(
+ X, y, test_size=0.2, random_state=1998
+)
+
transformer = Preprocessor()
transformer.fit(X)
X = transformer.transform(X)
+X_test = transformer.transform(X_test)
y = y.drop(["sig_id"], axis = 1).values.astype("float32")
+y_test = y_test.drop(["sig_id"], axis = 1).values.astype("float32")
# %% [code]
+# Define network architecture
+
n_input = X.shape[1]
n_output = y.shape[1]
hidden_units = 640
dropout = 0.2
net_obj = Sequential(
- nn.BatchNorm1d(n_input),
- nn.Dropout(dropout),
nn.Linear(n_input, hidden_units),
- nn.ReLU(),
nn.BatchNorm1d(hidden_units),
+ nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(hidden_units, hidden_units),
- nn.ReLU(),
nn.BatchNorm1d(hidden_units),
+ nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(hidden_units, n_output)
)
# %% [code]
-# zero the submission preds
-submission.iloc[:,1:207] = 0
+loss = nn.BCEWithLogitsLoss()
+# Initialize network
net = Network(
net_obj=net_obj,
- max_epochs=6,
+ max_epochs=20,
batch_size=128,
device=device,
- loss_fn=SmoothCrossEntropyLoss(smoothing=0.001, device=device),
- lr=0.001,
+ loss_fn=loss,
+ lr=0.01,
weight_decay=1e-6,
lr_scheduler="ReduceLROnPlateau"
)
-clipped_log_loss = ClippedCrossEntropyLoss(smoothing=0.001)
net.fit(
X=X,
y=y,
- eval_metric=[clipped_log_loss],
- patience=7,
+ eval_set=[(X_test, y_test)],
+ eval_metric=[loss],
verbose=2
)
-net.predict_proba(X)
-
# %% [code]
+def multi_log_loss(y_pred, y_true):
+ losses = -y_true * np.log(y_pred + 1e-15) - (1 - y_true) * np.log(1 - y_pred + 1e-15)
+ return np.mean(losses)
+
+preds = net.predict_proba(X)
+print("Train loss: ", multi_log_loss(preds, y))
+
+test_preds = net.predict_proba(X_test)
+print("Test loss: ", multi_log_loss(test_preds, y_test))
--
GitLab