my-vignette
library(autoScorecard)
#Quick Modeling
##step 1: Import Data
accepts <- read.csv(system.file("extdata", "accepts.csv", package = "autoScorecard" ))
##step 2: Create scorecard
##Considering efficiency and readability, many parameters of this automatic modeling function
##are default, which requires high-precision modeling and needs to be manually established step by step
auto_scorecard1 <- auto_scorecard( feature = accepts[1:2000,], key_var= "application_id",
y_var = "bad_ind",sample_rate = 0.7, points0 = 600, odds0=1/20, pdo = 50,
max_depth =3,tree_p = 0.1, missing_rate = 0, single_var_rate = 1, iv_set=0.02,
char_to_number = TRUE , na.omit = TRUE)
#> Start: AIC=873.27
#> bad_ind ~ application_id + vehicle_year + vehicle_make + tot_derog +
#> tot_tr + age_oldest_tr + tot_open_tr + tot_rev_tr + tot_rev_debt +
#> tot_rev_line + rev_util + fico_score + purch_price + msrp +
#> down_pyt + loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - application_id 1 831.27 871.27
#> - purch_price 1 831.40 871.40
#> - msrp 1 832.05 872.05
#> - tot_tr 1 832.12 872.12
#> - tot_open_tr 1 832.40 872.40
#> - vehicle_year 1 832.95 872.95
#> - tot_rev_line 1 833.09 873.09
#> <none> 831.27 873.27
#> - tot_income 1 833.94 873.94
#> - vehicle_make 1 834.31 874.31
#> - down_pyt 1 834.90 874.90
#> - loan_term 1 835.25 875.25
#> - tot_rev_tr 1 836.28 876.28
#> - veh_mileage 1 836.68 876.68
#> - loan_amt 1 836.72 876.72
#> - age_oldest_tr 1 837.63 877.63
#> - tot_derog 1 838.33 878.33
#> - tot_rev_debt 1 839.33 879.33
#> - rev_util 1 844.21 884.21
#> - ltv 1 848.13 888.13
#> - fico_score 1 853.94 893.94
#>
#> Step: AIC=871.27
#> bad_ind ~ vehicle_year + vehicle_make + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_tr + tot_rev_debt +
#> tot_rev_line + rev_util + fico_score + purch_price + msrp +
#> down_pyt + loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - purch_price 1 831.40 869.40
#> - msrp 1 832.05 870.05
#> - tot_tr 1 832.12 870.12
#> - tot_open_tr 1 832.40 870.40
#> - vehicle_year 1 832.95 870.95
#> - tot_rev_line 1 833.09 871.09
#> <none> 831.27 871.27
#> - tot_income 1 833.94 871.94
#> - vehicle_make 1 834.31 872.31
#> - down_pyt 1 834.90 872.90
#> - loan_term 1 835.25 873.25
#> + application_id 1 831.27 873.27
#> - tot_rev_tr 1 836.29 874.29
#> - veh_mileage 1 836.68 874.68
#> - loan_amt 1 836.72 874.72
#> - age_oldest_tr 1 837.63 875.63
#> - tot_derog 1 838.33 876.33
#> - tot_rev_debt 1 839.34 877.34
#> - rev_util 1 844.23 882.23
#> - ltv 1 848.17 886.17
#> - fico_score 1 853.98 891.98
#>
#> Step: AIC=869.4
#> bad_ind ~ vehicle_year + vehicle_make + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_tr + tot_rev_debt +
#> tot_rev_line + rev_util + fico_score + msrp + down_pyt +
#> loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - tot_tr 1 832.30 868.30
#> - tot_open_tr 1 832.56 868.56
#> - msrp 1 832.67 868.67
#> - vehicle_year 1 833.01 869.01
#> - tot_rev_line 1 833.23 869.23
#> <none> 831.40 869.40
#> - tot_income 1 834.19 870.19
#> - vehicle_make 1 834.44 870.44
#> - down_pyt 1 835.01 871.01
#> + purch_price 1 831.27 871.27
#> + application_id 1 831.40 871.40
#> - loan_term 1 835.48 871.48
#> - tot_rev_tr 1 836.36 872.36
#> - veh_mileage 1 836.91 872.91
#> - age_oldest_tr 1 837.75 873.75
#> - tot_derog 1 838.40 874.40
#> - loan_amt 1 838.49 874.49
#> - tot_rev_debt 1 839.57 875.57
#> - rev_util 1 844.38 880.38
#> - ltv 1 848.45 884.45
#> - fico_score 1 854.27 890.27
#>
#> Step: AIC=868.3
#> bad_ind ~ vehicle_year + vehicle_make + tot_derog + age_oldest_tr +
#> tot_open_tr + tot_rev_tr + tot_rev_debt + tot_rev_line +
#> rev_util + fico_score + msrp + down_pyt + loan_term + loan_amt +
#> ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - tot_open_tr 1 833.16 867.16
#> - msrp 1 833.62 867.62
#> - vehicle_year 1 833.94 867.94
#> <none> 832.30 868.30
#> - vehicle_make 1 835.20 869.20
#> + tot_tr 1 831.40 869.40
#> - tot_income 1 835.52 869.52
#> - tot_rev_line 1 835.68 869.68
#> - down_pyt 1 835.93 869.93
#> + purch_price 1 832.12 870.12
#> + application_id 1 832.30 870.30
#> - loan_term 1 836.40 870.40
#> - tot_rev_tr 1 837.03 871.03
#> - veh_mileage 1 837.89 871.89
#> - tot_derog 1 838.45 872.45
#> - loan_amt 1 839.30 873.30
#> - age_oldest_tr 1 840.67 874.67
#> - tot_rev_debt 1 840.80 874.80
#> - rev_util 1 845.37 879.37
#> - ltv 1 849.58 883.58
#> - fico_score 1 854.62 888.62
#>
#> Step: AIC=867.16
#> bad_ind ~ vehicle_year + vehicle_make + tot_derog + age_oldest_tr +
#> tot_rev_tr + tot_rev_debt + tot_rev_line + rev_util + fico_score +
#> msrp + down_pyt + loan_term + loan_amt + ltv + tot_income +
#> veh_mileage
#>
#> Df Deviance AIC
#> - msrp 1 834.49 866.49
#> - vehicle_year 1 835.01 867.01
#> <none> 833.16 867.16
#> - tot_income 1 836.11 868.11
#> - vehicle_make 1 836.14 868.14
#> - tot_rev_line 1 836.20 868.20
#> + tot_open_tr 1 832.30 868.30
#> + tot_tr 1 832.56 868.56
#> - down_pyt 1 836.77 868.77
#> + purch_price 1 832.96 868.96
#> - tot_rev_tr 1 837.03 869.03
#> + application_id 1 833.16 869.16
#> - loan_term 1 837.25 869.25
#> - veh_mileage 1 838.82 870.82
#> - tot_derog 1 839.20 871.20
#> - loan_amt 1 839.94 871.94
#> - tot_rev_debt 1 841.17 873.17
#> - age_oldest_tr 1 841.29 873.29
#> - rev_util 1 847.28 879.28
#> - ltv 1 850.90 882.90
#> - fico_score 1 855.29 887.29
#>
#> Step: AIC=866.49
#> bad_ind ~ vehicle_year + vehicle_make + tot_derog + age_oldest_tr +
#> tot_rev_tr + tot_rev_debt + tot_rev_line + rev_util + fico_score +
#> down_pyt + loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - vehicle_year 1 835.55 865.55
#> <none> 834.49 866.49
#> + msrp 1 833.16 867.16
#> + tot_open_tr 1 833.62 867.62
#> - vehicle_make 1 837.66 867.66
#> - tot_rev_line 1 837.67 867.67
#> - tot_income 1 837.68 867.68
#> + purch_price 1 833.71 867.71
#> + tot_tr 1 833.85 867.85
#> - tot_rev_tr 1 838.25 868.25
#> + application_id 1 834.49 868.49
#> - down_pyt 1 838.76 868.76
#> - loan_term 1 839.15 869.15
#> - veh_mileage 1 840.23 870.23
#> - tot_derog 1 840.24 870.24
#> - age_oldest_tr 1 842.58 872.58
#> - tot_rev_debt 1 842.89 872.89
#> - loan_amt 1 843.20 873.20
#> - rev_util 1 848.45 878.45
#> - ltv 1 851.91 881.91
#> - fico_score 1 857.15 887.15
#>
#> Step: AIC=865.55
#> bad_ind ~ vehicle_make + tot_derog + age_oldest_tr + tot_rev_tr +
#> tot_rev_debt + tot_rev_line + rev_util + fico_score + down_pyt +
#> loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> <none> 835.55 865.55
#> + vehicle_year 1 834.49 866.49
#> + tot_open_tr 1 834.51 866.51
#> - vehicle_make 1 838.58 866.58
#> - tot_rev_line 1 838.60 866.60
#> - tot_income 1 838.75 866.75
#> + tot_tr 1 834.93 866.93
#> + msrp 1 835.01 867.01
#> + purch_price 1 835.15 867.15
#> - tot_rev_tr 1 839.49 867.49
#> + application_id 1 835.55 867.55
#> - down_pyt 1 839.62 867.62
#> - loan_term 1 840.07 868.07
#> - veh_mileage 1 840.39 868.39
#> - tot_derog 1 841.35 869.35
#> - loan_amt 1 843.51 871.51
#> - age_oldest_tr 1 843.60 871.60
#> - tot_rev_debt 1 843.94 871.94
#> - rev_util 1 849.68 877.68
#> - ltv 1 853.00 881.00
#> - fico_score 1 857.67 885.67
#Step-by-Step Modeling
##step 1: Import Data
accepts <- read.csv(system.file("extdata", "accepts.csv", package = "autoScorecard" ))
##step 2: Data Description
data_detect1 <- data_detect( df = accepts, key_var = c("application_id","account_number") ,
y_var = "bad_ind" )
head(data_detect1)
#> variable class nrow missing_rate unique_count identical_rate min
#> 1 vehicle_year integer 5845 0.0001710864 21 0.356372968 0
#> 2 vehicle_make character 5845 0.0000000000 155 0.190248075 NA
#> 3 bankruptcy_ind character 5845 0.0000000000 3 0.886227545 NA
#> 4 tot_derog integer 5845 0.0364414029 30 0.488793841 0
#> 5 tot_tr integer 5845 0.0364414029 68 0.039349872 0
#> 6 age_oldest_tr integer 5845 0.0369546621 461 0.007869974 1
#> p25 p50 p75 max mean sd cv
#> 1 1997 1999 2000 9999 1901.793634 488.024392 0.2566127
#> 2 NA NA NA NA NA NA NA
#> 3 NA NA NA NA NA NA NA
#> 4 0 0 2 32 1.910156 3.274744 1.7143852
#> 5 9 16 24 77 17.084695 10.814056 0.6329675
#> 6 78 137 205 588 154.304317 99.940540 0.6476847
##step 3: Data Filtering
feature<- filter_var( df = accepts , key_var = c("application_id","account_number"), y_var = "bad_ind" ,
missing_rate = 0 , single_var_rate = 1 ,
iv_set = 0.02, char_to_number = TRUE , na.omit = TRUE )
##step 4: Select Training Sample
d = sort( sample( nrow( feature ), nrow( feature )*0.7))
train <- feature[d,]
test <- feature[-d,]
##step 5: Data Distribution Comparison
comparison_two_data( df1 = train , df2 = test ,key_var = c("application_id","account_number"), y_var="bad_ind")
##step 6: Automatic binning of data
##Decision Tree Binning
treebins_train <- bins_tree(df= train, key_var= c("application_id","account_number"), y_var="bad_ind",
max_depth=3, p=0.1)
##Equal Frequency Binning
binning_eqfreq1 <- binning_eqfreq( df= train, feat= 'tot_derog', label = 'bad_ind', nbins = 3)
##Equal Width Binning
binning_eqwid1 <- binning_eqwid( df = train, feat = 'tot_derog', label = 'bad_ind', nbins = 3)
##The K-means Binning
binning_kmean1 <- binning_kmean( df = train, feat= 'loan_term', label = 'bad_ind', nbins = 3)
##Chi-Square Binning
bins_chim1 <- bins_chim( df = train[1:200,], key_var = "application_id", y_var = "bad_ind" , alpha=0.1 )
##Unsupervised Automatic Binning Function
f_1 <-bins_unsupervised( df = feature[1:200,] , id="application_id" , label="bad_ind" ,
methods = c("k_means", "equal_width","equal_freq") , bin_nums=5 )
##The Combination of Two Bins Produces the Best Binning Result
best1 <- best_iv( df=f_1 ,bin=c('bins') , method = c('method') ,variable= c( "variable" ) ,label_iv='miv' )
vs1 <- best_vs( df1 = treebins_train[,-c(3)], df2 = best1[,-c(1:2)] ,variable="variable" ,label_iv='miv' )
##step 7: Replace Feature Data by Binning Template
woe_train <- rep_woe( df= train ,key_var="application_id", y_var="bad_ind" , tool=treebins_train ,
var_label= "variable",col_woe='woe', lower='lower' ,upper ='upper' )
woe_test <- rep_woe( df= test ,key_var="application_id", y_var="bad_ind" , tool=treebins_train ,
var_label= "variable",col_woe='woe', lower='lower' ,upper ='upper' )
##step 8: Modeling
lg<-stats::glm(bad_ind~.,family=stats::binomial(link='logit'),data= woe_train)
lg_both<-stats::step(lg,direction = "both")
#> Start: AIC=2478.97
#> bad_ind ~ application_id + account_number + vehicle_year + vehicle_make +
#> tot_derog + tot_tr + age_oldest_tr + tot_open_tr + tot_rev_tr +
#> tot_rev_debt + tot_rev_line + rev_util + fico_score + purch_price +
#> msrp + down_pyt + loan_term + loan_amt + ltv + tot_income +
#> veh_mileage
#>
#> Df Deviance AIC
#> - account_number 1 2435.0 2477.0
#> - vehicle_make 1 2435.0 2477.0
#> - tot_rev_tr 1 2435.0 2477.0
#> - msrp 1 2435.3 2477.3
#> - tot_income 1 2436.3 2478.3
#> - application_id 1 2436.9 2478.9
#> <none> 2435.0 2479.0
#> - tot_tr 1 2437.0 2479.0
#> - age_oldest_tr 1 2437.4 2479.4
#> - tot_rev_debt 1 2437.6 2479.6
#> - tot_open_tr 1 2438.4 2480.4
#> - tot_rev_line 1 2438.5 2480.5
#> - loan_term 1 2438.6 2480.6
#> - purch_price 1 2439.1 2481.1
#> - loan_amt 1 2440.4 2482.4
#> - rev_util 1 2441.5 2483.5
#> - tot_derog 1 2442.1 2484.1
#> - vehicle_year 1 2442.4 2484.4
#> - down_pyt 1 2443.7 2485.7
#> - veh_mileage 1 2451.1 2493.1
#> - ltv 1 2475.7 2517.7
#> - fico_score 1 2513.7 2555.7
#>
#> Step: AIC=2476.98
#> bad_ind ~ application_id + vehicle_year + vehicle_make + tot_derog +
#> tot_tr + age_oldest_tr + tot_open_tr + tot_rev_tr + tot_rev_debt +
#> tot_rev_line + rev_util + fico_score + purch_price + msrp +
#> down_pyt + loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - vehicle_make 1 2435.0 2475.0
#> - tot_rev_tr 1 2435.0 2475.0
#> - msrp 1 2435.3 2475.3
#> - tot_income 1 2436.3 2476.3
#> - application_id 1 2436.9 2476.9
#> <none> 2435.0 2477.0
#> - tot_tr 1 2437.0 2477.0
#> - age_oldest_tr 1 2437.4 2477.4
#> - tot_rev_debt 1 2437.6 2477.6
#> - tot_open_tr 1 2438.4 2478.4
#> - tot_rev_line 1 2438.6 2478.6
#> - loan_term 1 2438.7 2478.7
#> + account_number 1 2435.0 2479.0
#> - purch_price 1 2439.2 2479.2
#> - loan_amt 1 2440.5 2480.5
#> - rev_util 1 2441.6 2481.6
#> - tot_derog 1 2442.1 2482.1
#> - vehicle_year 1 2442.4 2482.4
#> - down_pyt 1 2443.7 2483.7
#> - veh_mileage 1 2451.1 2491.1
#> - ltv 1 2475.8 2515.8
#> - fico_score 1 2513.9 2553.9
#>
#> Step: AIC=2475
#> bad_ind ~ application_id + vehicle_year + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_tr + tot_rev_debt +
#> tot_rev_line + rev_util + fico_score + purch_price + msrp +
#> down_pyt + loan_term + loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - tot_rev_tr 1 2435.0 2473.0
#> - msrp 1 2435.3 2473.3
#> - tot_income 1 2436.3 2474.3
#> - application_id 1 2437.0 2475.0
#> <none> 2435.0 2475.0
#> - tot_tr 1 2437.0 2475.0
#> - age_oldest_tr 1 2437.4 2475.4
#> - tot_rev_debt 1 2437.6 2475.6
#> - tot_open_tr 1 2438.5 2476.5
#> - tot_rev_line 1 2438.6 2476.6
#> - loan_term 1 2438.7 2476.7
#> + vehicle_make 1 2435.0 2477.0
#> + account_number 1 2435.0 2477.0
#> - purch_price 1 2439.2 2477.2
#> - loan_amt 1 2440.5 2478.5
#> - rev_util 1 2441.6 2479.6
#> - tot_derog 1 2442.1 2480.1
#> - vehicle_year 1 2442.6 2480.6
#> - down_pyt 1 2443.7 2481.7
#> - veh_mileage 1 2451.2 2489.2
#> - ltv 1 2476.1 2514.1
#> - fico_score 1 2514.1 2552.1
#>
#> Step: AIC=2473.04
#> bad_ind ~ application_id + vehicle_year + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_debt + tot_rev_line +
#> rev_util + fico_score + purch_price + msrp + down_pyt + loan_term +
#> loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - msrp 1 2435.4 2471.4
#> - tot_income 1 2436.3 2472.3
#> - tot_tr 1 2437.0 2473.0
#> - application_id 1 2437.0 2473.0
#> <none> 2435.0 2473.0
#> - age_oldest_tr 1 2437.5 2473.5
#> - tot_rev_debt 1 2437.7 2473.7
#> - tot_open_tr 1 2438.6 2474.6
#> - tot_rev_line 1 2438.7 2474.7
#> - loan_term 1 2438.8 2474.8
#> + tot_rev_tr 1 2435.0 2475.0
#> + vehicle_make 1 2435.0 2475.0
#> + account_number 1 2435.0 2475.0
#> - purch_price 1 2439.2 2475.2
#> - loan_amt 1 2440.5 2476.5
#> - tot_derog 1 2442.1 2478.1
#> - rev_util 1 2442.4 2478.4
#> - vehicle_year 1 2442.6 2478.6
#> - down_pyt 1 2443.8 2479.8
#> - veh_mileage 1 2451.2 2487.2
#> - ltv 1 2476.2 2512.2
#> - fico_score 1 2515.8 2551.8
#>
#> Step: AIC=2471.39
#> bad_ind ~ application_id + vehicle_year + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_debt + tot_rev_line +
#> rev_util + fico_score + purch_price + down_pyt + loan_term +
#> loan_amt + ltv + tot_income + veh_mileage
#>
#> Df Deviance AIC
#> - tot_income 1 2436.6 2470.6
#> - tot_tr 1 2437.3 2471.3
#> - application_id 1 2437.4 2471.4
#> <none> 2435.4 2471.4
#> - age_oldest_tr 1 2437.8 2471.8
#> - tot_rev_debt 1 2438.0 2472.0
#> - tot_open_tr 1 2438.9 2472.9
#> - loan_term 1 2439.0 2473.0
#> + msrp 1 2435.0 2473.0
#> - tot_rev_line 1 2439.1 2473.1
#> - purch_price 1 2439.3 2473.3
#> + tot_rev_tr 1 2435.3 2473.3
#> + vehicle_make 1 2435.4 2473.4
#> + account_number 1 2435.4 2473.4
#> - loan_amt 1 2440.6 2474.6
#> - tot_derog 1 2442.5 2476.5
#> - rev_util 1 2442.7 2476.7
#> - vehicle_year 1 2443.3 2477.3
#> - down_pyt 1 2444.3 2478.3
#> - veh_mileage 1 2451.7 2485.7
#> - ltv 1 2476.2 2510.2
#> - fico_score 1 2515.9 2549.9
#>
#> Step: AIC=2470.6
#> bad_ind ~ application_id + vehicle_year + tot_derog + tot_tr +
#> age_oldest_tr + tot_open_tr + tot_rev_debt + tot_rev_line +
#> rev_util + fico_score + purch_price + down_pyt + loan_term +
#> loan_amt + ltv + veh_mileage
#>
#> Df Deviance AIC
#> <none> 2436.6 2470.6
#> - application_id 1 2438.7 2470.7
#> - tot_tr 1 2438.8 2470.8
#> - age_oldest_tr 1 2439.3 2471.3
#> - tot_rev_debt 1 2439.3 2471.3
#> + tot_income 1 2435.4 2471.4
#> - tot_open_tr 1 2440.2 2472.2
#> - loan_term 1 2440.3 2472.3
#> + msrp 1 2436.3 2472.3
#> + tot_rev_tr 1 2436.6 2472.6
#> + vehicle_make 1 2436.6 2472.6
#> + account_number 1 2436.6 2472.6
#> - tot_rev_line 1 2440.8 2472.8
#> - purch_price 1 2441.0 2473.0
#> - loan_amt 1 2441.6 2473.6
#> - tot_derog 1 2443.6 2475.6
#> - rev_util 1 2443.6 2475.6
#> - vehicle_year 1 2444.1 2476.1
#> - down_pyt 1 2445.6 2477.6
#> - veh_mileage 1 2452.8 2484.8
#> - ltv 1 2478.4 2510.4
#> - fico_score 1 2517.9 2549.9
logit<-stats::predict(lg_both,woe_test)
woe_test$lg_both_p<-exp(logit)/(1+exp(logit))
pred_both <- ROCR::prediction(woe_test$lg_both_p, woe_test$bad_ind)
perf_both <- ROCR::performance(pred_both,"tpr","fpr")
##step 9: Correlation Diagram
coe = (lg_both$coefficients)
cor1<-stats::cor( Xvar_df<- woe_train[-which(colnames(feature) %in% c("application_id","account_number","bad_ind"))])
corrplot::corrplot(cor1)
corrplot::corrplot(cor1,method = "number")
##step 10: Manually Input Parameters to Generate Scorecards
##scorecard
Score<-noauto_scorecard( bins_card= woe_test, fit= lg_both,bins_woe=treebins_train ,points0 = 600,
odds0 = 1/20, pdo = 50 ,k = 2)
Score_2<-noauto_scorecard( bins_card= woe_train, fit= lg_both,bins_woe=treebins_train ,points0 = 600,
odds0 = 1/20, pdo = 50 ,k = 3)
##scorecard2
Score<-noauto_scorecard2( bins_card= woe_test, fit= lg_both,bins_woe=treebins_train ,points0 = 600,
odds0 = 1/20, pdo = 50 ,k = 2)
Score_2<-noauto_scorecard2( bins_card= woe_train, fit= lg_both,bins_woe=treebins_train ,points0 = 600,
odds0 = 1/20, pdo = 50 ,k = 3)
##step 11: PSI
data_train <- Score_2$data_score
data_test <- Score$data_score
psi_1<-psi_cal( df_train = data_train , df_test = data_test,feat='Score',label='bad_ind' , nbins=10)
##step 12: Data Painter
plot_board( label= woe_test$bad_ind, pred = woe_test$lg_both_p )
