【分箱操作】决策树、卡方、分位数、等距和映射分箱操作代码实现

Table of Contents

• 1  数据准备
• 2  决策树分箱
• 3  分位数分箱
• 4  等距和等距log映射
• 5  卡方分箱

import numpy as np
import pandas as pd
from sklearn.datasets import load_breast_cancer

import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt 
import seaborn as sns
%matplotlib inline
plt.rcParams["font.sans-serif"] = ["FangSong"] 
plt.rcParams["axes.unicode_minus"] = False 
import warnings
warnings.filterwarnings("ignore")

from sklearn.tree import DecisionTreeClassifier

数据准备

使用x作为待分箱数据
y为分箱的目标标签

data = load_breast_cancer()
df = pd.DataFrame(data.data, columns=data.feature_names)

variable = "mean radius"
x = df[variable].values
y = data.target

test = pd.DataFrame({'x':x,'y':y})

test.head()

sns.kdeplot(test.x)

sns.boxplot(test.x)

woe和iv的计算过程和理解可见：
https://blog.csdn.net/xiezhen_zheng/article/details/82888653

定义计算函数

def woe_iv(data, x_col='x', y_col='y'):'''data  为含有y和分箱标签的dataframex_col 为分箱标签y_col 为分类标签'''rate_table = test.groupby(x_col)[y_col].agg(['count', 'sum']).rename(columns={'count': 'total', 'sum': 'bad'})  # 对X进行加总，计算等到每一类型下的总数量和坏样本数量rate_table['good'] = rate_table['total'] - \rate_table['bad']  # 计算每一类型下好样本的数量total_bad = rate_table.sum()['bad']  # 计算坏样本总量total_good = rate_table.sum()['good']  # 计算好样本总量rate_table['p_bad'] = rate_table['bad']/total_bad  # 就是每一类型下坏样本的概率rate_table['p_good'] = rate_table['good']/total_good  # 就是每一类型下好样本的概率woe = np.log(rate_table['p_bad']/rate_table['p_good'])  # 就是每一类型的WOE值iv = (rate_table['p_bad'] - rate_table['p_good'])*woe  # 就是每一类的IV值rate_table['iv'] = ivrate_table['woe'] = woereturn rate_table

决策树分箱

def optimal_binning_boundary(data, x_col = 'x',y_col = 'y',criterion='gini') -> list:'''利用决策树进行分箱data  为含有y和分箱标签的dataframex_col 为分箱标签y_col 为分类标签criterion 为决策树分类准则'''x0 = data[x_col]x = data[x_col].valuesy = data[y_col].valuesboundary = []  # 待return的分箱边界值列表clf = DecisionTreeClassifier(criterion=criterion,  # 决策树分类准则max_leaf_nodes=6,       # 最大叶子节点数min_samples_leaf=0.05)  # 叶子节点样本数量最小占比clf.fit(x.reshape(-1, 1), y)  # 训练决策树n_nodes = clf.tree_.node_countchildren_left = clf.tree_.children_leftchildren_right = clf.tree_.children_rightthreshold = clf.tree_.thresholdfor i in range(n_nodes):if children_left[i] != children_right[i]:  # 获得决策树节点上的划分边界值boundary.append(threshold[i])boundary.sort()min_x = x.min()max_x = x.max()  boundary = [min_x] + boundary + [max_x]data['bins_dtc'] = pd.cut(x0, bins=boundary, include_lowest=True, labels=False)return boundary, data

boundary, bins = optimal_binning_boundary(test)

woe_iv(bins,'bins_dtc')

分位数分箱

使用pd.qcut函数

bins = pd.qcut(df[variable] ,5,labels=False,)

test['bins_q'] = bins

woe_iv(test,'bins_q')

等距和等距log映射

np.floor_divide（a,b） a除b再取整
bins = np.floor（） 向下取整

bins = np.floor_divide(df[variable] ,6)

test['bins_divide'] = bins 

woe_iv(test,'bins_divide')

bins = np.floor(np.log2(df[variable]))
test['bins_log'] = bins 
woe_iv(test,'bins_log')

卡方分箱

卡方分箱的具体思路为：
https://mp.weixin.qq.com/s?__biz=MzA5Njc1MDA2Ng%3D%3D&idx=1&mid=2651650083&sn=a24381efa404500ae96ccfcc3716a614

def Chi2(df, total_col, bad_col,overallRate):'''#此函数计算卡方值:df dataFrame:total_col 每个值得总数量:bad_col 每个值的坏数据数量:overallRate 坏数据的占比: return 卡方值'''df2=df.copy()df2['expected']=df[total_col].apply(lambda x: x*overallRate)combined=zip(df2['expected'], df2[bad_col])chi=[(i[0]-i[1])**2/i[0] for i in combined]chi2=sum(chi)return chi2

def chiMerge(data,x_col = 'x',y_col = 'y',max_bins = 5):gro = data.groupby(x_col)[y_col].agg(['mean', 'count'])gro['bad'] = gro['count']*gro['mean']total_rate_bad = gro.sum()['bad']/gro.sum()['count']gro['exp_bad'] = gro['count']*total_rate_badgro['chi2'] = ((gro['exp_bad'] - gro['bad']) ** 2)/gro['exp_bad']gro.drop('mean', 1, inplace=True)bad = list(gro['bad'])exp_bad = list(gro['exp_bad'])chi_c = np.array((bad,exp_bad)).Tinterval = [[i] for i in list(gro.index)]chi2 = list(gro['chi2'].values)while len(interval) >max_bins:between_sum = [chi2[i] + chi2[i+1] for i in range(len(chi2)-1)]chi2_min_index = between_sum.index(min(between_sum))interval[chi2_min_index] = interval[chi2_min_index] + interval[chi2_min_index+1]interval.pop(chi2_min_index+1)chi_c[chi2_min_index] = chi_c[chi2_min_index] + chi_c[chi2_min_index + 1]chi_c = np.delete(chi_c,chi2_min_index + 1,0)chi2[chi2_min_index] = ((chi_c[chi2_min_index][0] - chi_c[chi2_min_index][1])**2)/chi_c[chi2_min_index][1]chi2.pop(chi2_min_index+1)interval_ = [min(i) for i in interval] + [data[x_col].max()]bins = pd.cut(data[x_col],interval_,labels=False,include_lowest=True)return bins,interval_

inter = chiMerge(test)

bins = inter[0]

test['bins_kafang'] = bins

woe_iv(test,'bins_kafang')

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！