2019华为算法精英赛(CTR)思路

第一次打数据比赛，排名就不说了/(ㄒoㄒ)/~~比赛过程中学到了很多pandas姿势。。

Contents

1 比赛链接

比赛链接

https://developer.huawei.com/consumer/cn/devservice/activity/devStarAI/info/712

1.赛题分析

题目数据

文件	描述	数据量
train_20190518.csv	训练集(有标签)	9,837,655(正样本)/159,837,655(总数)
test_20190518.csv	测试集(无标签)	1,000,000
user_info.csv	用户资料	51,743,538
ad_info.csv	广告资料	5223
content_info.csv	广告媒体资料	2321

train_20190518.csv  # 训练集
'''
label,用户id,        广告id， 操作时间                ，媒体id，广告位id，素材id，网络连接类型
['0', 'u142261668', '3918', '2019-03-28 20:31:52.350', '10', '10', '4686', '1']
['0', 'u135356337', '5119', '2019-03-29 04:50:20.672', '10', '12', '5878', '1']
['0', 'u100026472', '1720', '2019-03-28 21:54:31.673', '10', '30', '1824', '1']
['0', 'u109322182', '1461', '2019-03-31 19:38:43.852', '3', '32', '1372', '1']
['0', 'u149083403', '1384', '2019-03-30 05:51:11.364', '12', '49', '1142', '1']
'''

test_20190518.csv  # 测试集
'''
序号，用户id,      广告id， 操作时间                ，媒体id，广告位id，素材id，网络连接类型
['1', 'u100907139', '209', '2019-04-10 23:59:59.577', '10', '8', '265', '4']
['2', 'u136045483', '3574', '2019-04-10 23:59:59.455', '3', '16', '4197', '1']
['3', 'u126170991', '503', '2019-04-10 23:59:59.297', '3', '27', '538', '1']
['4', 'u103341727', '960', '2019-04-10 23:59:59.207', '8', '88', '821', '1']
['5', 'u118663766', '1576', '2019-04-10 23:59:58.999', '10', '8', '1507', '1']
'''

user_info.csv  # 用户信息
'''
id，            年龄段， 性别， 城市， 省份， 设备型号，运营商编号
['u129897413', 'NULL', 'NULL', 'NULL', 'NULL', '487', '']
['u108906458', '4', '1', '125', '28', '217', '2']
['u125870461', '3', '1', 'NULL', 'NULL', '474', '3']
['u133924361', 'NULL', 'NULL', 'NULL', 'NULL', '502', '2']
['u133924360', 'NULL', 'NULL', 'NULL', 'NULL', '502', '3']
['u133924359', '3', '3', '184', '34', '480', '2']
['u137951263', 'NULL', 'NULL', 'NULL', 'NULL', '112', '']
'''

ad_info.csv  # 广告信息
'''
id, 计费类型,广告主id,创意id,交互类型,广告对应的appid 
['5', 'CPC', '224', '3', '2', 'NULL']
['4', 'CPC', '2', '3', '2', 'NULL']
['3', 'CPC', '2', '3', '2', 'NULL']
['2', 'CPC', '2', '3', '2', 'NULL']
['1', 'CPM', '15', '2', '1', 'NULL']
'''

content_info.csv  # 内容分类
'''
内容id，一级分类，二级分类
['5198', '科技', 'NULL']
['5197', '家居', '家居#装修']
['5196', '家居', '家居#装修']
['5192', '游戏', '手游']
['5191', '社会', 'NULL']
['5190', '科技', '软件']
['5189', '家居', '家居#家具']
['5188', '社会', '购房']
'''

train_20190518.csv # 训练集

'''

label,用户id, 广告id，操作时间，媒体id，广告位id，素材id，网络连接类型

['0', 'u142261668', '3918', '2019-03-28 20:31:52.350', '10', '10', '4686', '1']

['0', 'u135356337', '5119', '2019-03-29 04:50:20.672', '10', '12', '5878', '1']

['0', 'u100026472', '1720', '2019-03-28 21:54:31.673', '10', '30', '1824', '1']

['0', 'u109322182', '1461', '2019-03-31 19:38:43.852', '3', '32', '1372', '1']

['0', 'u149083403', '1384', '2019-03-30 05:51:11.364', '12', '49', '1142', '1']

'''

test_20190518.csv # 测试集

'''

序号，用户id, 广告id，操作时间，媒体id，广告位id，素材id，网络连接类型

['1', 'u100907139', '209', '2019-04-10 23:59:59.577', '10', '8', '265', '4']

['2', 'u136045483', '3574', '2019-04-10 23:59:59.455', '3', '16', '4197', '1']

['3', 'u126170991', '503', '2019-04-10 23:59:59.297', '3', '27', '538', '1']

['4', 'u103341727', '960', '2019-04-10 23:59:59.207', '8', '88', '821', '1']

['5', 'u118663766', '1576', '2019-04-10 23:59:58.999', '10', '8', '1507', '1']

'''

user_info.csv # 用户信息

'''

id，年龄段，性别，城市，省份，设备型号，运营商编号

['u129897413', 'NULL', 'NULL', 'NULL', 'NULL', '487', '']

['u108906458', '4', '1', '125', '28', '217', '2']

['u125870461', '3', '1', 'NULL', 'NULL', '474', '3']

['u133924361', 'NULL', 'NULL', 'NULL', 'NULL', '502', '2']

['u133924360', 'NULL', 'NULL', 'NULL', 'NULL', '502', '3']

['u133924359', '3', '3', '184', '34', '480', '2']

['u137951263', 'NULL', 'NULL', 'NULL', 'NULL', '112', '']

'''

ad_info.csv # 广告信息

'''

id, 计费类型,广告主id,创意id,交互类型,广告对应的appid

['5', 'CPC', '224', '3', '2', 'NULL']

['4', 'CPC', '2', '3', '2', 'NULL']

['3', 'CPC', '2', '3', '2', 'NULL']

['2', 'CPC', '2', '3', '2', 'NULL']

['1', 'CPM', '15', '2', '1', 'NULL']

'''

content_info.csv # 内容分类

'''

内容id，一级分类，二级分类

['5198', '科技', 'NULL']

['5197', '家居', '家居#装修']

['5196', '家居', '家居#装修']

['5192', '游戏', '手游']

['5191', '社会', 'NULL']

['5190', '科技', '软件']

['5189', '家居', '家居#家具']

['5188', '社会', '购房']

'''

不同时刻的曝光量和点击率变化

with open('train_20190518.csv', 'r',encoding='utf-8') as f:
    csv_file = csv.reader(f)

    i = 0
    all = np.zeros(24)
    no_clicks = np.zeros(24)
    for row in csv_file:
        time = row[3]
        time = int(time[11:13])
        click = row[0]
        if click == '0':
            no_clicks[time] = no_clicks[time] + 1
        all[time] = all[time] + 1
        #print(time)
        # print(type(click))
        #if click == '1':
        #    clicks = clicks + 1
        i = i + 1
        if i % 1000000 == 0:
            print('%d/100' % (i/1000000))
        if i == 100000000:
            break

    print(all)

    print(no_clicks)
    x_axis = np.arange(24)

    bar1 = plt.bar(x_axis, all, color='darkorange')
    bar2 = plt.bar(x_axis, no_clicks, color='steelblue')

    plt.legend([bar2, bar1], ["0", "1"], loc=1)

    plt.xticks(x_axis)
    plt.xlabel('hour')
    #plt.ylabel('')

    plt.show()

with open('train_20190518.csv', 'r',encoding='utf-8') as f:

csv_file = csv.reader(f)

i = 0

all = np.zeros(24)

no_clicks = np.zeros(24)

for row in csv_file:

time = row[3]

time = int(time[11:13])

click = row[0]

if click == '0':

no_clicks[time] = no_clicks[time] + 1

all[time] = all[time] + 1

#print(time)

# print(type(click))

#if click == '1':

# clicks = clicks + 1

i = i + 1

if i % 1000000 == 0:

print('%d/100' % (i/1000000))

if i == 100000000:

break

print(all)

print(no_clicks)

x_axis = np.arange(24)

bar1 = plt.bar(x_axis, all, color='darkorange')

bar2 = plt.bar(x_axis, no_clicks, color='steelblue')

plt.legend([bar2, bar1], ["0", "1"], loc=1)

plt.xticks(x_axis)

plt.xlabel('hour')

#plt.ylabel('')

plt.show()

数据预处理

1.merge

　　(1)一次性merge
　　pandas读csv的时候要加上header=None，因为这次比赛数据的csv文件不带表头。如果不加最后合并的数据会少一行(999999)，多谢群里大佬提醒！

# encoding=utf-8

import numpy as np
import csv
import  pandas as pd

train_df = pd.read_csv('train_20190518.csv',header=None)
train_df.columns = ['index', 'uid', 'aid', 'time', 'media', 'loc', 'cid', 'net']

user_feature = pd.read_csv('user_info.csv',header=None)
user_feature.columns = ['uid', 'age', 'sex', 'city', 'province', 'device', 'service']

train = pd.merge(train_df, user_feature, on='uid', how='left')

train.to_csv("train_user.csv", index=False, sep=',',header=None)

# encoding=utf-8

import numpy as np

import csv

import pandas as pd

train_df = pd.read_csv('train_20190518.csv',header=None)

train_df.columns = ['index', 'uid', 'aid', 'time', 'media', 'loc', 'cid', 'net']

user_feature = pd.read_csv('user_info.csv',header=None)

user_feature.columns = ['uid', 'age', 'sex', 'city', 'province', 'device', 'service']

train = pd.merge(train_df, user_feature, on='uid', how='left')

train.to_csv("train_user.csv", index=False, sep=',',header=None)

　　这种方法需要60G以上空闲内存(两两merge)。如果一次性merge所有表需要100G以上内存。

　　(2)抽样

　　1. (hint给的方法)在user表中随机抽取一部分用户，然后按顺序扫描train表，如果在抽取的用户中就merge并加入训练集，构建一个较小的训练集训练。
　　2. 在抽取的过程中要控制正负样本的比例。
　　3. 抽取的用户数量大小取决于内存的大小。
　　4. 这种方法不能全面地反映train集的特性。

2.空缺值/异常值处理

train_df = train_df.fillna('0')

1 2	train_df = train_df.fillna('0')

3.字段处理

　　1. 字符串类型的全部转成id
　　2. 时间提取小时即可
　　3. content_info中的分类建立dictionary，按照索引号转换成单id的问题

# -*- coding: utf-8 -*-
import csv
import pickle

with open('content_info.csv', 'r', encoding='utf-8') as f:
    csv_file = csv.reader(f)
    '''
    读取的csv_file是一个iterator，每个元素代表一行
    '''
    d1 = dict()
    d2 = dict()
    c1 = 0
    c2 = 0
    with open('content_info_id.csv', 'w', newline='') as f3:
        csv_writer = csv.writer(f3)

        for row in csv_file:
            r1 = (row[1])
            if r1 not in d1:
                d1[r1] = c1
                c1 = c1 + 1
            row[1] = d1[row[1]]

            r2 = (row[2])
            if r2 not in d2:
                d2[r2] = c2
                c2 = c2 + 1
            row[2] = d2[row[2]]
            print(row)
            csv_writer.writerow(row)

    #  用pickle保存映射关系
    with open('c1.pk', 'wb') as f1:
        pickle.dump(d1, f1)

    with open('c2.pk', 'wb') as f2:
        pickle.dump(d2, f2)

# -*- coding: utf-8 -*-

import csv

import pickle

with open('content_info.csv', 'r', encoding='utf-8') as f:

csv_file = csv.reader(f)

'''

读取的csv_file是一个iterator，每个元素代表一行

'''

d1 = dict()

d2 = dict()

c1 = 0

c2 = 0

with open('content_info_id.csv', 'w', newline='') as f3:

csv_writer = csv.writer(f3)

for row in csv_file:

r1 = (row[1])

if r1 not in d1:

d1[r1] = c1

c1 = c1 + 1

row[1] = d1[row[1]]

r2 = (row[2])

if r2 not in d2:

d2[r2] = c2

c2 = c2 + 1

row[2] = d2[row[2]]

print(row)

csv_writer.writerow(row)

# 用pickle保存映射关系

with open('c1.pk', 'wb') as f1:

pickle.dump(d1, f1)

with open('c2.pk', 'wb') as f2:

pickle.dump(d2, f2)

4.正负样本均衡

　　由于正样本只占总数的6%左右，正负样本严重不平衡。需要对负样本作降采样处理。我们把原来的训练集生成5个子训练集。算法为：正样本在每个子训练集中都会全部保留，负样本按顺序保留在子训练集1、2、3、4、5中。这5个子训练集会被用来训练5个模型，最后进行ensemble。

特征工程

统计特征

模型

NFFM
XDeepFM
LBA

评价指标

# AUC
# labels是标签，preds是预测的概率
fpr, tpr, thresholds = metrics.roc_curve(labels, preds, pos_label=1)
auc = metrics.auc(fpr, tpr)

# AUC

# labels是标签，preds是预测的概率

fpr, tpr, thresholds = metrics.roc_curve(labels, preds, pos_label=1)

auc = metrics.auc(fpr, tpr)

参考

科大讯飞ctr比赛冠军总结(知乎专栏)
https://zhuanlan.zhihu.com/p/47807544

推荐系统遇上深度学习(二十二)–DeepFM升级版XDeepFM模型强势来袭
https://www.jianshu.com/p/b4128bc79df0

Tensorflow实现ffm
https://www.jianshu.com/p/8b57473e385a

很有参考价值的merge、数据清洗、填充的pandas代码实现(腾讯2018广告算法大赛第10)
https://github.com/keyunluo/Tencent2018_Lookalike_Rank10th/blob/master/src/make_dataset.py

腾讯2018广告算法大赛第三
https://github.com/DiligentPanda/Tencent_Ads_Algo_2018/tree/master/scripts/merge

FFM的一些csdn文章
https://blog.csdn.net/John_xyz/article/details/78933253
https://blog.csdn.net/a940902940902/article/details/89320879#FMFFM_1

FFM推导(结合这个看论文)
https://www.cnblogs.com/zhangchaoyang/articles/8157893.html

FFM论文
https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf

ctr-net-tool(github)
https://github.com/guoday/ctrNet-tool

pandas官方文档
http://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html