昨天工作：
dssm网络效果调优（40w数据量）。现在的效果是在网络训练过程中（acc:0.966， auc：0.884），测试阶段（acc:0.834， auc：0.884），训练结果良好，但是测试的结果还有问题，正在寻找出现该问题的原因。
今天计划：
测试阶段acc过低问题的定位和解决。使用dssm网络进行inference效果测试。
问题解决：
batch normalization的使用有问题，在训练阶段需要自动更新它的均值方差，在test阶段才能使用。具体细节参考下面的第0条。
遇到的问题：
https://www.gitmemory.com/issue/tensorflow/hub/44/494272642
batch normalization在test的时候的使用问题：
https://blog.csdn.net/huitailangyz/article/details/85015611batch normalization 问题的解决：
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="train_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_train_data_dssm_3,odps://graph_embedding/tables/hs_test_data_dssm_3" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=1000 --ckpt=hs_ugc_video_40w.ckpt" -DuseSparseClusterSchema=True;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="train_v4.py" -Dcluster='{"worker":{"count":30, "cpu":200, "memory":4000}, "ps":{"count":10, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_train_data_dssm_2,odps://graph_embedding/tables/hs_test_data_dssm_2" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt" -DuseSparseClusterSchema=True;
without update:训练正常，测试全零
with update:训练测试正常
修改后：使用40w数据进行训练测试
使用70y数据进行训练测试：
注：test的precision和f1_score低是因为/step->/step_test
inference 指令：
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_tmp_54" -Doutputs="odps://graph_embedding/tables/hs_tmp_56" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt-1" -DuseSparseClusterSchema=True;
之己给的参考：
pai -name tensorflow140 -Dscript="file:///Users/xuejinbao/zhiji/graph/query_co_video/xhs_dssm.tar.gz" -DentryFile="inference.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":10, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/zj_xhs_video_topic_infos_" -Doutputs="odps://graph_embedding/tables/zj_xhs_video_topic_emb_" -DcheckpointDir="oss://bucket-automl/xhs_video/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=1e-2 --batch_size=1024 --attention_type=1 --ckpt=xhs_video.ckpt-1" -DuseSparseClusterSchema=True;
- 准备inference数据
得到最新分区的数据
create table hs_tmp_57 as
select coalesce(get_json_object(body, '.entities.k0.item_id/l'),get_json_object(body, '
.entities.k3.item_id/l') ,get_json_object(body, '
.entities.k5.item_id/l'), get_json_object(body, '
.entities.k7.item_id/l'))as id, coalesce(get_json_object(body, '
.entities.k1.title/s'), get_json_object(body, '
.entities.k3.title/s'), get_json_object(body, '
.entities.k5.title/s'), get_json_object(body, '
.entities.k7.title/s')) as words from graph_embedding.jl_jingyan_query_related_video_pool where ds=max_pt('graph_embedding.jl_jingyan_query_related_video_pool') and type_biz=2;
create table hs_tmp_58 as
select row_number()over() as id, query as words from
graph_embedding.jl_jingyan_query_related_top_query where ds=max_pt('graph_embedding.jl_jingyan_query_related_top_query');
使用主搜进行分词
create table if not exists hs_tmp_59 LIFECYCLE 20 as select id, words, search_kg:alinlp_segment(words, "MAINSE", "0", "1") as words_mainse_ws from hs_tmp_57;
create table if not exists hs_tmp_60 LIFECYCLE 20 as select id, words, search_kg:alinlp_segment(words, "MAINSE", "0", "1") as words_mainse_ws from hs_tmp_58;
去除低频词:hs_tmp_dssm_inf_titles; hs_tmp_dssm_inf_querys
PAI -name FilterNoise -project algo_public
-DinputTableName=graph_embedding.hs_tmp_59
-DnoiseTableName=graph_embedding.hs_dirty_words_info_
-DoutputTableName=graph_embedding.hs_tmp_dssm_inf_titles
-DselectedColNames="words_mainse_ws"
-Dlifecycle=30;
PAI -name FilterNoise -project algo_public
-DinputTableName=graph_embedding.hs_tmp_60
-DnoiseTableName=graph_embedding.hs_dirty_words_info_
-DoutputTableName=graph_embedding.hs_tmp_dssm_inf_querys
-DselectedColNames="words_mainse_ws"
-Dlifecycle=30;
inference阶段 hs_dssm_result_query_0; hs_dssm_result_title_0
truncate table hs_dssm_result_query_0;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_tmp_dssm_inf_querys" -Doutputs="odps://graph_embedding/tables/hs_dssm_result_query_0" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt-1" -DuseSparseClusterSchema=True;
- knn求得最终结果
PAI -name am_vsearch_nearest_neighbor_014 -project algo_market
-Dcluster="{"worker":{"count":1,"gpu":100}}"
-Ddim=100
-Did_col="index"
-Dvector_col="words_mainse_emb"
-Dinput_slice=1
-Dtopk=50
-Dnprob=1024
-Dmetric="l2"
-Dinput="odps://graph_embedding/tables/hs_tmp_64"
-Dquery="odps://graph_embedding/tables/hs_dssm_result_query_1"
-Doutputs="odps://graph_embedding/tables/hs_dssm_result_0"
-DenableDynamicCluster=true -DmaxTrainingTimeInHour=60;
create table hs_dssm_result_query_1 as select distinct * from hs_dssm_result_query_0;
create table hs_tmp_61 as select bi_udf:bi_split_value(query, title, ",") as (query_id, item_id) from hs_dssm_result_1;
- 看一下出来的结果有没有什么问题
drop table hs_tmp_62;
yes
create table hs_tmp_62 as select bi_udf:bi_split_value(id, words_mainse_emb, ",") as (index, query_word) from hs_dssm_result_query_1;
drop table hs_tmp_63;
yes
create table hs_tmp_63 as select index, count(*) as freq from hs_tmp_62 group by index order by freq desc;
对结果进行修改之后，应该没有问题了，但是不知道为什么还是报错：
看来还是使用网络直接得到分数更合适啊。。。
- 根据group结果取前k大：
SELECT
*
FROM
yourtable
WHERE
id IN (SELECT
SUBSTRING_INDEX(GROUP_CONCAT(id
ORDER BY rate DESC),
',',
1) id
FROM
yourtable
GROUP BY year)
ORDER BY rate DESC;
