论文二次处理流程

#!/bin/sh
root_dir=/home/minelab/liweibo
source_file=/home/minelab/cctv2014/data_warehouse/sina_weibo.data
conf_dir=$root_dir/conf
raw_dir=$root_dir/raw_data

#在raw_data目录下面建立各个节目名称命名的文件夹,同时建立各个节目下的关键词文件
#echo "make the program dir..."
#while read line
#do
#       rm -rf $raw_dir/$line
#       mkdir  $raw_dir/$line
#       cat $conf_dir/program_keywords | grep $line | awk -F‘\t‘ ‘{for(i=1;i<=NF;i++) print $i}‘> $raw_dir/$line/$line.filterwords
#       echo $line" mkdir and get filter words is done@!"
#done < $conf_dir/program.list

echo ‘get the candidate tweet for each program filtering by the keywords...‘
program_list=`ls $raw_dir`
for program in $program_list
do
        rm -rf $raw_dir/$program/$program.data
        rm -rf $raw_dir/$program/$program.uniq
        while read line
        do  
                cat $source_file | grep $line | awk -F‘\t‘ ‘{print $2"\t"$3"\t"$5"\t"$11"\t"$12"\t"$13"\t"$6}‘>> $raw_dir/$program/$program.data
        done < $raw_dir/$program/$program.filterwords
        echo $program "filtering is done!"
        #去除链接以及文本去重
        sed -i ‘1,$s/http:\/\/t\.cn\/[a-zA-Z0-9]\{4,9\}//g‘ $raw_dir/$program/$program.data 
        echo $program "remove url is done..."
        cat $raw_dir/$program/$program.data | sort -t ‘ ‘ -k 7 | uniq -f 6 > $raw_dir/$program/$program.uniq
        echo $program "uniq is done ..."
        wc -l $raw_dir/$program/$program.uniq > $raw_dir/$program/$program.statistic
        echo $program "statistic is done..."
done
echo "preData is done..."