pil3aoekg
/
javac
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

持久化数据库完成

Browse Source
main
oeljeklaus-you 7 years ago
parent 69a0490f06
commit c2e142d595
5 changed files with 225 additions and 93 deletions
Show Stats Download Patch File Download Diff File

242
.idea/workspace.xml
Unescape View File

@ -2,12 +2,9 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a7505764-040b-48e2-b2fc-8c5b579e595f" name="Default" comment="">
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java" />
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java" />
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/domain/SessionDetail.java" />
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/domain/SessionRandomExtract.java" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/constant/Constants.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/constant/Constants.java" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java" />
</list>
<ignored path="$PROJECT_DIR$/out/" />
@ -24,27 +21,78 @@
<file leaf-file-name="UserVisitAnalyze.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="539">
<caret line="378" column="27" lean-forward="true" selection-start-line="378" selection-start-column="27" selection-end-line="378" selection-end-column="27" />
<state relative-caret-position="403">
<caret line="497" column="8" lean-forward="false" selection-start-line="497" selection-start-column="8" selection-end-line="497" selection-end-column="8" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#4882#5107#0" expanded="true" />
<element signature="e#5106#5107#0" expanded="true" />
<element signature="e#5337#8133#0" expanded="true" />
<element signature="e#8132#8133#0" expanded="true" />
<element signature="e#8353#8576#0" expanded="true" />
<element signature="e#8575#8576#0" expanded="true" />
<element signature="e#8829#9843#0" expanded="true" />
<element signature="e#9842#9843#0" expanded="true" />
<element signature="e#16022#16546#0" expanded="true" />
<element signature="e#16545#16546#0" expanded="true" />
<element signature="e#16787#16814#0" expanded="true" />
<element signature="e#17590#17623#0" expanded="true" />
<element signature="e#5059#5284#0" expanded="true" />
<element signature="e#5283#5284#0" expanded="true" />
<element signature="e#5514#8310#0" expanded="true" />
<element signature="e#8309#8310#0" expanded="true" />
<element signature="e#8530#8753#0" expanded="true" />
<element signature="e#8752#8753#0" expanded="true" />
<element signature="e#9006#10020#0" expanded="true" />
<element signature="e#10019#10020#0" expanded="true" />
<element signature="e#16239#16763#0" expanded="true" />
<element signature="e#16762#16763#0" expanded="true" />
<element signature="e#17004#17031#0" expanded="true" />
<element signature="e#17813#17846#0" expanded="true" />
<element signature="e#20024#22134#0" expanded="true" />
<element signature="e#20490#20514#0" expanded="true" />
<element signature="e#20719#20741#0" expanded="true" />
<element signature="e#22133#22134#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="SessionRandomExtractDao.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="133">
<caret line="7" column="47" lean-forward="false" selection-start-line="7" selection-start-column="47" selection-end-line="7" selection-end-column="47" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="SessionRandomExtractDaoImpl.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="17" column="34" lean-forward="false" selection-start-line="17" selection-start-column="34" selection-end-line="17" selection-end-column="34" />
<folding>
<element signature="e#511#521#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DaoFactory.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="26" column="49" lean-forward="false" selection-start-line="26" selection-start-column="49" selection-end-line="26" selection-end-column="49" />
<folding>
<element signature="e#702#703#0" expanded="true" />
<element signature="e#758#759#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="JDBCHelper.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/jdbc/JDBCHelper.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="154" column="17" lean-forward="false" selection-start-line="154" selection-start-column="17" selection-end-line="154" selection-end-column="17" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Constants.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/constant/Constants.java">
<provider selected="true" editor-type-id="text-editor">
@ -99,7 +147,6 @@
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionAggrStatDao.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/domain/SessionAggrStat.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionAggrStatDaoImpl.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java" />
<option value="$PROJECT_DIR$/src/test/java/cn/edu/hust/dao/SessionAggrDao.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/util/DateUtils.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/SessionAggrStatAccumulator.java" />
@ -110,6 +157,7 @@
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/domain/SessionDetail.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java" />
<option value="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java" />
</list>
</option>
@ -225,6 +273,25 @@
<item name="hust" type="462c0819:PsiDirectoryNode" />
<item name="constant" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="UserActionAnalyzePlatform" type="b2602c69:ProjectViewProjectNode" />
<item name="UserActionAnalyzePlatform" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hust" type="462c0819:PsiDirectoryNode" />
<item name="dao" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="UserActionAnalyzePlatform" type="b2602c69:ProjectViewProjectNode" />
<item name="UserActionAnalyzePlatform" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hust" type="462c0819:PsiDirectoryNode" />
<item name="dao" type="462c0819:PsiDirectoryNode" />
<item name="factory" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="UserActionAnalyzePlatform" type="b2602c69:ProjectViewProjectNode" />
<item name="UserActionAnalyzePlatform" type="462c0819:PsiDirectoryNode" />
@ -532,7 +599,7 @@
<workItem from="1529590333052" duration="2564000" />
<workItem from="1529630336674" duration="15098000" />
<workItem from="1529666057370" duration="13151000" />
<workItem from="1529717692529" duration="8174000" />
<workItem from="1529717692529" duration="10693000" />
</task>
<task id="LOCAL-00001" summary="1.环境搭建完成&#10;2.相关的工具类编写完成&#10;3.配置文件管理类编写完成">
<created>1529592741848</created>
@ -584,7 +651,7 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="40021000" />
<option name="totallyTimeSpent" value="42540000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="1440" height="900" extended-state="0" />
@ -693,14 +760,6 @@
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/constant/Constants.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1045">
<caret line="55" column="48" lean-forward="false" selection-start-line="55" selection-start-column="31" selection-end-line="55" selection-end-column="48" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/conf/ConfigurationManager.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
@ -852,19 +911,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/jdbc/JDBCHelper.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="212">
<caret line="107" column="10" lean-forward="false" selection-start-line="107" selection-start-column="10" selection-end-line="107" selection-end-column="10" />
<folding>
<element signature="imports" expanded="false" />
<element signature="e#422#434#0" expanded="false" />
<element signature="e#1428#1429#0" expanded="false" />
<element signature="e#1459#1460#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/cn/edu/hust/jdbc/JDBCHelperTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="434">
@ -1001,17 +1047,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="247">
<caret line="19" column="44" lean-forward="false" selection-start-line="19" selection-start-column="44" selection-end-line="19" selection-end-column="44" />
<folding>
<element signature="e#425#426#0" expanded="false" />
<element signature="e#476#477#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/mockData/MockData.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1177">
@ -1258,24 +1293,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="95">
<caret line="5" column="15" lean-forward="false" selection-start-line="5" selection-start-column="15" selection-end-line="5" selection-end-column="15" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="12" column="0" lean-forward="true" selection-start-line="12" selection-start-column="0" selection-end-line="12" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/domain/SessionRandomExtract.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="38">
@ -1305,24 +1322,75 @@
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.10/1.5.1/spark-core_2.10-1.5.1.jar!/org/apache/spark/api/java/JavaPairRDD.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="128" column="0" lean-forward="true" selection-start-line="128" selection-start-column="0" selection-end-line="128" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/jdbc/JDBCHelper.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="154" column="17" lean-forward="false" selection-start-line="154" selection-start-column="17" selection-end-line="154" selection-end-column="17" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/factory/DaoFactory.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="26" column="49" lean-forward="false" selection-start-line="26" selection-start-column="49" selection-end-line="26" selection-end-column="49" />
<folding>
<element signature="e#702#703#0" expanded="true" />
<element signature="e#758#759#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="17" column="34" lean-forward="false" selection-start-line="17" selection-start-column="34" selection-end-line="17" selection-end-column="34" />
<folding>
<element signature="e#511#521#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="133">
<caret line="7" column="47" lean-forward="false" selection-start-line="7" selection-start-column="47" selection-end-line="7" selection-end-column="47" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="539">
<caret line="378" column="27" lean-forward="true" selection-start-line="378" selection-start-column="27" selection-end-line="378" selection-end-column="27" />
<state relative-caret-position="403">
<caret line="497" column="8" lean-forward="false" selection-start-line="497" selection-start-column="8" selection-end-line="497" selection-end-column="8" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#4882#5107#0" expanded="true" />
<element signature="e#5106#5107#0" expanded="true" />
<element signature="e#5337#8133#0" expanded="true" />
<element signature="e#8132#8133#0" expanded="true" />
<element signature="e#8353#8576#0" expanded="true" />
<element signature="e#8575#8576#0" expanded="true" />
<element signature="e#8829#9843#0" expanded="true" />
<element signature="e#9842#9843#0" expanded="true" />
<element signature="e#16022#16546#0" expanded="true" />
<element signature="e#16545#16546#0" expanded="true" />
<element signature="e#16787#16814#0" expanded="true" />
<element signature="e#17590#17623#0" expanded="true" />
<element signature="e#5059#5284#0" expanded="true" />
<element signature="e#5283#5284#0" expanded="true" />
<element signature="e#5514#8310#0" expanded="true" />
<element signature="e#8309#8310#0" expanded="true" />
<element signature="e#8530#8753#0" expanded="true" />
<element signature="e#8752#8753#0" expanded="true" />
<element signature="e#9006#10020#0" expanded="true" />
<element signature="e#10019#10020#0" expanded="true" />
<element signature="e#16239#16763#0" expanded="true" />
<element signature="e#16762#16763#0" expanded="true" />
<element signature="e#17004#17031#0" expanded="true" />
<element signature="e#17813#17846#0" expanded="true" />
<element signature="e#20024#22134#0" expanded="true" />
<element signature="e#20490#20514#0" expanded="true" />
<element signature="e#20719#20741#0" expanded="true" />
<element signature="e#22133#22134#0" expanded="true" />
</folding>
</state>
</provider>

4
src/main/java/cn/edu/hust/dao/SessionRandomExtractDao.java
Unescape View File

@ -2,6 +2,8 @@ package cn.edu.hust.dao;
import cn.edu.hust.domain.SessionRandomExtract;
import java.util.List;
public interface SessionRandomExtractDao {
void batchInsert(SessionRandomExtract sessionRandomExtract);
void batchInsert(List<SessionRandomExtract> sessionRandomExtractList);
}

7
src/main/java/cn/edu/hust/dao/factory/DaoFactory.java
Unescape View File

@ -1,9 +1,12 @@
package cn.edu.hust.dao.factory;
import cn.edu.hust.dao.SessionAggrStatDao;
import cn.edu.hust.dao.SessionRandomExtractDao;
import cn.edu.hust.dao.TaskDao;
import cn.edu.hust.dao.impl.SessionAggrStatDaoImpl;
import cn.edu.hust.dao.impl.SessionRandomExtractDaoImpl;
import cn.edu.hust.dao.impl.TaskDaoImpl;
import cn.edu.hust.domain.SessionRandomExtract;
public class DaoFactory {
/**
@ -19,4 +22,8 @@ public class DaoFactory {
{
return new SessionAggrStatDaoImpl();
}
public static SessionRandomExtractDao getSessionRandomExtractDao(){
return new SessionRandomExtractDaoImpl();
}
}

14
src/main/java/cn/edu/hust/dao/impl/SessionRandomExtractDaoImpl.java
Unescape View File

@ -4,9 +4,19 @@ import cn.edu.hust.dao.SessionRandomExtractDao;
import cn.edu.hust.domain.SessionRandomExtract;
import cn.edu.hust.jdbc.JDBCHelper;
import java.util.ArrayList;
import java.util.List;
public class SessionRandomExtractDaoImpl implements SessionRandomExtractDao{
@Override
public void batchInsert(SessionRandomExtract sessionRandomExtract) {
//JDBCHelper.getInstance().
public void batchInsert(List<SessionRandomExtract> sessionRandomExtractList) {
String sql="insert into session_random_extract values(?,?,?,?,?)";
List<Object[]> paramList=new ArrayList<Object[]>();
for (SessionRandomExtract sessionRandomExtract:sessionRandomExtractList) {
Object[] params=new Object[]{sessionRandomExtract.getTaskId(),sessionRandomExtract.getSessionId()
,sessionRandomExtract.getStartTime(),sessionRandomExtract.getSearchKeyWords(),sessionRandomExtract.getClick_category_ids()};
paramList.add(params);
}
JDBCHelper.getInstance().excuteBatch(sql,paramList);
}
}

51
src/main/java/cn/edu/hust/session/UserVisitAnalyze.java
Unescape View File

@ -5,6 +5,7 @@ import cn.edu.hust.constant.Constants;
import cn.edu.hust.dao.TaskDao;
import cn.edu.hust.dao.factory.DaoFactory;
import cn.edu.hust.domain.SessionAggrStat;
import cn.edu.hust.domain.SessionRandomExtract;
import cn.edu.hust.domain.Task;
import cn.edu.hust.mockData.MockData;
import cn.edu.hust.util.*;
@ -16,7 +17,9 @@ import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
@ -95,7 +98,7 @@ public class UserVisitAnalyze {
/**
* 使用CountByKey算子实现随机抽取功能
*/
randomExtractSession(filteredSessionRDD);
randomExtractSession(taskId,filteredSessionRDD);
//计算各个session占比,并写入MySQL
calculateAndPersist(sessionAggrStatAccumulator.value(),taskId);
//关闭spark上下文
@ -361,9 +364,10 @@ public class UserVisitAnalyze {
/**
* 随机抽取Sesison功能
* @param taskId
* @param filteredSessionRDD
*/
private static void randomExtractSession(JavaPairRDD<String, String> filteredSessionRDD) {
private static void randomExtractSession(final Long taskId, JavaPairRDD<String, String> filteredSessionRDD) {
//1.先将过滤Seesion进行映射，映射成为Time,Info的数据格式
final JavaPairRDD<String,String> mapDataRDD=filteredSessionRDD.mapToPair(new PairFunction<Tuple2<String, String>, String, String>() {
@Override
@ -401,7 +405,7 @@ public class UserVisitAnalyze {
Random random=new Random();
//设计一个新的数据结构，用于存储随机索引,Key是每一天,Map是小时和随机索引列表构成的
Map<String,Map<String,List<Long>>> dateRandomExtractMap=new HashMap<String, Map<String, List<Long>>>();
final Map<String,Map<String,List<Long>>> dateRandomExtractMap=new HashMap<String, Map<String, List<Long>>>();
for (Map.Entry<String,Map<String,Long>> dateHourCount:dateHourCountMap.entrySet())
{
@ -451,6 +455,47 @@ public class UserVisitAnalyze {
}
}
//2.将上面计算的RDD进行分组，然后使用FlatMap进行压平，然后判断是否在索引中，如果在，那么将这个信息持久化
JavaPairRDD<String,Iterable<String>> time2GroupRDD=mapDataRDD.groupByKey();
//将抽取的信息持久化到数据库，并返回SessionIds对，然后和以前的信息Join
JavaPairRDD<String,String> sessionIds= time2GroupRDD.flatMapToPair(new PairFlatMapFunction<Tuple2<String,Iterable<String>>, String, String>() {
@Override
public Iterable<Tuple2<String, String>> call(Tuple2<String, Iterable<String>> tuple2) throws Exception {
String dateStr=tuple2._1;
String date=dateStr.split("_")[0];
String hour=dateStr.split("_")[1];
//使用一个List存储sessionId
List<Tuple2<String,String>> sessionIds=new ArrayList<Tuple2<String, String>>();
List<Long> indexList=dateRandomExtractMap.get(date).get(hour);
//使用一个list保存需要持久化到数据库的对象
List<SessionRandomExtract> sessionRandomExtractList=new ArrayList<SessionRandomExtract>();
int index=0;
for (String infos:
tuple2._2) {
if(indexList.contains(index))
{
//构建SessionRandomExtract
SessionRandomExtract sessionRandomExtract=new SessionRandomExtract();
final String sessionId=StringUtils.getFieldFromConcatString(infos,"\\|",Constants.FIELD_SESSIONID);
String startTime=StringUtils.getFieldFromConcatString(infos,"\\|",Constants.FIELD_START_TIME);
String searchKeyWards=StringUtils.getFieldFromConcatString(infos,"\\|",Constants.FIELD_SERACH_KEYWORDS);
String clickCategoryIds=StringUtils.getFieldFromConcatString(infos,"\\|",Constants.FIELD_CLICK_CATEGORYIDS);
sessionRandomExtract.set(taskId,sessionId,startTime,searchKeyWards,clickCategoryIds);
//添加到List中然后持久化到数据库中
sessionRandomExtractList.add(sessionRandomExtract);
sessionIds.add(new Tuple2<String, String>(sessionId,sessionId));
}
index++;
}
//持久化到数据库
DaoFactory.getSessionRandomExtractDao().batchInsert(sessionRandomExtractList);
return sessionIds;
}
});
//3. 获取session的明细数据保存到数据库
}
//计算各个范围的占比，并持久化到数据库

Write Preview
Loading…
Cancel
Save