pil3aoekg
/
javac
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

性能优化

Browse Source
main
oeljeklaus-you 7 years ago
parent 0480f96753
commit 98e1ffdcc5
2 changed files with 52 additions and 50 deletions
Show Stats Download Patch File Download Diff File

83
.idea/workspace.xml
Unescape View File

@ -2,8 +2,8 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a7505764-040b-48e2-b2fc-8c5b579e595f" name="Default" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/README.md" afterPath="$PROJECT_DIR$/README.md" />
<change beforePath="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java" afterPath="$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java" />
</list>
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
@ -16,21 +16,13 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="UserVisitAnalyze.java" pinned="false" current-in-tab="true">
<file leaf-file-name="UserVisitAnalyze.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-320">
<caret line="972" column="29" lean-forward="true" selection-start-line="972" selection-start-column="29" selection-end-line="972" selection-end-column="29" />
<state relative-caret-position="-377">
<caret line="972" column="29" lean-forward="false" selection-start-line="972" selection-start-column="29" selection-end-line="972" selection-end-column="29" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#43845#45127#0" expanded="true" />
<element signature="e#44185#44197#0" expanded="true" />
<element signature="e#44739#44760#0" expanded="true" />
<element signature="e#45126#45127#0" expanded="true" />
<element signature="e#45846#48382#0" expanded="true" />
<element signature="e#46138#46160#0" expanded="true" />
<element signature="e#46333#46357#0" expanded="true" />
<element signature="e#48381#48382#0" expanded="true" />
</folding>
</state>
</provider>
@ -39,19 +31,19 @@
<file leaf-file-name="MockData.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/mockData/MockData.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="281">
<caret line="93" column="50" lean-forward="true" selection-start-line="93" selection-start-column="50" selection-end-line="93" selection-end-column="50" />
<state relative-caret-position="1482">
<caret line="93" column="50" lean-forward="false" selection-start-line="93" selection-start-column="50" selection-end-line="93" selection-end-column="50" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="README.md" pinned="false" current-in-tab="false">
<file leaf-file-name="README.md" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="FIRST">
<first_editor relative-caret-position="345">
<caret line="352" column="68" lean-forward="false" selection-start-line="352" selection-start-column="68" selection-end-line="352" selection-end-column="68" />
<first_editor relative-caret-position="703">
<caret line="448" column="0" lean-forward="false" selection-start-line="448" selection-start-column="0" selection-end-line="448" selection-end-column="0" />
<folding />
</first_editor>
<second_editor />
@ -160,9 +152,10 @@
<option name="jreName" value="#JAVA_INTERNAL" />
<option name="vmOptions" value="-DarchetypeCatalog=internal" />
</component>
<component name="ProjectFrameBounds" fullScreen="true">
<component name="ProjectFrameBounds" extendedState="6" fullScreen="true">
<option name="y" value="23" />
<option name="width" value="1440" />
<option name="height" value="900" />
<option name="height" value="873" />
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
@ -205,8 +198,8 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="Scratches" />
<pane id="AndroidView" />
<pane id="ProjectPane">
<subPane>
<expand>
@ -282,7 +275,7 @@
</subPane>
</pane>
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="AndroidView" />
</panes>
</component>
<component name="PropertiesComponent">
@ -556,7 +549,8 @@
<workItem from="1529717692529" duration="30310000" />
<workItem from="1529837434323" duration="208000" />
<workItem from="1529845795654" duration="7752000" />
<workItem from="1529887211212" duration="6828000" />
<workItem from="1529887211212" duration="6851000" />
<workItem from="1529994772485" duration="146000" />
</task>
<task id="LOCAL-00001" summary="1.环境搭建完成&#10;2.相关的工具类编写完成&#10;3.配置文件管理类编写完成">
<created>1529592741848</created>
@ -608,15 +602,14 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="76945000" />
<option name="totallyTimeSpent" value="77114000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="1440" height="900" extended-state="0" />
<frame x="0" y="0" width="1440" height="900" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32889965" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Java Enterprise" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
@ -628,7 +621,7 @@
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.18955651" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.19170243" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.24964234" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
@ -640,6 +633,7 @@
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32889965" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
@ -1054,8 +1048,8 @@
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/mockData/MockData.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="281">
<caret line="93" column="50" lean-forward="true" selection-start-line="93" selection-start-column="50" selection-end-line="93" selection-end-column="50" />
<state relative-caret-position="1482">
<caret line="93" column="50" lean-forward="false" selection-start-line="93" selection-start-column="50" selection-end-line="93" selection-end-column="50" />
<folding />
</state>
</provider>
@ -1064,39 +1058,30 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="151">
<caret line="145" column="0" lean-forward="false" selection-start-line="145" selection-start-column="0" selection-end-line="145" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-377">
<caret line="972" column="29" lean-forward="false" selection-start-line="972" selection-start-column="29" selection-end-line="972" selection-end-column="29" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="FIRST">
<first_editor relative-caret-position="345">
<caret line="352" column="68" lean-forward="false" selection-start-line="352" selection-start-column="68" selection-end-line="352" selection-end-column="68" />
<first_editor relative-caret-position="703">
<caret line="448" column="0" lean-forward="false" selection-start-line="448" selection-start-column="0" selection-end-line="448" selection-end-column="0" />
<folding />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/cn/edu/hust/session/UserVisitAnalyze.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-320">
<caret line="972" column="29" lean-forward="true" selection-start-line="972" selection-start-column="29" selection-end-line="972" selection-end-column="29" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#43845#45127#0" expanded="true" />
<element signature="e#44185#44197#0" expanded="true" />
<element signature="e#44739#44760#0" expanded="true" />
<element signature="e#45126#45127#0" expanded="true" />
<element signature="e#45846#48382#0" expanded="true" />
<element signature="e#46138#46160#0" expanded="true" />
<element signature="e#46333#46357#0" expanded="true" />
<element signature="e#48381#48382#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>

19
README.md
Unescape View File

@ -429,4 +429,21 @@ Kryo序列化机制，速度快乐序列化之后数据更小，大概是java序
kyro序列化机制:
1.算子函数中使用的外部变量
2.持久化RDD进行序列化
3.shuffle过程
3.shuffle过程
### 性能优化之Json数据格式优化
FastUtil是扩展了Java标准集合框架的类库，提供了特定类型的Map,Set,List,Queue，ArrayList,HashMap能够提供更小的内存占用，更快的存取速度，
也提供了64位的Array,Set,List以及快速的和实用的IO类处理二进制和文本文件。最新版本要求Java7以及以上版本。我们使用FasyUtil提供的集合类，
来替代自己平时使用的JDK的原生的Map，List,Set好处在于FastUtil集合类，可以减少内存的占用，并且在集合遍历，根据索性获取元素的值和设置元素的值，
提供更快的存取速度。除了对象和原始类型，fastutil也提供引用类型的支持，但是使用等于号进行比较的，而不是equals（）方法。
Spark应用fastutil
1.如果算子函数使用了外部变量，那么第一，你可以使用Broadcast广播变量优化，第二可以使用kyro序列化类库，提升序列化性能和效率，
第三，如果外部变量是某种比较大的集合，那么可以考虑使用fastutil改写外部变量，首先从源头上减少内存的占比，通过广播变量进一步减少内存占用，
通过kyro序列化类库进一步减少内存占比。
Write Preview
Loading…
Cancel
Save