parent
96cf27a82a
commit
edba0b7fa8
@ -0,0 +1 @@
|
||||
pyg
|
||||
@ -0,0 +1,2 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
@ -0,0 +1,5 @@
|
||||
<component name="ProjectCodeStyleConfiguration">
|
||||
<state>
|
||||
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
|
||||
</state>
|
||||
</component>
|
||||
@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="batch-process" />
|
||||
<module name="canal-kafka" />
|
||||
<module name="real-process" />
|
||||
<module name="report" />
|
||||
<module name="sync-db" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Encoding">
|
||||
<file url="file://$PROJECT_DIR$/report" charset="UTF-8" />
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="HydraSettings">
|
||||
<option name="hydraStorePath" value="E:\IdeaProject\Flink-pyg\.hydra\idea" />
|
||||
<option name="noOfCores" value="4" />
|
||||
<option name="projectRoot" value="E:\IdeaProject\Flink-pyg" />
|
||||
<option name="sourcePartitioner" value="auto" />
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="JavaScriptSettings">
|
||||
<option name="languageLevel" value="ES6" />
|
||||
</component>
|
||||
<component name="MavenProjectsManager">
|
||||
<option name="originalFiles">
|
||||
<list>
|
||||
<option value="$PROJECT_DIR$/pom.xml" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,124 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Palette2">
|
||||
<group name="Swing">
|
||||
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
|
||||
</item>
|
||||
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
|
||||
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
|
||||
<initial-values>
|
||||
<property name="text" value="Button" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="RadioButton" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="CheckBox" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="Label" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
|
||||
<preferred-size width="-1" height="20" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
|
||||
</item>
|
||||
</group>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>pyg</artifactId>
|
||||
<groupId>com.henry</groupId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>batch-process</artifactId>
|
||||
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>pyg</artifactId>
|
||||
<groupId>com.henry</groupId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>canal-kafka</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<version>1.0.24</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.11</artifactId>
|
||||
<version>0.10.1.0</version>
|
||||
</dependency>
|
||||
<!--对象和json 互相转换的-->
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
<version>1.2.83</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,30 @@
|
||||
package com.henry.canal_kafka.util;
|
||||
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
public class GlobalConfigUtil {
|
||||
// 获取一个资源加载器
|
||||
// 资源加载器会自动去加载CLASSPATH中的application.properties配置文件
|
||||
private static ResourceBundle resourceBundle = ResourceBundle.getBundle("application");
|
||||
|
||||
// 使用ResourceBundle.getString方法来读取配置
|
||||
public static String canalHost = resourceBundle.getString("canal.host");
|
||||
public static String canalPort = resourceBundle.getString("canal.port");
|
||||
public static String canalInstance = resourceBundle.getString("canal.instance");
|
||||
public static String mysqlUsername = resourceBundle.getString("mysql.username");
|
||||
public static String mysqlPassword = resourceBundle.getString("mysql.password");
|
||||
public static String kafkaBootstrapServers = resourceBundle.getString("kafka.bootstrap.servers");
|
||||
public static String kafkaZookeeperConnect = resourceBundle.getString("kafka.zookeeper.connect");
|
||||
public static String kafkaInputTopic = resourceBundle.getString("kafka.input.topic");
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(canalHost);
|
||||
System.out.println(canalPort);
|
||||
System.out.println(canalInstance);
|
||||
System.out.println(mysqlUsername);
|
||||
System.out.println(mysqlPassword);
|
||||
System.out.println(kafkaBootstrapServers);
|
||||
System.out.println(kafkaZookeeperConnect);
|
||||
System.out.println(kafkaInputTopic);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package com.henry.canal_kafka.util;
|
||||
|
||||
import kafka.javaapi.producer.Producer;
|
||||
import kafka.producer.KeyedMessage;
|
||||
import kafka.producer.ProducerConfig;
|
||||
import kafka.serializer.StringEncoder;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Kafka生产消息工具类
|
||||
*/
|
||||
public class KafkaSender {
|
||||
private String topic;
|
||||
|
||||
public KafkaSender(String topic){
|
||||
super();
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送消息到Kafka指定topic
|
||||
*
|
||||
* @param topic topic名字
|
||||
* @param key 键值
|
||||
* @param data 数据
|
||||
*/
|
||||
public static void sendMessage(String topic , String key , String data){
|
||||
Producer<String, String> producer = createProducer();
|
||||
producer.send(new KeyedMessage<String , String>(topic , key , data));
|
||||
}
|
||||
|
||||
private static Producer<String , String> createProducer(){
|
||||
Properties properties = new Properties();
|
||||
|
||||
properties.put("metadata.broker.list" , GlobalConfigUtil.kafkaBootstrapServers);
|
||||
properties.put("zookeeper.connect" , GlobalConfigUtil.kafkaZookeeperConnect);
|
||||
properties.put("serializer.class" , StringEncoder.class.getName());
|
||||
|
||||
return new Producer<String, String>(new ProducerConfig(properties));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
#
|
||||
# canal\u914D\u7F6E
|
||||
#
|
||||
canal.host=master
|
||||
canal.port=11111
|
||||
canal.instance=example
|
||||
mysql.username=root
|
||||
mysql.password=123456
|
||||
#
|
||||
#kafka\u7684\u914D\u7F6E
|
||||
#
|
||||
kafka.bootstrap.servers=master:9092,slave1:9092,slave2:9092
|
||||
kafka.zookeeper.connect=master:2181,slave1:2181,slave2:2181
|
||||
kafka.input.topic=canal
|
||||
@ -0,0 +1,4 @@
|
||||
log4j.rootLogger=error,stdout
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n
|
||||
@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>com.henry</groupId>
|
||||
<artifactId>pyg</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<modules>
|
||||
<module>report</module> <!--上报服务-->
|
||||
<module>real-process</module> <!--实时处理-->
|
||||
<module>canal-kafka</module> <!--数据采集-->
|
||||
<module>sync-db</module> <!--数据库同步处理-->
|
||||
<module>batch-process</module> <!--批处理-->
|
||||
</modules>
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,2 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
@ -0,0 +1,296 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Define some default values that can be overridden by system properties
|
||||
hadoop.root.logger=ERROR,console
|
||||
hadoop.log.dir=.
|
||||
hadoop.log.file=hadoop.log
|
||||
|
||||
# Define the root logger to the system property "hadoop.root.logger".
|
||||
log4j.rootLogger=${hadoop.root.logger}, EventCounter
|
||||
|
||||
# Logging Threshold
|
||||
log4j.threshold=ALL
|
||||
|
||||
# Null Appender
|
||||
log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
|
||||
|
||||
#
|
||||
# Rolling File Appender - cap space usage at 5gb.
|
||||
#
|
||||
hadoop.log.maxfilesize=256MB
|
||||
hadoop.log.maxbackupindex=20
|
||||
log4j.appender.RFA=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
|
||||
|
||||
log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
|
||||
log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
|
||||
|
||||
log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
|
||||
|
||||
# Pattern format: Date LogLevel LoggerName LogMessage
|
||||
log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
# Debugging Pattern format
|
||||
#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
|
||||
|
||||
|
||||
#
|
||||
# Daily Rolling File Appender
|
||||
#
|
||||
|
||||
log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
|
||||
log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
|
||||
|
||||
# Rollver at midnight
|
||||
log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
|
||||
|
||||
# 30-day backup
|
||||
#log4j.appender.DRFA.MaxBackupIndex=30
|
||||
log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
|
||||
|
||||
# Pattern format: Date LogLevel LoggerName LogMessage
|
||||
log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
# Debugging Pattern format
|
||||
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
|
||||
|
||||
|
||||
#
|
||||
# console
|
||||
# Add "console" to rootlogger above if you want to use this
|
||||
#
|
||||
|
||||
log4j.appender.console=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.console.target=System.err
|
||||
log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.console.layout.ConversionPattern=%d{yy HH:mm:ss} %p %c{2}: %m%n
|
||||
|
||||
#
|
||||
# TaskLog Appender
|
||||
#
|
||||
|
||||
#Default values
|
||||
hadoop.tasklog.taskid=null
|
||||
hadoop.tasklog.iscleanup=false
|
||||
hadoop.tasklog.noKeepSplits=4
|
||||
hadoop.tasklog.totalLogFileSize=100
|
||||
hadoop.tasklog.purgeLogSplits=true
|
||||
hadoop.tasklog.logsRetainHours=12
|
||||
|
||||
log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
|
||||
log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
|
||||
log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
|
||||
log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
|
||||
|
||||
log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
|
||||
#
|
||||
# HDFS block state change log from block manager
|
||||
#
|
||||
# Uncomment the following to suppress normal block state change
|
||||
# messages from BlockManager in NameNode.
|
||||
#log4j.logger.BlockStateChange=WARN
|
||||
|
||||
#
|
||||
#Security appender
|
||||
#
|
||||
hadoop.security.logger=INFO,NullAppender
|
||||
hadoop.security.log.maxfilesize=256MB
|
||||
hadoop.security.log.maxbackupindex=20
|
||||
log4j.category.SecurityLogger=${hadoop.security.logger}
|
||||
hadoop.security.log.file=SecurityAuth-${user.name}.audit
|
||||
log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
|
||||
log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
|
||||
log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
|
||||
|
||||
#
|
||||
# Daily Rolling Security appender
|
||||
#
|
||||
log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
|
||||
log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
|
||||
log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
|
||||
|
||||
#
|
||||
# hadoop configuration logging
|
||||
#
|
||||
|
||||
# Uncomment the following line to turn off configuration deprecation warnings.
|
||||
# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
|
||||
|
||||
#
|
||||
# hdfs audit logging
|
||||
#
|
||||
hdfs.audit.logger=INFO,NullAppender
|
||||
hdfs.audit.log.maxfilesize=256MB
|
||||
hdfs.audit.log.maxbackupindex=20
|
||||
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
|
||||
log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
|
||||
log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
|
||||
log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
|
||||
log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
|
||||
log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
|
||||
|
||||
#
|
||||
# mapred audit logging
|
||||
#
|
||||
mapred.audit.logger=INFO,NullAppender
|
||||
mapred.audit.log.maxfilesize=256MB
|
||||
mapred.audit.log.maxbackupindex=20
|
||||
log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
|
||||
log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
|
||||
log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
|
||||
log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
|
||||
log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
|
||||
log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
|
||||
|
||||
# Custom Logging levels
|
||||
|
||||
#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
|
||||
#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
|
||||
#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
|
||||
|
||||
# Jets3t library
|
||||
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
|
||||
|
||||
# AWS SDK & S3A FileSystem
|
||||
log4j.logger.com.amazonaws=ERROR
|
||||
log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
|
||||
log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
|
||||
|
||||
#
|
||||
# Event Counter Appender
|
||||
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
|
||||
#
|
||||
log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
|
||||
|
||||
#
|
||||
# Job Summary Appender
|
||||
#
|
||||
# Use following logger to send summary to separate file defined by
|
||||
# hadoop.mapreduce.jobsummary.log.file :
|
||||
# hadoop.mapreduce.jobsummary.logger=INFO,JSA
|
||||
#
|
||||
hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
|
||||
hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
|
||||
hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
|
||||
hadoop.mapreduce.jobsummary.log.maxbackupindex=20
|
||||
log4j.appender.JSA=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
|
||||
log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
|
||||
log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
|
||||
log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
|
||||
log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
|
||||
log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
|
||||
|
||||
#
|
||||
# Yarn ResourceManager Application Summary Log
|
||||
#
|
||||
# Set the ResourceManager summary log filename
|
||||
yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
|
||||
# Set the ResourceManager summary log level and appender
|
||||
yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
|
||||
#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
|
||||
|
||||
# To enable AppSummaryLogging for the RM,
|
||||
# set yarn.server.resourcemanager.appsummary.logger to
|
||||
# <LEVEL>,RMSUMMARY in hadoop-env.sh
|
||||
|
||||
# Appender for ResourceManager Application Summary Log
|
||||
# Requires the following properties to be set
|
||||
# - hadoop.log.dir (Hadoop Log directory)
|
||||
# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
|
||||
# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
|
||||
|
||||
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
|
||||
log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
|
||||
log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
|
||||
log4j.appender.RMSUMMARY.MaxFileSize=256MB
|
||||
log4j.appender.RMSUMMARY.MaxBackupIndex=20
|
||||
log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
|
||||
|
||||
# HS audit log configs
|
||||
#mapreduce.hs.audit.logger=INFO,HSAUDIT
|
||||
#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger}
|
||||
#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false
|
||||
#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender
|
||||
#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log
|
||||
#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout
|
||||
#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
|
||||
#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd
|
||||
|
||||
# Http Server Request Logs
|
||||
#log4j.logger.http.requests.namenode=INFO,namenoderequestlog
|
||||
#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
|
||||
#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log
|
||||
#log4j.appender.namenoderequestlog.RetainDays=3
|
||||
|
||||
#log4j.logger.http.requests.datanode=INFO,datanoderequestlog
|
||||
#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
|
||||
#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log
|
||||
#log4j.appender.datanoderequestlog.RetainDays=3
|
||||
|
||||
#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog
|
||||
#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
|
||||
#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log
|
||||
#log4j.appender.resourcemanagerrequestlog.RetainDays=3
|
||||
|
||||
#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog
|
||||
#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
|
||||
#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log
|
||||
#log4j.appender.jobhistoryrequestlog.RetainDays=3
|
||||
|
||||
#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog
|
||||
#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
|
||||
#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
|
||||
#log4j.appender.nodemanagerrequestlog.RetainDays=3
|
||||
|
||||
|
||||
# WebHdfs request log on datanodes
|
||||
# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to
|
||||
# direct the log to a separate file.
|
||||
#datanode.webhdfs.logger=INFO,console
|
||||
#log4j.logger.datanode.webhdfs=${datanode.webhdfs.logger}
|
||||
#log4j.appender.HTTPDRFA=org.apache.log4j.DailyRollingFileAppender
|
||||
#log4j.appender.HTTPDRFA.File=${hadoop.log.dir}/hadoop-datanode-webhdfs.log
|
||||
#log4j.appender.HTTPDRFA.layout=org.apache.log4j.PatternLayout
|
||||
#log4j.appender.HTTPDRFA.layout.ConversionPattern=%d{ISO8601} %m%n
|
||||
#log4j.appender.HTTPDRFA.DatePattern=.yyyy-MM-dd
|
||||
|
||||
#
|
||||
# Fair scheduler state dump
|
||||
#
|
||||
# Use following logger to dump the state to a separate file
|
||||
|
||||
#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSSTATEDUMP
|
||||
#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false
|
||||
#log4j.appender.FSSTATEDUMP=org.apache.log4j.RollingFileAppender
|
||||
#log4j.appender.FSSTATEDUMP.File=${hadoop.log.dir}/fairscheduler-statedump.log
|
||||
#log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout
|
||||
#log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
|
||||
#log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize}
|
||||
#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
|
||||
@ -0,0 +1,12 @@
|
||||
package com.henry.realprocess.bean
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description:
|
||||
* @Date: Create in 2019/10/20 15:58
|
||||
**/
|
||||
case class Message (
|
||||
var clickLog:ClickLog,
|
||||
var count:Long,
|
||||
var timeStamp:Long
|
||||
)
|
||||
@ -0,0 +1,80 @@
|
||||
package com.henry.realprocess.task
|
||||
|
||||
import com.henry.realprocess.bean.ClickLogWide
|
||||
import com.henry.realprocess.task.ChannelBrowserTask.pvColName
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.flink.streaming.api.scala.{DataStream, KeyedStream, WindowedStream}
|
||||
import org.apache.flink.streaming.api.windowing.time.Time
|
||||
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description:
|
||||
* @Date: Create in 2019/11/3 10:42
|
||||
**/
|
||||
|
||||
trait BaseTask[T] {
|
||||
|
||||
|
||||
// 1、 转换
|
||||
def map(clickLogWideDataStream : DataStream[ClickLogWide]): DataStream[T]
|
||||
|
||||
// 2、 分组
|
||||
def keyBy(mapDataStream : DataStream[T]): KeyedStream[T, String]
|
||||
|
||||
// 3、 时间窗口
|
||||
def timeWindow(keyedStream: KeyedStream[T, String]) : WindowedStream[T, String, TimeWindow] = {
|
||||
// 因为所有自类都是 3 秒的时间窗口
|
||||
keyedStream.timeWindow(Time.seconds(3))
|
||||
}
|
||||
|
||||
// 4、 聚合
|
||||
def reduce(windowedStream : WindowedStream[T, String, TimeWindow]) : DataStream[T]
|
||||
|
||||
// 5、 落地 HBase
|
||||
def sink2HBase(reduceDataStream: DataStream[T])
|
||||
|
||||
|
||||
// 定义模板执行顺序
|
||||
def process(clickLogWideDataStream : DataStream[ClickLogWide]): Unit = {
|
||||
val mapDataStream: DataStream[T] = map(clickLogWideDataStream)
|
||||
val keyedStream: KeyedStream[T, String] = keyBy(mapDataStream)
|
||||
val windowedStream: WindowedStream[T, String, TimeWindow] = timeWindow(keyedStream)
|
||||
val reduceStream: DataStream[T] = reduce(windowedStream)
|
||||
sink2HBase(reduceStream)
|
||||
}
|
||||
|
||||
// 检测老用户是否第一次访问
|
||||
val isOld = (isNew: Int, isDateNew: Int) => if (isNew == 0 && isDateNew == 1) 1 else 0
|
||||
|
||||
// 创建 HBase 相关列
|
||||
var tableName = ""
|
||||
var clfName = "info"
|
||||
var rowkey = ""
|
||||
var channelIdColName = "channelID"
|
||||
var browserColName = "browser"
|
||||
var dateColName = "date"
|
||||
var pvColName = "pv"
|
||||
var uvColName = "uv"
|
||||
var newCountColName = "newCount"
|
||||
var oldCountColName = "oldCount"
|
||||
|
||||
|
||||
/* 累加相关列的值
|
||||
* @param resultMap map集合
|
||||
* @param column 待查询的列
|
||||
* @param currentValue 当前值
|
||||
* @return 累加后的值
|
||||
*/
|
||||
def getTotal(resultMap: Map[String, String],column:String,currentValue:Long):Long={
|
||||
|
||||
var total = currentValue
|
||||
// 如果resultMap不为空,并且可以去到相关列的值,那么就进行累加
|
||||
if (resultMap != null && StringUtils.isNotBlank(resultMap.getOrElse(column,""))) {
|
||||
total = resultMap(column).toLong + currentValue
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
val bootstrap.servers = config.getString("bootstrap.servers")
|
||||
val zookeeper.connect = config.getString("zookeeper.connect")
|
||||
val input.topic = config.getString("input.topic")
|
||||
val gruop.id = config.getString("gruop.id")
|
||||
val enable.auto.commit = config.getString("enable.auto.commit")
|
||||
val auto.commit.interval.ms = config.getString("auto.commit.interval.ms")
|
||||
val auto.offset.reset = config.getString("auto.offset.reset")
|
||||
@ -0,0 +1,18 @@
|
||||
package com.henry.report;
|
||||
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
/**
|
||||
* @Author: HongZhen
|
||||
* @Description:
|
||||
* @Date: Create in 2019/9/20 11:10
|
||||
**/
|
||||
|
||||
// 添加注解 @SpringBootApplication ，表示该类是一个启动类
|
||||
@SpringBootApplication
|
||||
public class ReportApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ReportApplication.class, args);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,136 @@
|
||||
package com.henry.report.bean;
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description: 点击流日志
|
||||
* @Date: Create in 2019/10/13 19:33
|
||||
**/
|
||||
|
||||
public class Clicklog {
|
||||
|
||||
// 频道 ID
|
||||
private long channelID;
|
||||
// 产品的类别 ID
|
||||
private long categoryID ;
|
||||
// 产品 ID
|
||||
private long produceID ;
|
||||
// 用户 ID
|
||||
private long userID ;
|
||||
|
||||
// 国家
|
||||
private String country;
|
||||
// 省份
|
||||
private String province;
|
||||
// 城市
|
||||
private String city;
|
||||
|
||||
// 网络方式
|
||||
private String network;
|
||||
// 来源方式
|
||||
private String source;
|
||||
|
||||
// 浏览器类型
|
||||
private String browserType;
|
||||
|
||||
// 进入网站时间
|
||||
private Long entryTime ;
|
||||
// 离开网站实际
|
||||
private long leaveTime;
|
||||
|
||||
public long getChannelID() {
|
||||
return channelID;
|
||||
}
|
||||
|
||||
public void setChannelID(long channelID) {
|
||||
this.channelID = channelID;
|
||||
}
|
||||
|
||||
public long getCategoryID() {
|
||||
return categoryID;
|
||||
}
|
||||
|
||||
public void setCategoryID(long categoryID) {
|
||||
this.categoryID = categoryID;
|
||||
}
|
||||
|
||||
public long getProduceID() {
|
||||
return produceID;
|
||||
}
|
||||
|
||||
public void setProduceID(long produceID) {
|
||||
this.produceID = produceID;
|
||||
}
|
||||
|
||||
public long getUserID() {
|
||||
return userID;
|
||||
}
|
||||
|
||||
public void setUserID(long userID) {
|
||||
this.userID = userID;
|
||||
}
|
||||
|
||||
public String getCountry() {
|
||||
return country;
|
||||
}
|
||||
|
||||
public void setCountry(String country) {
|
||||
this.country = country;
|
||||
}
|
||||
|
||||
public String getProvince() {
|
||||
return province;
|
||||
}
|
||||
|
||||
public void setProvince(String province) {
|
||||
this.province = province;
|
||||
}
|
||||
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
public String getNetwork() {
|
||||
return network;
|
||||
}
|
||||
|
||||
public void setNetwork(String network) {
|
||||
this.network = network;
|
||||
}
|
||||
|
||||
public String getSource() {
|
||||
return source;
|
||||
}
|
||||
|
||||
public void setSource(String source) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
public String getBrowserType() {
|
||||
return browserType;
|
||||
}
|
||||
|
||||
public void setBrowserType(String browserType) {
|
||||
this.browserType = browserType;
|
||||
}
|
||||
|
||||
public Long getEntryTime() {
|
||||
return entryTime;
|
||||
}
|
||||
|
||||
public void setEntryTime(Long entryTime) {
|
||||
this.entryTime = entryTime;
|
||||
}
|
||||
|
||||
public long getLeaveTime() {
|
||||
return leaveTime;
|
||||
}
|
||||
|
||||
public void setLeaveTime(long leaveTime) {
|
||||
this.leaveTime = leaveTime;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
package com.henry.report.bean;
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description: 消息实体类
|
||||
* @Date: Create in 2019/10/11 23:40
|
||||
**/
|
||||
public class Message {
|
||||
|
||||
// 消息次数
|
||||
private int count;
|
||||
|
||||
// 消息的时间戳
|
||||
private long timestamp;
|
||||
|
||||
// 消息体
|
||||
private String message;
|
||||
|
||||
public int getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
public void setCount(int count) {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public void setTimestamp(long timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Message{" +
|
||||
"count=" + count +
|
||||
", timestamp=" + timestamp +
|
||||
", message='" + message + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,139 @@
|
||||
package com.henry.report.util;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.henry.report.bean.Clicklog;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description: 点击流日志模拟器
|
||||
* @Date: Create in 2019/10/13 20:00
|
||||
**/
|
||||
public class ClickLogGenerator {
|
||||
|
||||
// ID 信息
|
||||
private static Long[] channelID = new Long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L};
|
||||
private static Long[] categoryID = new Long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L};
|
||||
private static Long[] produceID = new Long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L};
|
||||
private static Long[] userID = new Long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L};
|
||||
|
||||
// 地区
|
||||
private static String[] contrys = new String[]{"china"}; // 地区-国家集合
|
||||
private static String[] provinces = new String[]{"HeNan", "HeBeijing"}; // 地区-省集合
|
||||
private static String[] citys = new String[]{"ShiJiaZhuang", "ZhengZhou", "LuoyYang"}; // 地区-市集合
|
||||
|
||||
// 网络方式
|
||||
private static String[] networks = new String[]{"电信", "移动", "联通"};
|
||||
|
||||
// 来源方式
|
||||
private static String[] sources = new String[]{"直接输入", "百度跳转", "360搜索跳转", "必应跳转"};
|
||||
|
||||
// 浏览器
|
||||
private static String[] browser = new String[]{"火狐", "QQ浏览器", "360浏览器", "谷歌浏览器"};
|
||||
|
||||
// 打开方式，离开时间
|
||||
private static List<Long[]> usertimeLog = producetimes();
|
||||
|
||||
// 获取时间
|
||||
private static List<Long[]> producetimes() {
|
||||
List<Long[]> usertimelog = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
Long[] timearray = gettimes("2019-10-10 24:60:60:000");
|
||||
usertimelog.add(timearray);
|
||||
}
|
||||
return usertimelog;
|
||||
}
|
||||
|
||||
private static Long[] gettimes(String time) {
|
||||
DateFormat dataFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss:SSS");
|
||||
try {
|
||||
Date date = dataFormat.parse(time);
|
||||
long timetemp = date.getTime();
|
||||
Random random = new Random();
|
||||
int randomint = random.nextInt(10);
|
||||
long starttime = timetemp - randomint*3600*1000;
|
||||
long endtime = starttime + randomint*3600*1000;
|
||||
return new Long[]{starttime,endtime};
|
||||
}catch (ParseException e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
return new Long[]{0L, 0L};
|
||||
}
|
||||
|
||||
// 模拟发送 Http 请求到上报服务系统
|
||||
public static void send(String url, String json){
|
||||
try {
|
||||
CloseableHttpClient httpClient = HttpClientBuilder.create().build();
|
||||
HttpPost post = new HttpPost(url);
|
||||
JSONObject response = null ;
|
||||
try {
|
||||
StringEntity s = new StringEntity(json.toString(), "utf-8");
|
||||
s.setContentEncoding("utf-8");
|
||||
// 发送 json 数据需要设置 contentType
|
||||
s.setContentType("application/json");
|
||||
post.setEntity(s);
|
||||
|
||||
HttpResponse res = httpClient.execute(post);
|
||||
if(res.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
|
||||
// 返回 json 格式
|
||||
String result = EntityUtils.toString(res.getEntity());
|
||||
System.out.println(result);
|
||||
}
|
||||
}catch (Exception e){
|
||||
throw new RuntimeException();
|
||||
|
||||
}
|
||||
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Random random = new Random();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
// 频道id、类别id、产品id、用户id、打开时间、离开时间、地区、网络方式、来源方式、浏览器
|
||||
Clicklog clicklog = new Clicklog();
|
||||
|
||||
clicklog.setChannelID(channelID[random.nextInt(channelID.length)]);
|
||||
clicklog.setCategoryID(categoryID[random.nextInt(channelID.length)]);
|
||||
clicklog.setProduceID(produceID[random.nextInt(produceID.length)]);
|
||||
clicklog.setUserID(userID[random.nextInt(userID.length)]);
|
||||
clicklog.setCountry(contrys[random.nextInt(contrys.length)]);
|
||||
clicklog.setProvince(provinces[random.nextInt(provinces.length)]);
|
||||
clicklog.setCity(citys[random.nextInt(citys.length)]);
|
||||
clicklog.setNetwork(networks[random.nextInt(networks.length)]);
|
||||
clicklog.setSource(sources[random.nextInt(sources.length)]);
|
||||
clicklog.setBrowserType(browser[random.nextInt(browser.length)]);
|
||||
|
||||
Long[] times = usertimeLog.get(random.nextInt(usertimeLog.size()));
|
||||
clicklog.setEntryTime(times[0]);
|
||||
clicklog.setLeaveTime(times[1]);
|
||||
|
||||
// 将点击流日志转成字符串，发送到前端地址
|
||||
String jsonstr = JSONObject.toJSONString(clicklog);
|
||||
System.out.println(jsonstr);
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
}catch (InterruptedException e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
send("http://localhost:1234/receive", jsonstr);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
package com.henry.report.util;
|
||||
|
||||
import org.apache.kafka.clients.producer.Partitioner;
|
||||
import org.apache.kafka.common.Cluster;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description: 自定义分区
|
||||
* @Date: Create in 2019/10/9 23:00
|
||||
**/
|
||||
|
||||
public class RoundRobinPartitioner implements Partitioner {
|
||||
|
||||
// AtomicInteger 并发包下的多线程安全的整型类
|
||||
AtomicInteger counter = new AtomicInteger(0) ;
|
||||
|
||||
|
||||
// 返回值为分区号： 0、1、2
|
||||
@Override
|
||||
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
|
||||
|
||||
// 获取分区的数量
|
||||
Integer partitions = cluster.partitionCountForTopic(topic) ;
|
||||
|
||||
int curpartition = counter.incrementAndGet() % partitions ; // 当前轮询的 partition 号
|
||||
|
||||
if(counter.get() > 65535){
|
||||
counter.set(0);
|
||||
}
|
||||
|
||||
return curpartition;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(Map<String, ?> map) {
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
# 修改 Tomcat 端口号
|
||||
server.port=1234
|
||||
|
||||
#
|
||||
# Kafka
|
||||
#
|
||||
# kafka的服务器地址
|
||||
kafka.bootstrap_servers_config=master:9092,slave1:9092,slave2:9092
|
||||
# 如果出现发送失败的情况，允许重试的次数
|
||||
kafka.retries_config=0
|
||||
# 每个批次发送多大的数据，单位：字节
|
||||
kafka.batch_size_config=4096
|
||||
# 定时发送，达到 1ms 发送
|
||||
kafka.linger_ms_config=1
|
||||
# 缓存的大小，单位：字节
|
||||
kafka.buffer_memory_config=40960
|
||||
# TOPOC 名字
|
||||
kafka.topic=pyg
|
||||
@ -0,0 +1,29 @@
|
||||
package com.henry.report;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.kafka.core.KafkaTemplate;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
/**
|
||||
* @Author: Henry
|
||||
* @Description: 测试Kafka
|
||||
* @Date: Create in 2019/10/8 23:26
|
||||
**/
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest
|
||||
public class KafkaTest {
|
||||
|
||||
@Autowired
|
||||
KafkaTemplate kafkaTemplate;
|
||||
|
||||
@Test
|
||||
public void sendMsg(){
|
||||
for (int i = 0; i < 100; i++)
|
||||
kafkaTemplate.send("test", "key","this is test msg") ;
|
||||
}
|
||||
|
||||
}
|
||||
|
After Width: | Height: | Size: 253 KiB |
|
After Width: | Height: | Size: 139 KiB |
|
After Width: | Height: | Size: 55 KiB |
|
After Width: | Height: | Size: 113 KiB |
|
After Width: | Height: | Size: 205 KiB |
|
After Width: | Height: | Size: 34 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 90 KiB |
|
After Width: | Height: | Size: 191 KiB |
|
After Width: | Height: | Size: 180 KiB |
|
After Width: | Height: | Size: 60 KiB |
|
After Width: | Height: | Size: 137 KiB |
|
After Width: | Height: | Size: 430 KiB |
|
After Width: | Height: | Size: 61 KiB |
|
After Width: | Height: | Size: 74 KiB |
|
After Width: | Height: | Size: 93 KiB |
|
After Width: | Height: | Size: 137 KiB |
|
After Width: | Height: | Size: 52 KiB |
|
After Width: | Height: | Size: 157 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 155 KiB |
|
After Width: | Height: | Size: 181 KiB |
|
After Width: | Height: | Size: 3.1 KiB |
|
After Width: | Height: | Size: 228 KiB |
|
After Width: | Height: | Size: 77 KiB |
|
After Width: | Height: | Size: 144 KiB |
|
After Width: | Height: | Size: 70 KiB |
|
After Width: | Height: | Size: 160 KiB |
|
After Width: | Height: | Size: 286 KiB |
|
After Width: | Height: | Size: 21 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 120 KiB |
|
After Width: | Height: | Size: 609 KiB |
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 57 KiB |
|
After Width: | Height: | Size: 1.4 KiB |
|
After Width: | Height: | Size: 36 KiB |
|
After Width: | Height: | Size: 3.2 KiB |
|
After Width: | Height: | Size: 10 KiB |
|
After Width: | Height: | Size: 44 KiB |
|
After Width: | Height: | Size: 273 KiB |
|
After Width: | Height: | Size: 110 KiB |
|
After Width: | Height: | Size: 56 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 19 KiB |
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue