diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..6a3cba6 --- /dev/null +++ b/.classpath @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/.project b/.project new file mode 100644 index 0000000..0d2f4df --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + logcount + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..839d647 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..60105c1 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/README.md b/README.md index 2126131..f130fba 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,120 @@ -# logcount +# 日志分析系统 + +## 系统架构 + + 本使用kafka，spark，hbase开发日志分析系统。 + +![architecture](/docs/images/architecture.png "architecture") + +### 软件模块 + + * Kafka：作为日志事件的消息系统，具有分布式，可分区，可冗余的消息服务功能。 + * Spark：使用spark stream功能，实时分析消息系统中的数据，完成计算分析工作。 + * Hbase：做为后端存储，存储spark计算结构，供其他系统进行调用 + +## 环境部署 + +### 软件版本 + + * hadoop 版本 ： Hadoop相关软件如zookeeper、hadoop、hbase，使用的是cloudera的 cdh 5.2.0 版本。 + * Kafka ： 2.9.2-0.8.1.1 + +### 软件安装 + +a. 部署kafka + + tar -xzf kafka_2.9.2-0.8.1.1.tgz + +b. 编辑kafka 配置文件 + + config/server-1.properties: + broker.id=0 + port=9093 + log.dir=/tmp/kafka-logs + + config/server-2.properties: + broker.id=1 + port=9093 + log.dir=/tmp/kafka-logs + + config/server-3.properties: + broker.id=2 + port=9093 + log.dir=/tmp/kafka-logs + +c. 启动kafka + + bin/kafka-server-start.sh config/server-1.properties & + bin/kafka-server-start.sh config/server-2.properties & + bin/kafka-server-start.sh config/server-3.properties & + +d. 创建kafka topic + +> bin/kafka-topics.sh --create --zookeeper 10.10.102.191:2181, 10.10.102.192:2181, 10.10.102.193:2181 --replication-factor 3 --partitions 1 --topic recsys + +e. 查看是否创建成功 + +> bin/kafka-topics.sh --list --zookeeper localhost:2181 + +> bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic my-replicated-topic +Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs: + Topic: my-replicated-topic Partition: 0 Leader: 1 Replicas: 1,2,0 Isr: 1,2,0 + + + +f. kafka启动测试 + +> bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test +This is a message +This is another message + +> bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning +This is a message +This is another message + +g. 注意事项 + +在开发程序的时候，producer客户端必须要配置上broker的host映射信息，即使你的程序中使用的都是ip地址。 + +## 项目开发 + +### 程序部署目录 + + /libs + * Logback包：logback-classic-1.1.2.jar，logback-core-1.1.2.jar + * Kafka包（在kafka安装包lib目录中） + /conf + * Logback：logback.xml + + /webapps/recsys + * index.html + / + * logcount-1.0.jar + +### Spark_Streaming 处理数据 +### HBase 保存数据 + +创建hbase表 + + create ‘recsys_logs’,’f’ + +服务器端部署.服务器端启动了一个httpserver，该server需要将jar包中的html页面解压出来，所以先解压，后运行程序 + + jar xvf recsys-1.0.jar + + +#### 系统运行 + +客户端 + +> java -Dlogback.configurationFile=./conf/logback.xml -classpath .:libs/*:logcount-1.0.jar com.wankun.logcount.kafka.TailService dest.log + + 服务端 + +> spark-submit --class com.wankun.logcount.spark.LogStream --master spark://SparkMaster:7077 logcount-1.0.jar + + +### 注释 + + diff --git a/docs/images/architecture.png b/docs/images/architecture.png new file mode 100644 index 0000000..036c4d8 Binary files /dev/null and b/docs/images/architecture.png differ diff --git a/docs/日志分析系统.docx b/docs/日志分析系统.docx new file mode 100644 index 0000000..91fd8e3 Binary files /dev/null and b/docs/日志分析系统.docx differ diff --git a/docs/架构.ppt b/docs/架构.ppt new file mode 100644 index 0000000..6ea3b53 Binary files /dev/null and b/docs/架构.ppt differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..e2512bc --- /dev/null +++ b/pom.xml @@ -0,0 +1,87 @@ + + 4.0.0 + + com.wankun + logcount + 1.0 + jar + + logcount + http://maven.apache.org + + + + cloudera + https://repository.cloudera.com/cloudera/cloudera-repos + + + + + UTF-8 + 1.1.0-cdh5.2.0 + 0.98.6-cdh5.2.0 + + + + jdk.tools + jdk.tools + 1.7.0_51 + system + ${JAVA_HOME}/lib/tools.jar + + + ch.qos.logback + logback-classic + 1.1.2 + + + org.apache.kafka + kafka_2.10 + 0.8.1.1 + + + + org.apache.spark + spark-streaming_2.10 + ${spark.version} + + + org.apache.spark + spark-streaming-kafka_2.10 + ${spark.version} + + + + org.apache.hbase + hbase-client + ${hbase.version} + + + org.apache.hbase + hbase-server + ${hbase.version} + + + + junit + junit + 4.11 + test + + + + + + + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + UTF-8 + + + + + diff --git a/src/main/java/com/wankun/logcount/kafka/MsgSender.java b/src/main/java/com/wankun/logcount/kafka/MsgSender.java new file mode 100644 index 0000000..e8c1c45 --- /dev/null +++ b/src/main/java/com/wankun/logcount/kafka/MsgSender.java @@ -0,0 +1,54 @@ +package com.wankun.logcount.kafka; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.producer.ProducerConfig; + +public class MsgSender extends Thread { + private final static Logger logger = LoggerFactory.getLogger(MsgSender.class); + + private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HH:mm:ss"); + + private BlockingQueue queue; + private Producer producer; + + public MsgSender(BlockingQueue queue) { + this.queue = queue; + + Properties props = new Properties(); + props.put("metadata.broker.list", "10.10.102.191:9092,10.10.102.192:9092,10.10.102.193:9092"); + props.put("serializer.class", "kafka.serializer.StringEncoder"); + // props.put("partitioner.class", "example.producer.SimplePartitioner"); + props.put("request.required.acks", "1"); + + ProducerConfig config = new ProducerConfig(props); + + producer = new Producer(config); + } + + @Override + public void run() { + while (true) { + try { + String line = queue.take(); + if (line != null && !line.replace("\n", "").replace("\r", "").equals("")) { + String timestamp = sdf.format(new Date()); + KeyedMessage data = new KeyedMessage("recsys", timestamp, line); + logger.debug("sending kv :( {}:{})", timestamp, line); + producer.send(data); + } + } catch (InterruptedException e) { + logger.error("kafka producer 消息发送失败", e); + } + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/wankun/logcount/kafka/TailLog.java b/src/main/java/com/wankun/logcount/kafka/TailLog.java new file mode 100644 index 0000000..b013870 --- /dev/null +++ b/src/main/java/com/wankun/logcount/kafka/TailLog.java @@ -0,0 +1,132 @@ +package com.wankun.logcount.kafka; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.CharBuffer; +import java.util.concurrent.BlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TailLog extends Thread { + private final static Logger logger = LoggerFactory.getLogger(TailLog.class); + + private BlockingQueue queue; + private String logname; + + private CharBuffer buf = CharBuffer.allocate(4096); + + // private ByteBuffer buf = ByteBuffer.allocate(4096); + + public TailLog(BlockingQueue queue, String logname) { + this.queue = queue; + this.logname = logname; + } + + @Override + public void run() { + BufferedReader reader = null; + try { + // Path logpath=Paths.get(logname); + // File posfile = + // logpath.getParent().resolve("."+logpath.getFileName()+".pos").toFile(); + reader = new BufferedReader(new FileReader(new File(logname))); + + long filesize = 0; + while (true) { + // 判断文件是否已经切换 + if (filesize > new File(logname).length()) { + logger.debug("filesize :{} current system file size :{} . Log file switchover!", filesize, + new File(logname).length()); + try { + // 在切换读文件前，读取文件全部内容 + StringBuilder line = new StringBuilder(); + while (reader.read(buf) > 0) { + buf.flip(); + synchronized (buf) { + // 读buffer 并解析 + for (int i = 0; i < buf.limit(); i++) { + char c = buf.get(); + line.append(c); + if ((c == '\n') || (c == '\r')) + if (line.length() > 0) { + queue.put(line.toString()); + line = new StringBuilder(); + } + } + } + } + queue.put(line.toString()); + buf.clear(); + + // 切换读文件 + if (reader != null) + reader.close(); + reader = new BufferedReader(new FileReader(new File(logname))); + } catch (Exception e) { + logger.error("文件 {} 不存在", logname, e); + Thread.currentThread().sleep(10000); + continue; + } + } + + for (int retrys = 10; retrys > 0; retrys--) { + int bufread = reader.read(buf); + if (bufread < 0) { + if (retrys > 0) + Thread.currentThread().sleep(1000); + else { + // 等待10s后无新数据读出 + synchronized (buf) { + // 等待 cachetime 秒后文件仍未写入 + buf.flip(); + char[] dst = new char[buf.length()]; + buf.get(dst); + buf.clear(); + queue.put(new String(dst)); + } + } + } else { + filesize = new File(logname).length(); + retrys = -1; + + buf.flip(); + synchronized (buf) { + // 读buffer 并解析 + StringBuilder line = new StringBuilder(); + for (int i = 0; i < buf.limit(); i++) { + char c = buf.get(); + line.append(c); + if ((c == '\n') || (c == '\r')) + if (line.length() > 0) { + queue.put(line.toString()); + line = new StringBuilder(); + } + } + // 接着写不完整的数据 + buf.compact(); + if (line.length() > 0) { + buf.append(line); + } + } + break; + } + } + } + } catch (Exception e) { + logger.error("文件读取失败", e); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + logger.error("文件 reader 关闭失败", e); + } + } + } + + } + +} \ No newline at end of file diff --git a/src/main/java/com/wankun/logcount/kafka/TailService.java b/src/main/java/com/wankun/logcount/kafka/TailService.java new file mode 100644 index 0000000..4017fbd --- /dev/null +++ b/src/main/java/com/wankun/logcount/kafka/TailService.java @@ -0,0 +1,28 @@ +package com.wankun.logcount.kafka; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TailService { + + private final static Logger logger = LoggerFactory.getLogger(TailService.class); + + public static void main(String[] args) { + if (args.length < 1) { + logger.error("Usage : TailService [logfile]"); + System.exit(0); + } + + BlockingQueue queue = new ArrayBlockingQueue(10000); + + for (String arg : args) { + new TailLog(queue, arg).start(); + } + + new MsgSender(queue).start(); + } + +} diff --git a/src/main/java/com/wankun/logcount/spark/LogStream.java b/src/main/java/com/wankun/logcount/spark/LogStream.java new file mode 100644 index 0000000..1c307eb --- /dev/null +++ b/src/main/java/com/wankun/logcount/spark/LogStream.java @@ -0,0 +1,141 @@ +package com.wankun.logcount.spark; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.net.NetUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.streaming.Duration; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream; +import org.apache.spark.streaming.api.java.JavaStreamingContext; +import org.apache.spark.streaming.kafka.KafkaUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import scala.Tuple2; + +import com.google.common.collect.Maps; + +public class LogStream { + private final static Logger logger = LoggerFactory.getLogger(LogStream.class); + + private static HConnection connection = null; + private static HTableInterface table = null; + private static HttpServer2 infoServer = null; + + public static void openHBase(String tablename) throws IOException { + Configuration conf = HBaseConfiguration.create(); + synchronized (HConnection.class) { + if (connection == null) + connection = HConnectionManager.createConnection(conf); + } + + synchronized (HTableInterface.class) { + if (table == null) { + table = connection.getTable("recsys_logs"); + } + } + + /* start http info server */ + HttpServer2.Builder builder = new HttpServer2.Builder().setName("recsys").setConf(conf); + InetSocketAddress addr = NetUtils.createSocketAddr("0.0.0.0", 8089); + builder.addEndpoint(URI.create("http://" + NetUtils.getHostPortString(addr))); + infoServer = builder.build(); + + infoServer.addServlet("monitor", "/monitor", RecsysLogs.class); + infoServer.setAttribute("htable", table); + infoServer.setAttribute("conf", conf); + infoServer.start(); + } + + public static void closeHBase() { + if (table != null) + try { + table.close(); + } catch (IOException e) { + logger.error("关闭 table 出错", e); + } + if (connection != null) + try { + connection.close(); + } catch (IOException e) { + logger.error("关闭 connection 出错", e); + } + if (infoServer != null && infoServer.isAlive()) + try { + infoServer.stop(); + } catch (Exception e) { + logger.error("关闭 infoServer 出错", e); + } + } + + public static void main(String[] args) { + // open hbase + try { + openHBase("logcount"); + } catch (IOException e) { + logger.error("建立HBase 连接失败", e); + System.exit(-1); + } + + SparkConf conf = new SparkConf().setAppName("recsys log stream"); + JavaStreamingContext ssc = new JavaStreamingContext(conf, new Duration(1000)); + + Map topicMap = Maps.newHashMap(); + topicMap.put("recsys", 4); + JavaPairReceiverInputDStream logstream = KafkaUtils.createStream(ssc, + "10.10.102.191:2181,10.10.102.192:2181,10.10.102.193:2181", "recsys_group1", topicMap); + + JavaDStream lines = logstream.map(new Function, String>() { + private static final long serialVersionUID = -1801798365843350169L; + + @Override + public String call(Tuple2 tuple2) { + return tuple2._2(); + } + }).filter(new Function() { + private static final long serialVersionUID = 7786877762996470593L; + + @Override + public Boolean call(String msg) throws Exception { + return msg.indexOf("character service received paramters:") > 0; + } + }); + + // 统计Log中的数据，并保存到HBase中 + JavaDStream nums = lines.count(); + nums.foreachRDD(new Function, Void>() { + + private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HH:mm:ss"); + + @Override + public Void call(JavaRDD rdd) throws Exception { + Long num = rdd.take(1).get(0); + String ts = sdf.format(new Date()); + Put put = new Put(Bytes.toBytes(ts)); + put.add(Bytes.toBytes("f"), Bytes.toBytes("nums"), Bytes.toBytes(num)); + table.put(put); + return null; + } + }); + + ssc.start(); + ssc.awaitTermination(); + } + +} diff --git a/src/main/java/com/wankun/logcount/spark/RecsysLogs.java b/src/main/java/com/wankun/logcount/spark/RecsysLogs.java new file mode 100644 index 0000000..cb5b804 --- /dev/null +++ b/src/main/java/com/wankun/logcount/spark/RecsysLogs.java @@ -0,0 +1,49 @@ +package com.wankun.logcount.spark; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; + +public class RecsysLogs extends HttpServlet { + + private static final long serialVersionUID = 4289573629015709424L; + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + HTableInterface htable = (HTableInterface) getServletContext().getAttribute("htable"); + Scan scan = new Scan(); + scan.addColumn(Bytes.toBytes("f"), Bytes.toBytes("nums")); + scan.setReversed(true); + // scan.setMaxResultSize(20); + scan.setFilter(new PageFilter(20)); + ResultScanner scanner = htable.getScanner(scan); + StringBuilder sb = new StringBuilder(); + for (Result res : scanner) { + Cell cell = res.getColumnLatestCell(Bytes.toBytes("f"), Bytes.toBytes("nums")); + Long nums = Bytes.toLong(CellUtil.cloneValue(cell)); + String key = Bytes.toString(CellUtil.cloneRow(cell)); + sb.append(key + " : " + nums + "\n"); + } + scanner.close(); + + resp.getWriter().write(sb.toString()); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + this.doGet(req, resp); + } + +} diff --git a/src/main/resources/hbase-site.xml b/src/main/resources/hbase-site.xml new file mode 100644 index 0000000..1fa2c4f --- /dev/null +++ b/src/main/resources/hbase-site.xml @@ -0,0 +1,214 @@ + + + + + + hbase.cluster.distributed + true + + + + hbase.rootdir + hdfs://dwztgame/hbase + + + + dfs.datanode.max.xcievers + 40960 + + + + hbase.zookeeper.quorum + node01.dw.ztgame.com:2181,node02.dw.ztgame.com:2181,node03.dw.ztgame.com:2181 + + + + hbase.regionserver.handler.count + 200 + Count of RPC Server instances spun up on RegionServers + Same property is used by the Master for count of master handlers. + Default is 10. + + + + hbase.regionserver.flushlogentries + 500 + Sync the HLog to HDFS when it has accumulated this many + entries. Default 1. Value is checked on every HLog.hflush + + + + hbase.regionserver.optionallogflushinterval + 2000 + Sync the HLog to the HDFS after this interval if it has not + accumulated enough entries to trigger a sync. Default 1 second. Units: + milliseconds. + + + + hbase.regionserver.thread.splitcompactcheckfrequency + 600000 + How often a region server runs the split/compaction check. + + + + hbase.regions.slop + 0 + Rebalance if any regionserver has average + (average * slop) regions. + Default is 0% slop. + + + + hbase.server.thread.wakefrequency + 5000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as log roller. + + + + hbase.hregion.memstore.flush.size + 134217728 + Memstore will be flushed to disk if size of the memstore + exceeds this number of bytes. Value is checked by a thread that runs + every hbase.server.thread.wakefrequency. + + + + hbase.hregion.memstore.block.multiplier + 6 + + Block updates if memstore has hbase.hregion.block.memstore + time hbase.hregion.flush.size bytes. Useful preventing + runaway memstore during spikes in update traffic. Without an + upper-bound, memstore fills such that when it flushes the + resultant flush files take a long time to compact or split, or + worse, we OOME. + + + + hbase.hregion.memstore.mslab.enabled + true + Experimental: Enables the MemStore-Local Allocation Buffer, + a feature which works to prevent heap fragmentation under + heavy write loads. This can reduce the frequency of stop-the-world + GC pauses on large heaps. + + + + hfile.block.cache.size + 0.2 + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by HFile/StoreFile. Default of 0.2 means allocate 20%. + Set to 0 to disable. + + + + hbase.regionserver.nbreservationblocks + 10 + The number of resevoir blocks of memory release on + OOME so we can cleanup properly before server shutdown. + + + + hbase.regionserver.global.memstore.upperLimit + 0.5 + Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced. Defaults to 40% of heap + + + + hbase.regionserver.global.memstore.lowerLimit + 0.4 + When memstores are being forced to flush to make room in + memory, keep flushing until we hit this mark. Defaults to 35% of heap. + This value equaggl to hbase.regionserver.global.memstore.upperLimit causes + the minimum possible flushing to occur when updates are blocked due to + memstore limiting. + + + + hbase.hregion.max.filesize + 2684354560 + + Maximum HStoreFile size. If any one of a column families' HStoreFiles has + grown to exceed this value, the hosting HRegion is split in two. + Default: 256M. + + + + hbase.snapshot.enabled + true + + + + hbase.regionserver.regionSplitLimit + 200 + Limit for the number of regions after which no more region + splitting should take place. This is not a hard limit for the number of + regions but acts as a guideline for the regionserver to stop splitting after + a certain limit. Default is set to MAX_INT; i.e. do not block splitting. + + + + hbase.hstore.compactionThreshold + 4 + If more than this number of HStoreFiles in any one HStore + (one HStoreFile is written per flush of memstore) then a compaction + is run to rewrite all HStoreFiles files as one. Larger numbers + put off compaction but when it runs, it takes longer to complete. + + + + hbase.hstore.blockingStoreFiles + 12 + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + + + + hbase.hstore.compaction.max + 6 + Max number of HStoreFiles to compact per 'minor' compaction. + + + + hbase.hregion.majorcompaction + 172800000 + The time (in miliseconds) between 'major' compactions of all + HStoreFiles in a region. Default: 1 day. + .set to 0 to disable automated major compactions. + + + + io.storefile.bloom.enabled + true + + + + hbase.replication + true + + + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..8dd2893 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,53 @@ + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + logs/recsys.log + + logs/recsys.log.%i.bak + 1 + 12 + + + 100MB + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/src/main/resources/webapps/recsys/index.html b/src/main/resources/webapps/recsys/index.html new file mode 100644 index 0000000..f9ba4cf --- /dev/null +++ b/src/main/resources/webapps/recsys/index.html @@ -0,0 +1,11 @@ + + + + recsys logs 监控系统 + + + +

欢迎使用recsys logs 监控系统！


+ 查看日志最新情况 + +