@ -0,0 +1,14 @@
|
||||
# https://editorconfig.org/
|
||||
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.md] # markdown 尾随空格不允许删除
|
||||
trim_trailing_whitespace = false
|
||||
@ -0,0 +1,89 @@
|
||||
{
|
||||
"adultSZT": 4.75,
|
||||
"buslist": [
|
||||
{
|
||||
"segmentlist": [
|
||||
{
|
||||
"busid": "440300024064",
|
||||
"endname": "会展中心",
|
||||
"passdepotname": "老街 大剧院 科学馆 华强路 岗厦",
|
||||
"startname": "国贸"
|
||||
},
|
||||
{
|
||||
"busid": "440300024075",
|
||||
"endname": "白石龙",
|
||||
"passdepotname": "会展中心 市民中心 少年宫 莲花北 上梅林 民乐 白石龙",
|
||||
"startname": "会展中心"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"childSZT": 2.5,
|
||||
"count": "1",
|
||||
"lineList": [
|
||||
{
|
||||
"arriveStation": "会展中心站",
|
||||
"code": "0108",
|
||||
"departureStation": "国贸站",
|
||||
"endTime": "23:52",
|
||||
"firstTime": "06:28",
|
||||
"highNterval": "2分45秒",
|
||||
"line": "01",
|
||||
"lineName": "罗宝线",
|
||||
"nextStation": "老街站",
|
||||
"passdepotname": "国贸|老街|大剧院|科学馆|华强路|岗厦|会展中心",
|
||||
"shortNterval": "5分30秒",
|
||||
"stationNO": 6,
|
||||
"terminus": "机场东站",
|
||||
"transferDistance": "100",
|
||||
"transferTime": "30",
|
||||
"travelTime": 580,
|
||||
"zhandianwanzhi": "guomao.html"
|
||||
},
|
||||
{
|
||||
"arriveStation": "白石龙站",
|
||||
"code": "0409",
|
||||
"departureStation": "会展中心站",
|
||||
"endTime": "23:12",
|
||||
"firstTime": "06:42",
|
||||
"highNterval": "6分钟",
|
||||
"line": "04",
|
||||
"lineName": "龙华线",
|
||||
"nextStation": "市民中心站",
|
||||
"passdepotname": "会展中心|市民中心|少年宫|莲花北|上梅林|民乐|白石龙",
|
||||
"shortNterval": "6分钟",
|
||||
"stationNO": 6,
|
||||
"terminus": "清湖站",
|
||||
"transferDistance": "1",
|
||||
"transferTime": "1",
|
||||
"travelTime": 910,
|
||||
"zhandianwanzhi": "4hxhuizhanzhongxin.html"
|
||||
},
|
||||
{
|
||||
"arriveStation": "",
|
||||
"code": "",
|
||||
"departureStation": "",
|
||||
"endTime": "",
|
||||
"firstTime": "",
|
||||
"highNterval": "",
|
||||
"line": "xx",
|
||||
"lineName": "",
|
||||
"nextStation": "",
|
||||
"passdepotname": "",
|
||||
"shortNterval": "",
|
||||
"stationNO": 0,
|
||||
"terminus": "",
|
||||
"transferDistance": "",
|
||||
"transferTime": "",
|
||||
"travelTime": 0,
|
||||
"zhandianwanzhi": "baishilong.html"
|
||||
}
|
||||
],
|
||||
"qidiancode": "0102",
|
||||
"starStation": "国贸站",
|
||||
"terminus": "白石龙站",
|
||||
"ticketPrice": 5,
|
||||
"times": 1,
|
||||
"useTime": 1616,
|
||||
"zhondiancode": "0409"
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
### 深圳地铁官网线路规划查询接口
|
||||
|
||||
### 最小化查询
|
||||
POST https://www.szmc.net/algorithm/Ticketing/MinTimeJson.do
|
||||
Content-Type: application/x-www-form-urlencoded; charset=UTF-8
|
||||
|
||||
departureStation=0102&arriveStation=0409&ridingType=0
|
||||
|
||||
|
||||
### 错误示例
|
||||
POST https://www.szmc.net/algorithm/Ticketing/MinTimeJson.do
|
||||
Content-Type: application/x-www-form-urlencoded; charset=UTF-8
|
||||
|
||||
departureStation=0103&arriveStation=0310&ridingType=0
|
||||
|
||||
###
|
||||
@ -0,0 +1,7 @@
|
||||
### appKey 自己申请 https://opendata.sz.gov.cn/data/api/toApiDetails/29200_00403601
|
||||
GET https://opendata.sz.gov.cn/api/29200_00403601/1/service.xhtml?page=1&rows=100&appKey={换成你的appKey}
|
||||
|
||||
### 参考 .file/.api/page1x100.json
|
||||
|
||||
### 133,7000 理论总数，单次返回 1000 条会很慢
|
||||
GET https://opendata.sz.gov.cn/api/29200_00403601/1/service.xhtml?page=1337&rows=1000&appKey={换成你的appKey}
|
||||
|
After Width: | Height: | Size: 936 KiB |
|
After Width: | Height: | Size: 99 KiB |
|
After Width: | Height: | Size: 56 KiB |
|
After Width: | Height: | Size: 80 KiB |
|
After Width: | Height: | Size: 66 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 162 KiB |
|
After Width: | Height: | Size: 1.7 KiB |
|
After Width: | Height: | Size: 489 KiB |
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 122 KiB |
|
After Width: | Height: | Size: 93 KiB |
|
After Width: | Height: | Size: 94 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 97 KiB |
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 104 KiB |
|
After Width: | Height: | Size: 93 KiB |
|
After Width: | Height: | Size: 92 KiB |
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 108 KiB |
|
After Width: | Height: | Size: 102 KiB |
|
After Width: | Height: | Size: 110 KiB |
|
After Width: | Height: | Size: 124 KiB |
|
After Width: | Height: | Size: 98 KiB |
|
After Width: | Height: | Size: 106 KiB |
|
After Width: | Height: | Size: 100 KiB |
|
After Width: | Height: | Size: 100 KiB |
|
After Width: | Height: | Size: 98 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 5.4 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 4.9 KiB |
|
After Width: | Height: | Size: 102 KiB |
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 7.8 KiB |
|
After Width: | Height: | Size: 55 KiB |
|
After Width: | Height: | Size: 14 KiB |
|
After Width: | Height: | Size: 9.2 KiB |
|
After Width: | Height: | Size: 4.0 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 8.4 KiB |
|
After Width: | Height: | Size: 7.3 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 115 KiB |
|
After Width: | Height: | Size: 115 KiB |
|
After Width: | Height: | Size: 117 KiB |
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 94 KiB |
|
After Width: | Height: | Size: 142 KiB |
|
After Width: | Height: | Size: 255 KiB |
|
After Width: | Height: | Size: 143 KiB |
|
After Width: | Height: | Size: 36 KiB |
|
After Width: | Height: | Size: 131 KiB |
|
After Width: | Height: | Size: 56 KiB |
|
After Width: | Height: | Size: 75 KiB |
|
After Width: | Height: | Size: 101 KiB |
|
After Width: | Height: | Size: 203 KiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 122 KiB |
|
After Width: | Height: | Size: 59 KiB |
|
After Width: | Height: | Size: 377 KiB |
|
After Width: | Height: | Size: 124 KiB |
|
After Width: | Height: | Size: 95 KiB |
|
After Width: | Height: | Size: 66 KiB |
|
After Width: | Height: | Size: 23 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 110 KiB |
|
After Width: | Height: | Size: 212 KiB |
|
After Width: | Height: | Size: 173 KiB |
|
After Width: | Height: | Size: 163 KiB |
|
After Width: | Height: | Size: 394 KiB |
|
After Width: | Height: | Size: 52 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 2.4 MiB |
@ -0,0 +1 @@
|
||||
custom: ['https://raw.githubusercontent.com/geekyouth/SZT-bigdata/master/.file/.pic/.pay/pay-mini.png']
|
||||
@ -0,0 +1,14 @@
|
||||
---
|
||||
name: Custom issue template | 自定义提问模板
|
||||
about: Please file issue with rule | 请按照规范提问交流
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
# 我的主机操作系统和版本号：
|
||||
|
||||
# 问题截图：
|
||||
|
||||
# 期望的运行结果：
|
||||
@ -0,0 +1,3 @@
|
||||
**/.idea/
|
||||
**/target/
|
||||
**.iml
|
||||
@ -0,0 +1,13 @@
|
||||
Copyright [2022] [极客青年 forsupergeeker@gmail.com]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -0,0 +1,46 @@
|
||||
Copyright (c) <year> <copyright holders>
|
||||
|
||||
"Anti 996" License Version 1.0 (Draft)
|
||||
|
||||
Permission is hereby granted to any individual or legal entity
|
||||
obtaining a copy of this licensed work (including the source code,
|
||||
documentation and/or related items, hereinafter collectively referred
|
||||
to as the "licensed work"), free of charge, to deal with the licensed
|
||||
work for any purpose, including without limitation, the rights to use,
|
||||
reproduce, modify, prepare derivative works of, distribute, publish
|
||||
and sublicense the licensed work, subject to the following conditions:
|
||||
|
||||
1. The individual or the legal entity must conspicuously display,
|
||||
without modification, this License and the notice on each redistributed
|
||||
or derivative copy of the Licensed Work.
|
||||
|
||||
2. The individual or the legal entity must strictly comply with all
|
||||
applicable laws, regulations, rules and standards of the jurisdiction
|
||||
relating to labor and employment where the individual is physically
|
||||
located or where the individual was born or naturalized; or where the
|
||||
legal entity is registered or is operating (whichever is stricter). In
|
||||
case that the jurisdiction has no such laws, regulations, rules and
|
||||
standards or its laws, regulations, rules and standards are
|
||||
unenforceable, the individual or the legal entity are required to
|
||||
comply with Core International Labor Standards.
|
||||
|
||||
3. The individual or the legal entity shall not induce, suggest or force
|
||||
its employee(s), whether full-time or part-time, or its independent
|
||||
contractor(s), in any methods, to agree in oral or written form, to
|
||||
directly or indirectly restrict, weaken or relinquish his or her
|
||||
rights or remedies under such laws, regulations, rules and standards
|
||||
relating to labor and employment as mentioned above, no matter whether
|
||||
such written or oral agreements are enforceable under the laws of the
|
||||
said jurisdiction, nor shall such individual or the legal entity
|
||||
limit, in any methods, the rights of its employee(s) or independent
|
||||
contractor(s) from reporting or complaining to the copyright holder or
|
||||
relevant authorities monitoring the compliance of the license about
|
||||
its violation(s) of the said license.
|
||||
|
||||
THE LICENSED WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
OTHERWISE, ARISING FROM, OUT OF OR IN ANY WAY CONNECTION WITH THE
|
||||
LICENSED WORK OR THE USE OR OTHER DEALINGS IN THE LICENSED WORK.
|
||||
@ -0,0 +1,200 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>cn.java666</groupId>
|
||||
<artifactId>SZT-ETL</artifactId>
|
||||
<version>0.1</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>ETL-Flink</artifactId>
|
||||
|
||||
<description>
|
||||
该模块涉及技术栈：
|
||||
flink-1.10
|
||||
kafka-2.1
|
||||
elasticsearch-7.0
|
||||
hbase-1.4 (兼容 cdh-6.2 hbase-2.1)
|
||||
redis-2.8 (兼容 redis-3.5)
|
||||
|
||||
主要实现 flink 各种连接器的读写和 ETL
|
||||
</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>cn.java666</groupId>
|
||||
<artifactId>SZT-common</artifactId>
|
||||
<version>0.1</version>
|
||||
</dependency>
|
||||
|
||||
<!-- flink 核心依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-scala_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>scala-library</artifactId>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>scala-parser-combinators_2.11</artifactId>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- 各种连接器 -->
|
||||
<!-- redis 官方连接器，只能写不能读 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.bahir</groupId>
|
||||
<artifactId>flink-connector-redis_2.11</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- ES7 支持 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-connector-elasticsearch7_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
</dependency>
|
||||
<!-- https://stackoverflow.com/questions/48313461/elasticsearch-6-0-1-nosuchfielderror-lucene-6-0-0/60613430#60613430 -->
|
||||
<!-- 解决 lucene 兼容问题 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>8.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Kafka 最佳 CP -->
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-kafka-0.11 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-connector-kafka-0.11_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-hbase -->
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-hbase_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 工具类 -->
|
||||
<dependency>
|
||||
<groupId>cn.hutool</groupId>
|
||||
<artifactId>hutool-all</artifactId>
|
||||
<version>5.2.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
<version>1.2.83</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<!--<scope>compile</scope>-->
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
<version>2.8.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.0.3</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- 该插件用于将Scala代码编译成class文件 -->
|
||||
<plugin>
|
||||
<groupId>net.alchim31.maven</groupId>
|
||||
<artifactId>scala-maven-plugin</artifactId>
|
||||
<version>3.4.6</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<!-- 声明绑定到maven的compile阶段 -->
|
||||
<goals>
|
||||
<goal>testCompile</goal>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.1.0</version>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<!-- 主类 -->
|
||||
<!--<mainClass>cn.java666.etlflink.sink.Jsons2Redis</mainClass>-->
|
||||
</transformer>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
|
||||
<resource>reference.conf</resource>
|
||||
</transformer>
|
||||
</transformers>
|
||||
<filters>
|
||||
<filter>
|
||||
<!--不要拷贝 META-INF 目录下的签名，否则会引起 SecurityExceptions 。 -->
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,5 @@
|
||||
kafka.broker-list=cdh231:9092,cdh232:9092,cdh233:9092
|
||||
|
||||
#kafka.producer.topic=topic-flink-szt-all
|
||||
#kafka.producer.topic=topic-flink-szt-test
|
||||
kafka.producer.topic=topic-flink-szt
|
||||