commit
861eb45025
@ -0,0 +1,22 @@
|
||||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
target/
|
||||
/var
|
||||
/*/var/
|
||||
pom.xml.versionsBackup
|
||||
test-output/
|
||||
/atlassian-ide-plugin.xml
|
||||
.idea
|
||||
.DS_Store
|
||||
.classpath
|
||||
.settings
|
||||
.project
|
||||
temp-testng-customsuite.xml
|
||||
test-output
|
||||
.externalToolBuilders
|
||||
*~
|
||||
benchmark_outputs
|
||||
*.pyc
|
||||
*.class
|
||||
.checkstyle
|
||||
@ -0,0 +1,19 @@
|
||||
language: java
|
||||
|
||||
jdk:
|
||||
- oraclejdk8
|
||||
|
||||
env:
|
||||
global:
|
||||
- MAVEN_OPTS="-Xmx256M"
|
||||
|
||||
sudo: false
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.m2/io
|
||||
- $HOME/.m2/org
|
||||
|
||||
install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V -q -T 2
|
||||
|
||||
script: mvn test -Dair.check.skip-dependency=true
|
||||
@ -0,0 +1,11 @@
|
||||
# Contributing to Presto
|
||||
|
||||
## Contributor License Agreement ("CLA")
|
||||
|
||||
In order to accept your pull request, we need you to submit a CLA. You only need to do this once, so if you've done this for another Facebook open source project, you're good to go. If you are submitting a pull request for the first time, just let us know that you have completed the CLA and we can cross-check with your GitHub username.
|
||||
|
||||
Complete your CLA here: <https://code.facebook.com/cla>
|
||||
|
||||
## License
|
||||
|
||||
By contributing to Presto, you agree that your contributions will be licensed under the [Apache License Version 2.0 (APLv2)](LICENSE).
|
||||
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -0,0 +1,73 @@
|
||||
# Presto
|
||||
|
||||
Presto is a distributed SQL query engine for big data.
|
||||
|
||||
See the [User Manual](https://prestodb.io/docs/current/) for deployment instructions and end user documentation.
|
||||
|
||||
## Requirements
|
||||
|
||||
* Mac OS X or Linux
|
||||
* Java 8, 64-bit
|
||||
* Maven 3.2.3+ (for building)
|
||||
* Python 2.4+ (for running with the launcher script)
|
||||
|
||||
## Building Presto
|
||||
|
||||
Presto is a standard Maven project. Simply run the following command from the project root directory:
|
||||
|
||||
mvn clean install
|
||||
|
||||
On the first build, Maven will download all the dependencies from the internet and cache them in the local repository (`~/.m2/repository`), which can take a considerable amount of time. Subsequent builds will be faster.
|
||||
|
||||
Presto has a comprehensive set of unit tests that can take several minutes to run. You can disable the tests when building:
|
||||
|
||||
mvn clean install -DskipTests
|
||||
|
||||
## Running Presto in your IDE
|
||||
|
||||
### Overview
|
||||
|
||||
After building Presto for the first time, you can load the project into your IDE and run the server. We recommend using [IntelliJ IDEA](http://www.jetbrains.com/idea/). Because Presto is a standard Maven project, you can import it into your IDE using the root `pom.xml` file. In IntelliJ, choose Open Project from the Quick Start box or choose Open from the File menu and select the root `pom.xml` file.
|
||||
|
||||
After opening the project in IntelliJ, double check that the Java SDK is properly configured properly for the project:
|
||||
|
||||
* Open the File menu and select Project Structure
|
||||
* In the SDKs section, ensure that a 1.8 JDK is selected (create one if none exist)
|
||||
* In the Project section, ensure the Project language level is set to 8.0 as Presto makes use of several Java 8 language features
|
||||
|
||||
Presto comes with sample configuration that should work out-of-the-box for development. Use the following options to create a run configuration:
|
||||
|
||||
* Main Class: `com.facebook.presto.server.PrestoServer`
|
||||
* VM Options: `-ea -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties`
|
||||
* Working directory: `$MODULE_DIR$`
|
||||
* Use classpath of module: `presto-main`
|
||||
|
||||
The working directory should be the `presto-main` subdirectory. In IntelliJ, using `$MODULE_DIR$` accomplishes this automatically.
|
||||
|
||||
Additionally, the Hive plugin must be configured with location of your Hive metastore Thrift service. Add the following to the list of VM options, replacing `localhost:9083` with the correct host and port (or use the below value if you do not have a Hive metastore):
|
||||
|
||||
-Dhive.metastore.uri=thrift://localhost:9083
|
||||
|
||||
### Using SOCKS for Hive or HDFS
|
||||
|
||||
If your Hive metastore or HDFS cluster is not directly accessible to your local machine, you can use SSH port forwarding to access it. Setup a dynamic SOCKS proxy with SSH listening on local port 1080:
|
||||
|
||||
ssh -v -N -D 1080 server
|
||||
|
||||
Then add the following to the list of VM options:
|
||||
|
||||
-Dhive.metastore.thrift.client.socks-proxy=localhost:1080
|
||||
|
||||
### Running the CLI
|
||||
|
||||
Start the CLI to connect to the server and run SQL queries:
|
||||
|
||||
presto-cli/target/presto-cli-*-executable.jar
|
||||
|
||||
Run a query to see the nodes in the cluster:
|
||||
|
||||
SELECT * FROM system.runtime.nodes;
|
||||
|
||||
In the sample configuration, the Hive connector is mounted in the `hive` catalog, so you can run the following queries to show the tables in the Hive database `default`:
|
||||
|
||||
SHOW TABLES FROM hive.default;
|
||||
@ -0,0 +1,916 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>airbase</artifactId>
|
||||
<version>39</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>0.107</version>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<name>presto-root</name>
|
||||
<description>Presto</description>
|
||||
<url>https://github.com/facebook/presto</url>
|
||||
|
||||
<inceptionYear>2012</inceptionYear>
|
||||
|
||||
<licenses>
|
||||
<license>
|
||||
<name>Apache License 2.0</name>
|
||||
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
|
||||
<scm>
|
||||
<connection>scm:git:git://github.com/facebook/presto.git</connection>
|
||||
<url>https://github.com/facebook/presto</url>
|
||||
<tag>0.107</tag>
|
||||
</scm>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.basedir}</air.main.basedir>
|
||||
|
||||
<air.check.skip-extended>true</air.check.skip-extended>
|
||||
<air.check.skip-license>false</air.check.skip-license>
|
||||
|
||||
<air.check.fail-checkstyle>true</air.check.fail-checkstyle>
|
||||
<air.check.skip-checkstyle>false</air.check.skip-checkstyle>
|
||||
|
||||
<dep.antlr.version>4.3</dep.antlr.version>
|
||||
<dep.airlift.version>0.110</dep.airlift.version>
|
||||
<dep.packaging.version>${dep.airlift.version}</dep.packaging.version>
|
||||
<dep.slice.version>0.14</dep.slice.version>
|
||||
|
||||
<cli.skip-execute>true</cli.skip-execute>
|
||||
<cli.main-class>None</cli.main-class>
|
||||
|
||||
<!-- use a fractional hour timezone offset for tests -->
|
||||
<air.test.timezone>Asia/Katmandu</air.test.timezone>
|
||||
<air.test.parallel>methods</air.test.parallel>
|
||||
<air.test.thread-count>4</air.test.thread-count>
|
||||
<air.test.jvmsize>1792m</air.test.jvmsize>
|
||||
|
||||
<air.javadoc.lint>-missing</air.javadoc.lint>
|
||||
</properties>
|
||||
|
||||
<modules>
|
||||
<module>presto-spi</module>
|
||||
<module>presto-kafka</module>
|
||||
<module>presto-cassandra</module>
|
||||
<module>presto-orc</module>
|
||||
<module>presto-hive</module>
|
||||
<module>presto-hive-hadoop1</module>
|
||||
<module>presto-hive-hadoop2</module>
|
||||
<module>presto-hive-cdh4</module>
|
||||
<module>presto-hive-cdh5</module>
|
||||
<module>presto-example-http</module>
|
||||
<module>presto-tpch</module>
|
||||
<module>presto-raptor</module>
|
||||
<module>presto-base-jdbc</module>
|
||||
<module>presto-mysql</module>
|
||||
<module>presto-sqlserver</module>
|
||||
<module>presto-postgresql</module>
|
||||
<module>presto-client</module>
|
||||
<module>presto-parser</module>
|
||||
<module>presto-main</module>
|
||||
<module>presto-ml</module>
|
||||
<module>presto-benchmark</module>
|
||||
<module>presto-tests</module>
|
||||
<module>presto-jdbc</module>
|
||||
<module>presto-cli</module>
|
||||
<module>presto-benchmark-driver</module>
|
||||
<module>presto-server</module>
|
||||
<module>presto-docs</module>
|
||||
<module>presto-verifier</module>
|
||||
<module>presto-oracle</module>
|
||||
</modules>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-spi</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-orc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-hive</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-hive-cdh4</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>zip</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-example-http</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>zip</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-hive</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-tpch</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-base-jdbc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-mysql</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-sqlserver</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-raptor</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-cli</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-client</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-parser</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-parser</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-main</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-main</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-jdbc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-server</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-tests</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-benchmark</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto.hadoop</groupId>
|
||||
<artifactId>hadoop-apache1</artifactId>
|
||||
<version>0.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto.hadoop</groupId>
|
||||
<artifactId>hadoop-apache2</artifactId>
|
||||
<version>0.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto.hadoop</groupId>
|
||||
<artifactId>hadoop-cdh4</artifactId>
|
||||
<version>0.8</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto.hive</groupId>
|
||||
<artifactId>hive-apache</artifactId>
|
||||
<version>0.14</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.hive</groupId>
|
||||
<artifactId>hive-dwrf</artifactId>
|
||||
<version>0.8</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.iq80.snappy</groupId>
|
||||
<artifactId>snappy</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.facebook.presto.hadoop</groupId>
|
||||
<artifactId>hadoop-cdh4</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>it.unimi.dsi</groupId>
|
||||
<artifactId>fastutil</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.hive</groupId>
|
||||
<artifactId>hive-dwrf-shims</artifactId>
|
||||
<version>0.8</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<groupId>commons-logging</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>2.4.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log-manager</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>units</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>concurrent</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>configuration</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>discovery</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>node</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>bootstrap</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>event</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>http-server</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>jaxrs</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>jmx</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>trace-token</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>dbpool</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>jmx-http</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>http-client</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>stats</artifactId>
|
||||
<version>${dep.airlift.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>joni</artifactId>
|
||||
<version>2.1.5.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift.tpch</groupId>
|
||||
<artifactId>tpch</artifactId>
|
||||
<version>0.4</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.ow2.asm</groupId>
|
||||
<artifactId>asm-all</artifactId>
|
||||
<version>4.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<version>1.3.170</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.sonatype.aether</groupId>
|
||||
<artifactId>aether-api</artifactId>
|
||||
<version>1.13.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift.resolver</groupId>
|
||||
<artifactId>resolver</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>airline</artifactId>
|
||||
<version>0.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.iq80.snappy</groupId>
|
||||
<artifactId>snappy</artifactId>
|
||||
<version>0.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.openjdk.jol</groupId>
|
||||
<artifactId>jol-core</artifactId>
|
||||
<version>0.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jetbrains</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
<version>13.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>it.unimi.dsi</groupId>
|
||||
<artifactId>fastutil</artifactId>
|
||||
<version>6.5.9</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.thirdparty</groupId>
|
||||
<artifactId>libsvm</artifactId>
|
||||
<version>3.18.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>5.1.35</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>9.3-1102-jdbc41</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-runtime</artifactId>
|
||||
<version>${dep.antlr.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-annotations</artifactId>
|
||||
<version>${dep.antlr.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
<version>2.12</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jdbi</groupId>
|
||||
<artifactId>jdbi</artifactId>
|
||||
<version>2.55</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.thrift</groupId>
|
||||
<artifactId>libthrift</artifactId>
|
||||
<version>0.9.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sf.opencsv</groupId>
|
||||
<artifactId>opencsv</artifactId>
|
||||
<version>2.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<version>3.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
<version>3.7.0.Final</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift.discovery</groupId>
|
||||
<artifactId>discovery-server</artifactId>
|
||||
<version>1.24</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk</artifactId>
|
||||
<version>1.8.9.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing-mysql-server</artifactId>
|
||||
<version>0.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing-postgresql-server</artifactId>
|
||||
<version>0.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.10</artifactId>
|
||||
<version>0.8.1.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-jdk14</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.xerial.snappy</groupId>
|
||||
<artifactId>snappy-java</artifactId>
|
||||
<version>1.1.1.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>3.3.6</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>junit</artifactId>
|
||||
<groupId>junit</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.101tec</groupId>
|
||||
<artifactId>zkclient</artifactId>
|
||||
<version>0.4</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jgrapht</groupId>
|
||||
<artifactId>jgrapht-core</artifactId>
|
||||
<version>0.9.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-maven-plugin</artifactId>
|
||||
<version>${dep.antlr.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>antlr4</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<visitor>true</visitor>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>2.3</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.skife.maven</groupId>
|
||||
<artifactId>really-executable-jar-maven-plugin</artifactId>
|
||||
<version>1.0.5</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<version>1.2.1</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.tomdz.maven</groupId>
|
||||
<artifactId>sphinx-maven-plugin</artifactId>
|
||||
<version>1.0.3</version>
|
||||
</plugin>
|
||||
|
||||
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
|
||||
<plugin>
|
||||
<!--suppress MavenModelInspection -->
|
||||
<groupId>org.eclipse.m2e</groupId>
|
||||
<!--suppress MavenModelInspection -->
|
||||
<artifactId>lifecycle-mapping</artifactId>
|
||||
<!--suppress MavenModelInspection -->
|
||||
<version>1.0.0</version>
|
||||
<configuration>
|
||||
<lifecycleMappingMetadata>
|
||||
<pluginExecutions>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<versionRange>[2.5.1,)</versionRange>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
<goal>analyze-dep-mgt</goal>
|
||||
<goal>analyze-duplicate</goal>
|
||||
<goal>analyze-only</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<versionRange>[0.6.2.201302030002,)</versionRange>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>com.mycila</groupId>
|
||||
<artifactId>license-maven-plugin</artifactId>
|
||||
<versionRange>[2.3,)</versionRange>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>com.ning.maven.plugins</groupId>
|
||||
<artifactId>maven-duplicate-finder-plugin</artifactId>
|
||||
<versionRange>[1.0.4,)</versionRange>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<versionRange>[0,)</versionRange>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>io.takari.maven.plugins</groupId>
|
||||
<artifactId>presto-maven-plugin</artifactId>
|
||||
<versionRange>[0,)</versionRange>
|
||||
<goals>
|
||||
<goal>generate-service-descriptor</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>io.takari.maven.plugins</groupId>
|
||||
<artifactId>takari-lifecycle-plugin</artifactId>
|
||||
<versionRange>[0,)</versionRange>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
<goal>process-resources</goal>
|
||||
<goal>process-test-resources</goal>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.gaul</groupId>
|
||||
<artifactId>modernizer-maven-plugin</artifactId>
|
||||
<versionRange>[0,)</versionRange>
|
||||
<goals>
|
||||
<goal>modernizer</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore/>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
</pluginExecutions>
|
||||
</lifecycleMappingMetadata>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.gaul</groupId>
|
||||
<artifactId>modernizer-maven-plugin</artifactId>
|
||||
<version>1.2.2</version>
|
||||
<configuration>
|
||||
<javaVersion>1.8</javaVersion>
|
||||
<failOnViolations>false</failOnViolations>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>modernizer</id>
|
||||
<goals>
|
||||
<goal>modernizer</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<version>2.15</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<skip>${air.check.skip-checkstyle}</skip>
|
||||
<failOnViolation>${air.check.fail-checkstyle}</failOnViolation>
|
||||
<consoleOutput>true</consoleOutput>
|
||||
<includeTestSourceDirectory>true</includeTestSourceDirectory>
|
||||
<configLocation>${air.main.basedir}/src/checkstyle/checks.xml</configLocation>
|
||||
<excludes>**/com/facebook/presto/operator/PagesIndexOrdering.java</excludes>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
<dependencies>
|
||||
<!-- The version of checkstyle the plugin depends on doesn't support Java 8,
|
||||
so override it manually until a new version of the plugin is released.
|
||||
This is copied verbatim from the dependency declaration in the checkstyle
|
||||
plugin -->
|
||||
<dependency>
|
||||
<groupId>com.puppycrawl.tools</groupId>
|
||||
<artifactId>checkstyle</artifactId>
|
||||
<version>6.6</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>io.takari.maven.plugins</groupId>
|
||||
<artifactId>presto-maven-plugin</artifactId>
|
||||
<version>0.1.5</version>
|
||||
<extensions>true</extensions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>io.takari.maven.plugins</groupId>
|
||||
<artifactId>provisio-maven-plugin</artifactId>
|
||||
<version>0.1.11</version>
|
||||
<extensions>true</extensions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration combine.children="append">
|
||||
<fork>false</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<!-- run cli for development: mvn -am -pl presto-cli -P cli compile exec:java -->
|
||||
<profile>
|
||||
<id>cli</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>${cli.skip-execute}</skip>
|
||||
<executable>${java.home}/bin/java</executable>
|
||||
<mainClass>${cli.main-class}</mainClass>
|
||||
<arguments>
|
||||
<argument>--debug</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@ -0,0 +1,201 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>0.107</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-base-jdbc</artifactId>
|
||||
<name>presto-base-jdbc</name>
|
||||
<description>Presto - Base JDBC Connector</description>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>bootstrap</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>configuration</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>concurrent</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.inject</groupId>
|
||||
<artifactId>guice</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.validation</groupId>
|
||||
<artifactId>validation-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Presto SPI -->
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-spi</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>slice</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.inject</groupId>
|
||||
<artifactId>javax.inject</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-dbutils</groupId>
|
||||
<artifactId>commons-dbutils</artifactId>
|
||||
<version>1.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-main</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>units</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- for testing -->
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-tpch</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift.tpch</groupId>
|
||||
<artifactId>tpch</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-tests</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- integration tests take a very long time so only run them in the CI server -->
|
||||
<excludes>
|
||||
<exclude>**/TestJdbcDistributedQueries.java</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>ci</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override" />
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@ -0,0 +1,540 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcCacheConfig;
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcCacheSplit;
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcJavaBean;
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcResultCache;
|
||||
import com.facebook.presto.plugin.jdbc.subtable.JdbcSubTableConfig;
|
||||
import com.facebook.presto.plugin.jdbc.subtable.JdbcSubTableManager;
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ColumnMetadata;
|
||||
import com.facebook.presto.spi.ConnectorPartition;
|
||||
import com.facebook.presto.spi.ConnectorPartitionResult;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.ConnectorTableMetadata;
|
||||
import com.facebook.presto.spi.FixedSplitSource;
|
||||
import com.facebook.presto.spi.HostAddress;
|
||||
import com.facebook.presto.spi.PrestoException;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.TableNotFoundException;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.airlift.log.Logger;
|
||||
import io.airlift.slice.Slice;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.Driver;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND;
|
||||
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.BooleanType.BOOLEAN;
|
||||
import static com.facebook.presto.spi.type.DateType.DATE;
|
||||
import static com.facebook.presto.spi.type.DoubleType.DOUBLE;
|
||||
import static com.facebook.presto.spi.type.TimeType.TIME;
|
||||
import static com.facebook.presto.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
|
||||
import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP;
|
||||
import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
|
||||
import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static com.google.common.collect.Maps.fromProperties;
|
||||
import static java.util.Collections.nCopies;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public class BaseJdbcClient
|
||||
implements JdbcClient
|
||||
{
|
||||
private static final Logger log = Logger.get(BaseJdbcClient.class);
|
||||
public static final int TYPE_MYSQL = 1;
|
||||
public static final int TYPE_ORACLE = 2;
|
||||
public static final int TYPE_SQLSERVER = 3;
|
||||
|
||||
private static final Map<Type, String> SQL_TYPES = ImmutableMap.<Type, String>builder()
|
||||
.put(BOOLEAN, "boolean")
|
||||
.put(BIGINT, "bigint")
|
||||
.put(DOUBLE, "double precision")
|
||||
.put(VARCHAR, "varchar")
|
||||
.put(VARBINARY, "varbinary")
|
||||
.put(DATE, "date")
|
||||
.put(TIME, "time")
|
||||
.put(TIME_WITH_TIME_ZONE, "time with timezone")
|
||||
.put(TIMESTAMP, "timestamp")
|
||||
.put(TIMESTAMP_WITH_TIME_ZONE, "timestamp with timezone")
|
||||
.build();
|
||||
|
||||
protected final String connectorId;
|
||||
protected final Driver driver;
|
||||
protected final String connectionUrl;
|
||||
protected final Properties connectionProperties;
|
||||
protected final String identifierQuote;
|
||||
protected int dbType;
|
||||
|
||||
protected final boolean jdbcSubTableEnable;
|
||||
private JdbcSubTableManager subTableManager;
|
||||
|
||||
protected final boolean cacheEnable;
|
||||
private JdbcResultCache jdbcResultCache;
|
||||
|
||||
public BaseJdbcClient(JdbcConnectorId connectorId,
|
||||
BaseJdbcConfig config,
|
||||
String identifierQuote,
|
||||
Driver driver,
|
||||
JdbcSubTableConfig subTableConfig,
|
||||
JdbcCacheConfig cacheConfig)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
|
||||
this.identifierQuote = checkNotNull(identifierQuote, "identifierQuote is null");
|
||||
this.driver = checkNotNull(driver, "driver is null");
|
||||
|
||||
checkNotNull(config, "config is null");
|
||||
connectionUrl = config.getConnectionUrl();
|
||||
|
||||
connectionProperties = new Properties();
|
||||
if (config.getConnectionUser() != null) {
|
||||
connectionProperties.setProperty("user", config.getConnectionUser());
|
||||
}
|
||||
if (config.getConnectionPassword() != null) {
|
||||
connectionProperties.setProperty("password", config.getConnectionPassword());
|
||||
}
|
||||
|
||||
// sub table
|
||||
jdbcSubTableEnable = subTableConfig.getJdbcSubTableEnable();
|
||||
if (jdbcSubTableEnable) {
|
||||
this.subTableManager = new JdbcSubTableManager(this.connectorId, identifierQuote,
|
||||
driver, connectionUrl, connectionProperties, subTableConfig);
|
||||
}
|
||||
// jdbc cache
|
||||
cacheEnable = cacheConfig.getJdbcCacheEnable();
|
||||
if (cacheEnable) {
|
||||
this.jdbcResultCache = new JdbcResultCache(identifierQuote, driver, connectionProperties, cacheConfig);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getSchemaNames()
|
||||
{
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties);
|
||||
ResultSet resultSet = connection.getMetaData().getSchemas()) {
|
||||
ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder();
|
||||
while (resultSet.next()) {
|
||||
String schemaName = resultSet.getString("TABLE_SCHEM").toLowerCase(ENGLISH);
|
||||
// skip internal schemas
|
||||
if (!schemaName.equals("information_schema")) {
|
||||
schemaNames.add(schemaName);
|
||||
}
|
||||
}
|
||||
return schemaNames.build();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SchemaTableName> getTableNames(@Nullable String schema)
|
||||
{
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties)) {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
if (metadata.storesUpperCaseIdentifiers() && (schema != null)) {
|
||||
schema = schema.toUpperCase(ENGLISH);
|
||||
}
|
||||
try (ResultSet resultSet = getTables(connection, schema, null)) {
|
||||
ImmutableList.Builder<SchemaTableName> list = ImmutableList.builder();
|
||||
while (resultSet.next()) {
|
||||
list.add(getSchemaTableName(resultSet));
|
||||
}
|
||||
return list.build();
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public JdbcTableHandle getTableHandle(SchemaTableName schemaTableName)
|
||||
{
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties)) {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
String jdbcSchemaName = schemaTableName.getSchemaName();
|
||||
String jdbcTableName = schemaTableName.getTableName();
|
||||
if (metadata.storesUpperCaseIdentifiers()) {
|
||||
jdbcSchemaName = jdbcSchemaName.toUpperCase(ENGLISH);
|
||||
jdbcTableName = jdbcTableName.toUpperCase(ENGLISH);
|
||||
}
|
||||
try (ResultSet resultSet = getTables(connection, jdbcSchemaName, jdbcTableName)) {
|
||||
List<JdbcTableHandle> tableHandles = new ArrayList<>();
|
||||
while (resultSet.next()) {
|
||||
tableHandles.add(new JdbcTableHandle(
|
||||
connectorId,
|
||||
schemaTableName,
|
||||
resultSet.getString("TABLE_CAT"),
|
||||
resultSet.getString("TABLE_SCHEM"),
|
||||
resultSet.getString("TABLE_NAME")));
|
||||
}
|
||||
if (tableHandles.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (tableHandles.size() > 1) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "Multiple tables matched: " + schemaTableName);
|
||||
}
|
||||
return getOnlyElement(tableHandles);
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<JdbcColumnHandle> getColumns(JdbcTableHandle tableHandle)
|
||||
{
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties)) {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
try (ResultSet resultSet = metadata.getColumns(tableHandle.getCatalogName(), tableHandle.getSchemaName(), tableHandle.getTableName(), null)) {
|
||||
List<JdbcColumnHandle> columns = new ArrayList<>();
|
||||
boolean found = false;
|
||||
while (resultSet.next()) {
|
||||
found = true;
|
||||
Type columnType = toPrestoType(resultSet.getInt("DATA_TYPE"));
|
||||
// skip unsupported column types
|
||||
if (columnType != null) {
|
||||
String columnName = resultSet.getString("COLUMN_NAME");
|
||||
columns.add(new JdbcColumnHandle(connectorId, columnName, columnType));
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
throw new TableNotFoundException(tableHandle.getSchemaTableName());
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "Table has no supported column types: " + tableHandle.getSchemaTableName());
|
||||
}
|
||||
return ImmutableList.copyOf(columns);
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPartitionResult getPartitions(JdbcTableHandle jdbcTableHandle, TupleDomain<ColumnHandle> tupleDomain)
|
||||
{
|
||||
// currently we don't support partitions
|
||||
return new ConnectorPartitionResult(
|
||||
ImmutableList.<ConnectorPartition>of(new JdbcPartition(jdbcTableHandle, tupleDomain)),
|
||||
tupleDomain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorSplitSource getPartitionSplits(JdbcPartition jdbcPartition)
|
||||
{
|
||||
if (jdbcSubTableEnable) {
|
||||
return subTableManager.getTableSplits(jdbcPartition);
|
||||
}
|
||||
JdbcTableHandle jdbcTableHandle = jdbcPartition.getJdbcTableHandle();
|
||||
List<HostAddress> of = ImmutableList.of();
|
||||
JdbcSplit jdbcSplit = new JdbcSplit(
|
||||
connectorId,
|
||||
jdbcTableHandle.getCatalogName(),
|
||||
jdbcTableHandle.getSchemaName(),
|
||||
jdbcTableHandle.getTableName(),
|
||||
connectionUrl,
|
||||
fromProperties(connectionProperties),
|
||||
jdbcPartition.getTupleDomain(),
|
||||
"", of, true, "", "", "", "", System.nanoTime(), 1, false, "");
|
||||
return new FixedSplitSource(connectorId, ImmutableList.of(jdbcSplit));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Connection getConnection(JdbcSplit split)
|
||||
throws SQLException
|
||||
{
|
||||
Connection connection = driver.connect(split.getConnectionUrl(), toProperties(split.getConnectionProperties()));
|
||||
try {
|
||||
connection.setReadOnly(true);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
connection.close();
|
||||
throw e;
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildSql(JdbcSplit split, List<JdbcColumnHandle> columnHandles)
|
||||
{
|
||||
return new QueryBuilder(identifierQuote).buildSql(
|
||||
dbType,
|
||||
split.getCatalogName(),
|
||||
split.getSchemaName(),
|
||||
split.getTableName(),
|
||||
columnHandles,
|
||||
split.getTupleDomain());
|
||||
}
|
||||
|
||||
@Override
|
||||
public JdbcOutputTableHandle beginCreateTable(ConnectorTableMetadata tableMetadata)
|
||||
{
|
||||
SchemaTableName schemaTableName = tableMetadata.getTable();
|
||||
String schema = schemaTableName.getSchemaName();
|
||||
String table = schemaTableName.getTableName();
|
||||
|
||||
if (!getSchemaNames().contains(schema)) {
|
||||
throw new PrestoException(NOT_FOUND, "Schema not found: " + schema);
|
||||
}
|
||||
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties)) {
|
||||
boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers();
|
||||
if (uppercase) {
|
||||
schema = schema.toUpperCase(ENGLISH);
|
||||
table = table.toUpperCase(ENGLISH);
|
||||
}
|
||||
String catalog = connection.getCatalog();
|
||||
|
||||
String temporaryName = "tmp_presto_" + UUID.randomUUID().toString().replace("-", "");
|
||||
StringBuilder sql = new StringBuilder()
|
||||
.append("CREATE TABLE ")
|
||||
.append(quoted(catalog, schema, temporaryName))
|
||||
.append(" (");
|
||||
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
|
||||
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
|
||||
ImmutableList.Builder<String> columnList = ImmutableList.builder();
|
||||
for (ColumnMetadata column : tableMetadata.getColumns()) {
|
||||
String columnName = column.getName();
|
||||
if (uppercase) {
|
||||
columnName = columnName.toUpperCase(ENGLISH);
|
||||
}
|
||||
columnNames.add(columnName);
|
||||
columnTypes.add(column.getType());
|
||||
columnList.add(new StringBuilder()
|
||||
.append(quoted(columnName))
|
||||
.append(" ")
|
||||
.append(toSqlType(column.getType()))
|
||||
.toString());
|
||||
}
|
||||
Joiner.on(", ").appendTo(sql, columnList.build());
|
||||
sql.append(")");
|
||||
|
||||
execute(connection, sql.toString());
|
||||
|
||||
return new JdbcOutputTableHandle(
|
||||
connectorId,
|
||||
catalog,
|
||||
schema,
|
||||
table,
|
||||
columnNames.build(),
|
||||
columnTypes.build(),
|
||||
tableMetadata.getOwner(),
|
||||
temporaryName,
|
||||
connectionUrl,
|
||||
fromProperties(connectionProperties));
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commitCreateTable(JdbcOutputTableHandle handle, Collection<Slice> fragments)
|
||||
{
|
||||
StringBuilder sql = new StringBuilder()
|
||||
.append("ALTER TABLE ")
|
||||
.append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()))
|
||||
.append(" RENAME TO ")
|
||||
.append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()));
|
||||
|
||||
try (Connection connection = getConnection(handle)) {
|
||||
execute(connection, sql.toString());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropTable(JdbcTableHandle handle)
|
||||
{
|
||||
StringBuilder sql = new StringBuilder()
|
||||
.append("DROP TABLE ")
|
||||
.append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()));
|
||||
|
||||
try (Connection connection = driver.connect(connectionUrl, connectionProperties)) {
|
||||
execute(connection, sql.toString());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildInsertSql(JdbcOutputTableHandle handle)
|
||||
{
|
||||
String vars = Joiner.on(',').join(nCopies(handle.getColumnNames().size(), "?"));
|
||||
return new StringBuilder()
|
||||
.append("INSERT INTO ")
|
||||
.append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()))
|
||||
.append(" VALUES (").append(vars).append(")")
|
||||
.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Connection getConnection(JdbcOutputTableHandle handle)
|
||||
throws SQLException
|
||||
{
|
||||
return driver.connect(handle.getConnectionUrl(), toProperties(handle.getConnectionProperties()));
|
||||
}
|
||||
|
||||
protected ResultSet getTables(Connection connection, String schemaName, String tableName)
|
||||
throws SQLException
|
||||
{
|
||||
return connection.getMetaData().getTables(connection.getCatalog(), schemaName, tableName, new String[] {"TABLE", "VIEW"});
|
||||
}
|
||||
|
||||
protected SchemaTableName getSchemaTableName(ResultSet resultSet)
|
||||
throws SQLException
|
||||
{
|
||||
return new SchemaTableName(
|
||||
resultSet.getString("TABLE_SCHEM").toLowerCase(ENGLISH),
|
||||
resultSet.getString("TABLE_NAME").toLowerCase(ENGLISH));
|
||||
}
|
||||
|
||||
protected void execute(Connection connection, String query)
|
||||
throws SQLException
|
||||
{
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
log.debug("Execute: %s", query);
|
||||
statement.execute(query);
|
||||
}
|
||||
}
|
||||
|
||||
protected Type toPrestoType(int jdbcType)
|
||||
{
|
||||
switch (jdbcType) {
|
||||
case Types.BIT:
|
||||
case Types.BOOLEAN:
|
||||
return BOOLEAN;
|
||||
case Types.TINYINT:
|
||||
case Types.SMALLINT:
|
||||
case Types.INTEGER:
|
||||
case Types.BIGINT:
|
||||
return BIGINT;
|
||||
case Types.FLOAT:
|
||||
case Types.REAL:
|
||||
case Types.DOUBLE:
|
||||
case Types.NUMERIC:
|
||||
case Types.DECIMAL:
|
||||
return DOUBLE;
|
||||
case Types.CHAR:
|
||||
case Types.NCHAR:
|
||||
case Types.VARCHAR:
|
||||
case Types.NVARCHAR:
|
||||
case Types.LONGVARCHAR:
|
||||
case Types.LONGNVARCHAR:
|
||||
return VARCHAR;
|
||||
case Types.BINARY:
|
||||
case Types.VARBINARY:
|
||||
case Types.LONGVARBINARY:
|
||||
return VARBINARY;
|
||||
case Types.DATE:
|
||||
return DATE;
|
||||
case Types.TIME:
|
||||
return TIME;
|
||||
case Types.TIMESTAMP:
|
||||
return TIMESTAMP;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
protected String toSqlType(Type type)
|
||||
{
|
||||
String sqlType = SQL_TYPES.get(type);
|
||||
if (sqlType != null) {
|
||||
return sqlType;
|
||||
}
|
||||
throw new PrestoException(NOT_SUPPORTED, "Unsuported column type: " + type.getTypeSignature());
|
||||
}
|
||||
|
||||
protected String quoted(String name)
|
||||
{
|
||||
name = name.replace(identifierQuote, identifierQuote + identifierQuote);
|
||||
return identifierQuote + name + identifierQuote;
|
||||
}
|
||||
|
||||
protected String quoted(String catalog, String schema, String table)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (!isNullOrEmpty(catalog)) {
|
||||
sb.append(quoted(catalog)).append(".");
|
||||
}
|
||||
if (!isNullOrEmpty(schema)) {
|
||||
sb.append(quoted(schema)).append(".");
|
||||
}
|
||||
sb.append(quoted(table));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static Properties toProperties(Map<String, String> map)
|
||||
{
|
||||
Properties properties = new Properties();
|
||||
for (Map.Entry<String, String> entry : map.entrySet()) {
|
||||
properties.setProperty(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
public synchronized List<JdbcJavaBean> getTableDataSet(JdbcCacheSplit key)
|
||||
{
|
||||
return jdbcResultCache.getResult(key);
|
||||
}
|
||||
|
||||
public boolean isCacheTable(String tableName)
|
||||
{
|
||||
return cacheEnable && jdbcResultCache != null && jdbcResultCache.isCacheTable(tableName);
|
||||
}
|
||||
|
||||
public void commitPdboLogs(JdbcSplit split, long rowCount)
|
||||
{
|
||||
this.subTableManager.commitPdboLogs(split, rowCount);
|
||||
}
|
||||
|
||||
public int getDb_type()
|
||||
{
|
||||
return dbType;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import io.airlift.configuration.Config;
|
||||
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
||||
public class BaseJdbcConfig
|
||||
{
|
||||
private String connectionUrl;
|
||||
private String connectionUser;
|
||||
private String connectionPassword;
|
||||
|
||||
@NotNull
|
||||
public String getConnectionUrl()
|
||||
{
|
||||
return connectionUrl;
|
||||
}
|
||||
|
||||
@Config("connection-url")
|
||||
public BaseJdbcConfig setConnectionUrl(String connectionUrl)
|
||||
{
|
||||
this.connectionUrl = connectionUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getConnectionUser()
|
||||
{
|
||||
return connectionUser;
|
||||
}
|
||||
|
||||
@Config("connection-user")
|
||||
public BaseJdbcConfig setConnectionUser(String connectionUser)
|
||||
{
|
||||
this.connectionUser = connectionUser;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getConnectionPassword()
|
||||
{
|
||||
return connectionPassword;
|
||||
}
|
||||
|
||||
@Config("connection-password")
|
||||
public BaseJdbcConfig setConnectionPassword(String connectionPassword)
|
||||
{
|
||||
this.connectionPassword = connectionPassword;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorPartitionResult;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.ConnectorTableMetadata;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import io.airlift.slice.Slice;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public interface JdbcClient
|
||||
{
|
||||
Set<String> getSchemaNames();
|
||||
|
||||
List<SchemaTableName> getTableNames(@Nullable String schema);
|
||||
|
||||
@Nullable
|
||||
JdbcTableHandle getTableHandle(SchemaTableName schemaTableName);
|
||||
|
||||
List<JdbcColumnHandle> getColumns(JdbcTableHandle tableHandle);
|
||||
|
||||
ConnectorPartitionResult getPartitions(JdbcTableHandle jdbcTableHandle, TupleDomain<ColumnHandle> tupleDomain);
|
||||
|
||||
ConnectorSplitSource getPartitionSplits(JdbcPartition jdbcPartition);
|
||||
|
||||
Connection getConnection(JdbcSplit split)
|
||||
throws SQLException;
|
||||
|
||||
String buildSql(JdbcSplit split, List<JdbcColumnHandle> columnHandles);
|
||||
|
||||
JdbcOutputTableHandle beginCreateTable(ConnectorTableMetadata tableMetadata);
|
||||
|
||||
void commitCreateTable(JdbcOutputTableHandle handle, Collection<Slice> fragments);
|
||||
|
||||
void dropTable(JdbcTableHandle jdbcTableHandle);
|
||||
|
||||
String buildInsertSql(JdbcOutputTableHandle handle);
|
||||
|
||||
Connection getConnection(JdbcOutputTableHandle handle)
|
||||
throws SQLException;
|
||||
}
|
||||
@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnMetadata;
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.google.common.base.MoreObjects.toStringHelper;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public final class JdbcColumnHandle
|
||||
implements ColumnHandle
|
||||
{
|
||||
private final String connectorId;
|
||||
private final String columnName;
|
||||
private final Type columnType;
|
||||
|
||||
@JsonCreator
|
||||
public JdbcColumnHandle(
|
||||
@JsonProperty("connectorId") String connectorId,
|
||||
@JsonProperty("columnName") String columnName,
|
||||
@JsonProperty("columnType") Type columnType)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null");
|
||||
this.columnName = checkNotNull(columnName, "columnName is null");
|
||||
this.columnType = checkNotNull(columnType, "columnType is null");
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getColumnName()
|
||||
{
|
||||
return columnName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public Type getColumnType()
|
||||
{
|
||||
return columnType;
|
||||
}
|
||||
|
||||
public ColumnMetadata getColumnMetadata()
|
||||
{
|
||||
return new ColumnMetadata(columnName, columnType, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if ((obj == null) || (getClass() != obj.getClass())) {
|
||||
return false;
|
||||
}
|
||||
JdbcColumnHandle o = (JdbcColumnHandle) obj;
|
||||
return Objects.equals(this.connectorId, o.connectorId) &&
|
||||
Objects.equals(this.columnName, o.columnName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(connectorId, columnName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return toStringHelper(this)
|
||||
.add("connectorId", connectorId)
|
||||
.add("columnName", columnName)
|
||||
.add("columnType", columnType)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.Connector;
|
||||
import com.facebook.presto.spi.ConnectorHandleResolver;
|
||||
import com.facebook.presto.spi.ConnectorMetadata;
|
||||
import com.facebook.presto.spi.ConnectorRecordSetProvider;
|
||||
import com.facebook.presto.spi.ConnectorRecordSinkProvider;
|
||||
import com.facebook.presto.spi.ConnectorSplitManager;
|
||||
import io.airlift.bootstrap.LifeCycleManager;
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcConnector
|
||||
implements Connector
|
||||
{
|
||||
private static final Logger log = Logger.get(JdbcConnector.class);
|
||||
|
||||
private final LifeCycleManager lifeCycleManager;
|
||||
private final JdbcMetadata jdbcMetadata;
|
||||
private final JdbcSplitManager jdbcSplitManager;
|
||||
private final JdbcRecordSetProvider jdbcRecordSetProvider;
|
||||
private final JdbcHandleResolver jdbcHandleResolver;
|
||||
private final JdbcRecordSinkProvider jdbcRecordSinkProvider;
|
||||
|
||||
@Inject
|
||||
public JdbcConnector(
|
||||
LifeCycleManager lifeCycleManager,
|
||||
JdbcMetadata jdbcMetadata,
|
||||
JdbcSplitManager jdbcSplitManager,
|
||||
JdbcRecordSetProvider jdbcRecordSetProvider,
|
||||
JdbcHandleResolver jdbcHandleResolver,
|
||||
JdbcRecordSinkProvider jdbcRecordSinkProvider)
|
||||
{
|
||||
this.lifeCycleManager = checkNotNull(lifeCycleManager, "lifeCycleManager is null");
|
||||
this.jdbcMetadata = checkNotNull(jdbcMetadata, "jdbcMetadata is null");
|
||||
this.jdbcSplitManager = checkNotNull(jdbcSplitManager, "jdbcSplitManager is null");
|
||||
this.jdbcRecordSetProvider = checkNotNull(jdbcRecordSetProvider, "jdbcRecordSetProvider is null");
|
||||
this.jdbcHandleResolver = checkNotNull(jdbcHandleResolver, "jdbcHandleResolver is null");
|
||||
this.jdbcRecordSinkProvider = checkNotNull(jdbcRecordSinkProvider, "jdbcRecordSinkProvider is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorMetadata getMetadata()
|
||||
{
|
||||
return jdbcMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorSplitManager getSplitManager()
|
||||
{
|
||||
return jdbcSplitManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorRecordSetProvider getRecordSetProvider()
|
||||
{
|
||||
return jdbcRecordSetProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorHandleResolver getHandleResolver()
|
||||
{
|
||||
return jdbcHandleResolver;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorRecordSinkProvider getRecordSinkProvider()
|
||||
{
|
||||
return jdbcRecordSinkProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void shutdown()
|
||||
{
|
||||
try {
|
||||
lifeCycleManager.stop();
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.error(e, "Error shutting down connector");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.Connector;
|
||||
import com.facebook.presto.spi.ConnectorFactory;
|
||||
import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.inject.Injector;
|
||||
import com.google.inject.Module;
|
||||
import io.airlift.bootstrap.Bootstrap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
|
||||
public class JdbcConnectorFactory
|
||||
implements ConnectorFactory
|
||||
{
|
||||
private final String name;
|
||||
private final Module module;
|
||||
private final Map<String, String> optionalConfig;
|
||||
private final ClassLoader classLoader;
|
||||
|
||||
public JdbcConnectorFactory(String name, Module module, Map<String, String> optionalConfig, ClassLoader classLoader)
|
||||
{
|
||||
checkArgument(!isNullOrEmpty(name), "name is null or empty");
|
||||
this.name = name;
|
||||
this.module = checkNotNull(module, "module is null");
|
||||
this.optionalConfig = ImmutableMap.copyOf(checkNotNull(optionalConfig, "optionalConfig is null"));
|
||||
this.classLoader = checkNotNull(classLoader, "classLoader is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Connector create(String connectorId, Map<String, String> requiredConfig)
|
||||
{
|
||||
checkNotNull(requiredConfig, "requiredConfig is null");
|
||||
checkNotNull(optionalConfig, "optionalConfig is null");
|
||||
|
||||
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) {
|
||||
Bootstrap app = new Bootstrap(new JdbcModule(connectorId), module);
|
||||
|
||||
Injector injector = app
|
||||
.strictConfig()
|
||||
.doNotInitializeLogging()
|
||||
.setRequiredConfigurationProperties(requiredConfig)
|
||||
.setOptionalConfigurationProperties(optionalConfig)
|
||||
.initialize();
|
||||
|
||||
return injector.getInstance(JdbcConnector.class);
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public final class JdbcConnectorId
|
||||
{
|
||||
private final String id;
|
||||
|
||||
public JdbcConnectorId(String id)
|
||||
{
|
||||
this.id = checkNotNull(id, "id is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if ((obj == null) || (getClass() != obj.getClass())) {
|
||||
return false;
|
||||
}
|
||||
JdbcConnectorId other = (JdbcConnectorId) obj;
|
||||
return Objects.equals(this.id, other.id);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorHandleResolver;
|
||||
import com.facebook.presto.spi.ConnectorOutputTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorSplit;
|
||||
import com.facebook.presto.spi.ConnectorTableHandle;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcHandleResolver
|
||||
implements ConnectorHandleResolver
|
||||
{
|
||||
private final String connectorId;
|
||||
|
||||
@Inject
|
||||
public JdbcHandleResolver(JdbcConnectorId clientId)
|
||||
{
|
||||
this.connectorId = checkNotNull(clientId, "clientId is null").toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canHandle(ConnectorTableHandle tableHandle)
|
||||
{
|
||||
return tableHandle instanceof JdbcTableHandle && ((JdbcTableHandle) tableHandle).getConnectorId().equals(connectorId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canHandle(ColumnHandle columnHandle)
|
||||
{
|
||||
return columnHandle instanceof JdbcColumnHandle && ((JdbcColumnHandle) columnHandle).getConnectorId().equals(connectorId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canHandle(ConnectorSplit split)
|
||||
{
|
||||
return split instanceof JdbcSplit && ((JdbcSplit) split).getConnectorId().equals(connectorId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canHandle(ConnectorOutputTableHandle tableHandle)
|
||||
{
|
||||
return (tableHandle instanceof JdbcOutputTableHandle) && ((JdbcOutputTableHandle) tableHandle).getConnectorId().equals(connectorId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends ConnectorTableHandle> getTableHandleClass()
|
||||
{
|
||||
return JdbcTableHandle.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends ColumnHandle> getColumnHandleClass()
|
||||
{
|
||||
return JdbcColumnHandle.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends ConnectorSplit> getSplitClass()
|
||||
{
|
||||
return JdbcSplit.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
|
||||
{
|
||||
return JdbcOutputTableHandle.class;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,209 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnMetadata;
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorInsertTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorMetadata;
|
||||
import com.facebook.presto.spi.ConnectorOutputTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorSession;
|
||||
import com.facebook.presto.spi.ConnectorTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorTableMetadata;
|
||||
import com.facebook.presto.spi.InsertOption;
|
||||
import com.facebook.presto.spi.PrestoException;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.SchemaTablePrefix;
|
||||
import com.facebook.presto.spi.TableNotFoundException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.airlift.slice.Slice;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.Types.checkType;
|
||||
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
|
||||
import static com.facebook.presto.spi.StandardErrorCode.PERMISSION_DENIED;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcMetadata
|
||||
implements ConnectorMetadata
|
||||
{
|
||||
private final JdbcClient jdbcClient;
|
||||
private final boolean allowDropTable;
|
||||
|
||||
@Inject
|
||||
public JdbcMetadata(JdbcConnectorId connectorId, JdbcClient jdbcClient, JdbcMetadataConfig config)
|
||||
{
|
||||
this.jdbcClient = checkNotNull(jdbcClient, "client is null");
|
||||
|
||||
checkNotNull(config, "config is null");
|
||||
allowDropTable = config.isAllowDropTable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listSchemaNames(ConnectorSession session)
|
||||
{
|
||||
return ImmutableList.copyOf(jdbcClient.getSchemaNames());
|
||||
}
|
||||
|
||||
@Override
|
||||
public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
|
||||
{
|
||||
return jdbcClient.getTableHandle(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorTableMetadata getTableMetadata(ConnectorTableHandle table)
|
||||
{
|
||||
JdbcTableHandle handle = checkType(table, JdbcTableHandle.class, "tableHandle");
|
||||
|
||||
ImmutableList.Builder<ColumnMetadata> columnMetadata = ImmutableList.builder();
|
||||
for (JdbcColumnHandle column : jdbcClient.getColumns(handle)) {
|
||||
columnMetadata.add(column.getColumnMetadata());
|
||||
}
|
||||
return new ConnectorTableMetadata(handle.getSchemaTableName(), columnMetadata.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull)
|
||||
{
|
||||
return jdbcClient.getTableNames(schemaNameOrNull);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnHandle getSampleWeightColumnHandle(ConnectorTableHandle tableHandle)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ColumnHandle> getColumnHandles(ConnectorTableHandle tableHandle)
|
||||
{
|
||||
JdbcTableHandle jdbcTableHandle = checkType(tableHandle, JdbcTableHandle.class, "tableHandle");
|
||||
|
||||
ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
|
||||
for (JdbcColumnHandle column : jdbcClient.getColumns(jdbcTableHandle)) {
|
||||
columnHandles.put(column.getColumnMetadata().getName(), column);
|
||||
}
|
||||
return columnHandles.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
|
||||
{
|
||||
ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
|
||||
for (SchemaTableName tableName : listTables(session, prefix.getSchemaName())) {
|
||||
try {
|
||||
JdbcTableHandle tableHandle = jdbcClient.getTableHandle(tableName);
|
||||
if (tableHandle == null) {
|
||||
continue;
|
||||
}
|
||||
columns.put(tableName, getTableMetadata(tableHandle).getColumns());
|
||||
}
|
||||
catch (TableNotFoundException e) {
|
||||
// table disappeared during listing operation
|
||||
}
|
||||
}
|
||||
return columns.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnMetadata getColumnMetadata(ConnectorTableHandle tableHandle, ColumnHandle columnHandle)
|
||||
{
|
||||
checkType(tableHandle, JdbcTableHandle.class, "tableHandle");
|
||||
return checkType(columnHandle, JdbcColumnHandle.class, "columnHandle").getColumnMetadata();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canCreateSampledTables(ConnectorSession session)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
|
||||
{
|
||||
throw new PrestoException(NOT_SUPPORTED, "This connector does not support creating tables");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropTable(ConnectorTableHandle tableHandle)
|
||||
{
|
||||
if (!allowDropTable) {
|
||||
throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this catalog");
|
||||
}
|
||||
JdbcTableHandle handle = checkType(tableHandle, JdbcTableHandle.class, "tableHandle");
|
||||
jdbcClient.dropTable(handle);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
|
||||
{
|
||||
return jdbcClient.beginCreateTable(tableMetadata);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commitCreateTable(ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments)
|
||||
{
|
||||
JdbcOutputTableHandle handle = checkType(tableHandle, JdbcOutputTableHandle.class, "tableHandle");
|
||||
jdbcClient.commitCreateTable(handle, fragments);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void renameTable(ConnectorTableHandle tableHandle, SchemaTableName newTableName)
|
||||
{
|
||||
throw new PrestoException(NOT_SUPPORTED, "This connector does not support renaming tables");
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, InsertOption insertOption)
|
||||
{
|
||||
throw new PrestoException(NOT_SUPPORTED, "This connector does not support inserts");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commitInsert(ConnectorInsertTableHandle insertHandle, Collection<Slice> fragments)
|
||||
{
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createView(ConnectorSession session, SchemaTableName viewName, String viewData, boolean replace)
|
||||
{
|
||||
throw new PrestoException(NOT_SUPPORTED, "This connector does not support creating views");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropView(ConnectorSession session, SchemaTableName viewName)
|
||||
{
|
||||
throw new PrestoException(NOT_SUPPORTED, "This connector does not support dropping views");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SchemaTableName> listViews(ConnectorSession session, String schemaNameOrNull)
|
||||
{
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<SchemaTableName, String> getViews(ConnectorSession session, SchemaTablePrefix prefix)
|
||||
{
|
||||
return ImmutableMap.of();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import io.airlift.configuration.Config;
|
||||
import io.airlift.configuration.ConfigDescription;
|
||||
|
||||
public class JdbcMetadataConfig
|
||||
{
|
||||
private boolean allowDropTable;
|
||||
|
||||
public boolean isAllowDropTable()
|
||||
{
|
||||
return allowDropTable;
|
||||
}
|
||||
|
||||
@Config("allow-drop-table")
|
||||
@ConfigDescription("Allow connector to drop tables")
|
||||
public JdbcMetadataConfig setAllowDropTable(boolean allowDropTable)
|
||||
{
|
||||
this.allowDropTable = allowDropTable;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.google.inject.Binder;
|
||||
import com.google.inject.Module;
|
||||
import com.google.inject.Scopes;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static io.airlift.configuration.ConfigBinder.configBinder;
|
||||
|
||||
public class JdbcModule
|
||||
implements Module
|
||||
{
|
||||
private final String connectorId;
|
||||
|
||||
public JdbcModule(String connectorId)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connector id is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
binder.bind(JdbcConnectorId.class).toInstance(new JdbcConnectorId(connectorId));
|
||||
binder.bind(JdbcMetadata.class).in(Scopes.SINGLETON);
|
||||
binder.bind(JdbcSplitManager.class).in(Scopes.SINGLETON);
|
||||
binder.bind(JdbcRecordSetProvider.class).in(Scopes.SINGLETON);
|
||||
binder.bind(JdbcHandleResolver.class).in(Scopes.SINGLETON);
|
||||
binder.bind(JdbcRecordSinkProvider.class).in(Scopes.SINGLETON);
|
||||
binder.bind(JdbcConnector.class).in(Scopes.SINGLETON);
|
||||
configBinder(binder).bindConfig(JdbcMetadataConfig.class);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ConnectorOutputTableHandle;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static java.lang.String.format;
|
||||
|
||||
public class JdbcOutputTableHandle
|
||||
implements ConnectorOutputTableHandle
|
||||
{
|
||||
private final String connectorId;
|
||||
private final String catalogName;
|
||||
private final String schemaName;
|
||||
private final String tableName;
|
||||
private final List<String> columnNames;
|
||||
private final List<Type> columnTypes;
|
||||
private final String tableOwner;
|
||||
private final String temporaryTableName;
|
||||
private final String connectionUrl;
|
||||
private final Map<String, String> connectionProperties;
|
||||
|
||||
@JsonCreator
|
||||
public JdbcOutputTableHandle(
|
||||
@JsonProperty("connectorId") String connectorId,
|
||||
@JsonProperty("catalogName") @Nullable String catalogName,
|
||||
@JsonProperty("schemaName") @Nullable String schemaName,
|
||||
@JsonProperty("tableName") String tableName,
|
||||
@JsonProperty("columnNames") List<String> columnNames,
|
||||
@JsonProperty("columnTypes") List<Type> columnTypes,
|
||||
@JsonProperty("tableOwner") String tableOwner,
|
||||
@JsonProperty("temporaryTableName") String temporaryTableName,
|
||||
@JsonProperty("connectionUrl") String connectionUrl,
|
||||
@JsonProperty("connectionProperties") Map<String, String> connectionProperties)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null");
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.tableName = checkNotNull(tableName, "tableName is null");
|
||||
this.tableOwner = checkNotNull(tableOwner, "tableOwner is null");
|
||||
this.temporaryTableName = checkNotNull(temporaryTableName, "temporaryTableName is null");
|
||||
this.connectionUrl = checkNotNull(connectionUrl, "connectionUrl is null");
|
||||
this.connectionProperties = ImmutableMap.copyOf(checkNotNull(connectionProperties, "connectionProperties is null"));
|
||||
|
||||
checkNotNull(columnNames, "columnNames is null");
|
||||
checkNotNull(columnTypes, "columnTypes is null");
|
||||
checkArgument(columnNames.size() == columnTypes.size(), "columnNames and columnTypes sizes don't match");
|
||||
this.columnNames = ImmutableList.copyOf(columnNames);
|
||||
this.columnTypes = ImmutableList.copyOf(columnTypes);
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getCatalogName()
|
||||
{
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getSchemaName()
|
||||
{
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getTableName()
|
||||
{
|
||||
return tableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public List<String> getColumnNames()
|
||||
{
|
||||
return columnNames;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public List<Type> getColumnTypes()
|
||||
{
|
||||
return columnTypes;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getTableOwner()
|
||||
{
|
||||
return tableOwner;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getTemporaryTableName()
|
||||
{
|
||||
return temporaryTableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectionUrl()
|
||||
{
|
||||
return connectionUrl;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public Map<String, String> getConnectionProperties()
|
||||
{
|
||||
return connectionProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return format("jdbc:%s.%s.%s", catalogName, schemaName, tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(
|
||||
connectorId,
|
||||
catalogName,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnNames,
|
||||
columnTypes,
|
||||
tableOwner,
|
||||
temporaryTableName,
|
||||
connectionUrl,
|
||||
connectionProperties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (obj == null || getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
JdbcOutputTableHandle other = (JdbcOutputTableHandle) obj;
|
||||
return Objects.equals(this.connectorId, other.connectorId) &&
|
||||
Objects.equals(this.catalogName, other.catalogName) &&
|
||||
Objects.equals(this.schemaName, other.schemaName) &&
|
||||
Objects.equals(this.tableName, other.tableName) &&
|
||||
Objects.equals(this.columnNames, other.columnNames) &&
|
||||
Objects.equals(this.columnTypes, other.columnTypes) &&
|
||||
Objects.equals(this.tableOwner, other.tableOwner) &&
|
||||
Objects.equals(this.temporaryTableName, other.temporaryTableName) &&
|
||||
Objects.equals(this.connectionUrl, other.connectionUrl) &&
|
||||
Objects.equals(this.connectionProperties, other.connectionProperties);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorPartition;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
|
||||
import static com.google.common.base.MoreObjects.toStringHelper;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcPartition
|
||||
implements ConnectorPartition
|
||||
{
|
||||
private final JdbcTableHandle jdbcTableHandle;
|
||||
private final TupleDomain<ColumnHandle> domain;
|
||||
|
||||
public JdbcPartition(JdbcTableHandle jdbcTableHandle, TupleDomain<ColumnHandle> domain)
|
||||
{
|
||||
this.jdbcTableHandle = checkNotNull(jdbcTableHandle, "jdbcTableHandle is null");
|
||||
this.domain = checkNotNull(domain, "domain is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPartitionId()
|
||||
{
|
||||
return jdbcTableHandle.toString();
|
||||
}
|
||||
|
||||
public JdbcTableHandle getJdbcTableHandle()
|
||||
{
|
||||
return jdbcTableHandle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TupleDomain<ColumnHandle> getTupleDomain()
|
||||
{
|
||||
return domain;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return toStringHelper(this)
|
||||
.add("jdbcTableHandle", jdbcTableHandle)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ConnectorFactory;
|
||||
import com.facebook.presto.spi.Plugin;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.inject.Module;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
|
||||
public class JdbcPlugin
|
||||
implements Plugin
|
||||
{
|
||||
private final String name;
|
||||
private final Module module;
|
||||
private Map<String, String> optionalConfig = ImmutableMap.of();
|
||||
|
||||
public JdbcPlugin(String name, Module module)
|
||||
{
|
||||
checkArgument(!isNullOrEmpty(name), "name is null or empty");
|
||||
this.name = name;
|
||||
this.module = checkNotNull(module, "module is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setOptionalConfig(Map<String, String> optionalConfig)
|
||||
{
|
||||
this.optionalConfig = ImmutableMap.copyOf(checkNotNull(optionalConfig, "optionalConfig is null"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> List<T> getServices(Class<T> type)
|
||||
{
|
||||
if (type == ConnectorFactory.class) {
|
||||
return ImmutableList.of(type.cast(new JdbcConnectorFactory(name, module, optionalConfig, getClassLoader())));
|
||||
}
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
private static ClassLoader getClassLoader()
|
||||
{
|
||||
return firstNonNull(Thread.currentThread().getContextClassLoader(), JdbcPlugin.class.getClassLoader());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,362 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcCacheSplit;
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcJavaBean;
|
||||
import com.facebook.presto.spi.PrestoException;
|
||||
import com.facebook.presto.spi.RecordCursor;
|
||||
import com.facebook.presto.spi.type.BigintType;
|
||||
import com.facebook.presto.spi.type.DateType;
|
||||
import com.facebook.presto.spi.type.TimeType;
|
||||
import com.facebook.presto.spi.type.TimestampType;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.spi.type.VarbinaryType;
|
||||
import com.facebook.presto.spi.type.VarcharType;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.airlift.log.Logger;
|
||||
import io.airlift.slice.Slice;
|
||||
import org.joda.time.chrono.ISOChronology;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Date;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Time;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static com.facebook.presto.spi.StandardErrorCode.INTERNAL_ERROR;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static io.airlift.slice.Slices.utf8Slice;
|
||||
import static io.airlift.slice.Slices.wrappedBuffer;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
public class JdbcRecordCursor
|
||||
implements RecordCursor
|
||||
{
|
||||
private static final Logger log = Logger.get(JdbcRecordCursor.class);
|
||||
|
||||
private static final ISOChronology UTC_CHRONOLOGY = ISOChronology.getInstance(UTC);
|
||||
|
||||
private final List<JdbcColumnHandle> columnHandles;
|
||||
|
||||
private Connection connection;
|
||||
private Statement statement;
|
||||
private ResultSet resultSet;
|
||||
private boolean closed;
|
||||
|
||||
private List<JdbcJavaBean> tableDataSet;
|
||||
private boolean isCacheTable = false;
|
||||
private AtomicLong rowRecord = new AtomicLong(0);
|
||||
private BaseJdbcClient client;
|
||||
private JdbcSplit split;
|
||||
|
||||
public JdbcRecordCursor(JdbcClient jdbcClient, JdbcSplit split, List<JdbcColumnHandle> columnHandles)
|
||||
{
|
||||
this.client = (BaseJdbcClient) jdbcClient;
|
||||
this.split = split;
|
||||
isCacheTable = client.isCacheTable(split.getBaseTableName());
|
||||
this.columnHandles = ImmutableList.copyOf(checkNotNull(columnHandles, "columnHandles is null"));
|
||||
if (isCacheTable) {
|
||||
JdbcCacheSplit key = new JdbcCacheSplit(split.getConnectorId(), split.getCatalogName(),
|
||||
split.getSchemaName(), split.getTableName(), split.getConnectionUrl(), split.getBaseTableName());
|
||||
tableDataSet = client.getTableDataSet(key);
|
||||
}
|
||||
else {
|
||||
String sql = jdbcClient.buildSql(split, columnHandles);
|
||||
try {
|
||||
connection = jdbcClient.getConnection(split);
|
||||
statement = connection.createStatement();
|
||||
statement.setFetchSize(1000);
|
||||
|
||||
String whereCondition = split.getSplitPart();
|
||||
if (!isNullOrEmpty(whereCondition)) {
|
||||
if (whereCondition.indexOf("LIMIT") != -1) {
|
||||
sql += split.getSplitPart();
|
||||
}
|
||||
else {
|
||||
if (sql.indexOf("WHERE") != -1) {
|
||||
sql += " AND " + split.getSplitPart();
|
||||
}
|
||||
else {
|
||||
sql += " WHERE " + split.getSplitPart();
|
||||
}
|
||||
}
|
||||
}
|
||||
long startTime = System.currentTimeMillis();
|
||||
log.info("JdbcRecordCursor Executing: %s ", sql);
|
||||
resultSet = statement.executeQuery(sql);
|
||||
log.debug("The connection url: %s ,JdbcRecordCursor Executing: %s ,spend time : %s", split.getConnectionUrl(), sql, (System.currentTimeMillis() - startTime));
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error("Execute sql [%s] error, connection url : %s", sql, split.getConnectionUrl());
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getReadTimeNanos()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalBytes()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getCompletedBytes()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Type getType(int field)
|
||||
{
|
||||
return columnHandles.get(field).getColumnType();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean advanceNextPosition()
|
||||
{
|
||||
if (closed) {
|
||||
return false;
|
||||
}
|
||||
boolean result;
|
||||
if (isCacheTable) {
|
||||
long andIncrement = rowRecord.incrementAndGet();
|
||||
result = andIncrement <= tableDataSet.size();
|
||||
}
|
||||
else {
|
||||
try {
|
||||
result = resultSet.next();
|
||||
if (result) {
|
||||
rowRecord.getAndIncrement();
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
if (!result) {
|
||||
close();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean getBoolean(int field)
|
||||
{
|
||||
checkState(!closed, "cursor is closed");
|
||||
if (isCacheTable) {
|
||||
return (boolean) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
try {
|
||||
return resultSet.getBoolean(field + 1);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLong(int field)
|
||||
{
|
||||
checkState(!closed, "cursor is closed");
|
||||
try {
|
||||
Type type = getType(field);
|
||||
if (type.equals(BigintType.BIGINT)) {
|
||||
if (isCacheTable) {
|
||||
return (long) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
return resultSet.getLong(field + 1);
|
||||
}
|
||||
}
|
||||
if (type.equals(DateType.DATE)) {
|
||||
Date date = null;
|
||||
if (isCacheTable) {
|
||||
date = (Date) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
date = resultSet.getDate(field + 1);
|
||||
}
|
||||
// JDBC returns a date using a timestamp at midnight in the JVM timezone
|
||||
long localMillis = date.getTime();
|
||||
// Convert it to a midnight in UTC
|
||||
long utcMillis = ISOChronology.getInstance().getZone().getMillisKeepLocal(UTC, localMillis);
|
||||
// convert to days
|
||||
return TimeUnit.MILLISECONDS.toDays(utcMillis);
|
||||
}
|
||||
if (type.equals(TimeType.TIME)) {
|
||||
Time time = null;
|
||||
if (isCacheTable) {
|
||||
time = (Time) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
time = resultSet.getTime(field + 1);
|
||||
}
|
||||
return UTC_CHRONOLOGY.millisOfDay().get(time.getTime());
|
||||
}
|
||||
if (type.equals(TimestampType.TIMESTAMP)) {
|
||||
Timestamp timestamp = null;
|
||||
if (isCacheTable) {
|
||||
timestamp = (Timestamp) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
timestamp = resultSet.getTimestamp(field + 1);
|
||||
}
|
||||
return timestamp.getTime();
|
||||
}
|
||||
throw new PrestoException(INTERNAL_ERROR, "Unhandled type for long: " + type.getTypeSignature());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public double getDouble(int field)
|
||||
{
|
||||
checkState(!closed, "cursor is closed");
|
||||
if (isCacheTable) {
|
||||
return (double) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
try {
|
||||
return resultSet.getDouble(field + 1);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Slice getSlice(int field)
|
||||
{
|
||||
checkState(!closed, "cursor is closed");
|
||||
try {
|
||||
Type type = getType(field);
|
||||
if (type.equals(VarcharType.VARCHAR)) {
|
||||
String str = null;
|
||||
if (isCacheTable) {
|
||||
str = (String) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
str = resultSet.getString(field + 1);
|
||||
}
|
||||
return utf8Slice(str);
|
||||
}
|
||||
if (type.equals(VarbinaryType.VARBINARY)) {
|
||||
byte[] bytes = null;
|
||||
if (isCacheTable) {
|
||||
bytes = (byte[]) getFieldValue(field);
|
||||
}
|
||||
else {
|
||||
bytes = resultSet.getBytes(field + 1);
|
||||
}
|
||||
return wrappedBuffer(bytes);
|
||||
}
|
||||
throw new PrestoException(INTERNAL_ERROR, "Unhandled type for slice: " + type.getTypeSignature());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNull(int field)
|
||||
{
|
||||
checkState(!closed, "cursor is closed");
|
||||
checkArgument(field < columnHandles.size(), "Invalid field index");
|
||||
|
||||
try {
|
||||
if (isCacheTable) {
|
||||
Object feildValue = getFieldValue(field);
|
||||
return feildValue == null;
|
||||
}
|
||||
// JDBC is kind of dumb: we need to read the field and then ask
|
||||
// if it was null, which means we are wasting effort here.
|
||||
// We could save the result of the field access if it matters.
|
||||
resultSet.getObject(field + 1);
|
||||
|
||||
return resultSet.wasNull();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw handleSqlException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({"UnusedDeclaration", "EmptyTryBlock"})
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
if (!isNullOrEmpty(split.getSplitField()) && split.isCalcStepEnable()) {
|
||||
client.commitPdboLogs(split, rowRecord.get());
|
||||
}
|
||||
closed = true;
|
||||
try {
|
||||
if (statement != null) {
|
||||
statement.cancel();
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
// use try with resources to close everything properly
|
||||
try (ResultSet resultSet = this.resultSet;
|
||||
Statement statement = this.statement;
|
||||
Connection connection = this.connection) {
|
||||
// do nothing
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
private Object getFieldValue(int field)
|
||||
{
|
||||
String lowerCase = columnHandles.get(field).getColumnName().toLowerCase();
|
||||
JdbcJavaBean jdbcJavaBean = tableDataSet.get(rowRecord.intValue() - 1);
|
||||
return jdbcJavaBean.getFieldObjectValue(jdbcJavaBean.getColumns().indexOf(lowerCase));
|
||||
}
|
||||
|
||||
private RuntimeException handleSqlException(SQLException e)
|
||||
{
|
||||
try {
|
||||
close();
|
||||
}
|
||||
catch (Exception closeException) {
|
||||
e.addSuppressed(closeException);
|
||||
}
|
||||
return Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.RecordCursor;
|
||||
import com.facebook.presto.spi.RecordSet;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcRecordSet
|
||||
implements RecordSet
|
||||
{
|
||||
private final JdbcClient jdbcClient;
|
||||
private final List<JdbcColumnHandle> columnHandles;
|
||||
private final List<Type> columnTypes;
|
||||
private final JdbcSplit split;
|
||||
|
||||
public JdbcRecordSet(JdbcClient jdbcClient, JdbcSplit split, List<JdbcColumnHandle> columnHandles)
|
||||
{
|
||||
this.jdbcClient = checkNotNull(jdbcClient, "jdbcClient is null");
|
||||
this.split = checkNotNull(split, "split is null");
|
||||
|
||||
checkNotNull(split, "split is null");
|
||||
this.columnHandles = checkNotNull(columnHandles, "column handles is null");
|
||||
ImmutableList.Builder<Type> types = ImmutableList.builder();
|
||||
for (JdbcColumnHandle column : columnHandles) {
|
||||
types.add(column.getColumnType());
|
||||
}
|
||||
this.columnTypes = types.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Type> getColumnTypes()
|
||||
{
|
||||
return columnTypes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordCursor cursor()
|
||||
{
|
||||
return new JdbcRecordCursor(jdbcClient, split, columnHandles);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorRecordSetProvider;
|
||||
import com.facebook.presto.spi.ConnectorSplit;
|
||||
import com.facebook.presto.spi.RecordSet;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.Types.checkType;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcRecordSetProvider
|
||||
implements ConnectorRecordSetProvider
|
||||
{
|
||||
private final JdbcClient jdbcClient;
|
||||
|
||||
@Inject
|
||||
public JdbcRecordSetProvider(JdbcClient jdbcClient)
|
||||
{
|
||||
this.jdbcClient = checkNotNull(jdbcClient, "jdbcClient is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordSet getRecordSet(ConnectorSplit split, List<? extends ColumnHandle> columns)
|
||||
{
|
||||
JdbcSplit jdbcSplit = checkType(split, JdbcSplit.class, "split");
|
||||
|
||||
ImmutableList.Builder<JdbcColumnHandle> handles = ImmutableList.builder();
|
||||
for (ColumnHandle handle : columns) {
|
||||
handles.add(checkType(handle, JdbcColumnHandle.class, "columnHandle"));
|
||||
}
|
||||
|
||||
return new JdbcRecordSet(jdbcClient, jdbcSplit, handles.build());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.RecordSink;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.airlift.slice.Slice;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.chrono.ISOChronology;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Date;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.facebook.presto.spi.type.DateType.DATE;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
public class JdbcRecordSink
|
||||
implements RecordSink
|
||||
{
|
||||
private final Connection connection;
|
||||
private final PreparedStatement statement;
|
||||
|
||||
private final int fieldCount;
|
||||
private final List<Type> columnTypes;
|
||||
private int field = -1;
|
||||
private int batchSize;
|
||||
|
||||
public JdbcRecordSink(JdbcOutputTableHandle handle, JdbcClient jdbcClient)
|
||||
{
|
||||
try {
|
||||
connection = jdbcClient.getConnection(handle);
|
||||
connection.setAutoCommit(false);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
|
||||
try {
|
||||
statement = connection.prepareStatement(jdbcClient.buildInsertSql(handle));
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
|
||||
fieldCount = handle.getColumnNames().size();
|
||||
columnTypes = handle.getColumnTypes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beginRecord(long sampleWeight)
|
||||
{
|
||||
checkState(field == -1, "already in record");
|
||||
field = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishRecord()
|
||||
{
|
||||
checkState(field != -1, "not in record");
|
||||
checkState(field == fieldCount, "not all fields set");
|
||||
field = -1;
|
||||
|
||||
try {
|
||||
statement.addBatch();
|
||||
batchSize++;
|
||||
|
||||
if (batchSize >= 1000) {
|
||||
statement.executeBatch();
|
||||
connection.commit();
|
||||
connection.setAutoCommit(false);
|
||||
batchSize = 0;
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendNull()
|
||||
{
|
||||
try {
|
||||
statement.setObject(next(), null);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendBoolean(boolean value)
|
||||
{
|
||||
try {
|
||||
statement.setBoolean(next(), value);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendLong(long value)
|
||||
{
|
||||
try {
|
||||
if (DATE.equals(columnTypes.get(field))) {
|
||||
// convert to midnight in default time zone
|
||||
long utcMillis = TimeUnit.DAYS.toMillis(value);
|
||||
long localMillis = ISOChronology.getInstanceUTC().getZone().getMillisKeepLocal(DateTimeZone.getDefault(), utcMillis);
|
||||
statement.setDate(next(), new Date(localMillis));
|
||||
}
|
||||
else {
|
||||
statement.setLong(next(), value);
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendDouble(double value)
|
||||
{
|
||||
try {
|
||||
statement.setDouble(next(), value);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendString(byte[] value)
|
||||
{
|
||||
try {
|
||||
statement.setString(next(), new String(value, UTF_8));
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Slice> commit()
|
||||
{
|
||||
// commit and close
|
||||
try (Connection connection = this.connection) {
|
||||
if (batchSize > 0) {
|
||||
statement.executeBatch();
|
||||
connection.commit();
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
// the committer does not need any additional info
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
@SuppressWarnings("UnusedDeclaration")
|
||||
@Override
|
||||
public void rollback()
|
||||
{
|
||||
// rollback and close
|
||||
try (Connection connection = this.connection;
|
||||
PreparedStatement statement = this.statement) {
|
||||
connection.rollback();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Type> getColumnTypes()
|
||||
{
|
||||
return columnTypes;
|
||||
}
|
||||
|
||||
private int next()
|
||||
{
|
||||
checkState(field != -1, "not in record");
|
||||
checkState(field < fieldCount, "all fields already set");
|
||||
field++;
|
||||
return field;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ConnectorInsertTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorOutputTableHandle;
|
||||
import com.facebook.presto.spi.ConnectorRecordSinkProvider;
|
||||
import com.facebook.presto.spi.RecordSink;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.Types.checkType;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcRecordSinkProvider
|
||||
implements ConnectorRecordSinkProvider
|
||||
{
|
||||
private final JdbcClient jdbcClient;
|
||||
|
||||
@Inject
|
||||
public JdbcRecordSinkProvider(JdbcClient jdbcClient)
|
||||
{
|
||||
this.jdbcClient = checkNotNull(jdbcClient, "jdbcClient is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordSink getRecordSink(ConnectorOutputTableHandle tableHandle)
|
||||
{
|
||||
return new JdbcRecordSink(checkType(tableHandle, JdbcOutputTableHandle.class, "tableHandle"), jdbcClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecordSink getRecordSink(ConnectorInsertTableHandle tableHandle)
|
||||
{
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorSplit;
|
||||
import com.facebook.presto.spi.HostAddress;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcSplit
|
||||
implements ConnectorSplit
|
||||
{
|
||||
private final String connectorId;
|
||||
private final String catalogName;
|
||||
private final String schemaName;
|
||||
private final String tableName;
|
||||
private final String connectionUrl;
|
||||
private final Map<String, String> connectionProperties;
|
||||
private final TupleDomain<ColumnHandle> tupleDomain;
|
||||
private final String splitPart;
|
||||
private final List<HostAddress> addresses;
|
||||
private final boolean remotelyAccessible;
|
||||
private final String baseTableName;
|
||||
private final String splitField;
|
||||
private final String beginIndex;
|
||||
private final String endIndex;
|
||||
private final long timeStamp;
|
||||
private final int scanNodes;
|
||||
private final boolean isCalcStepEnable;
|
||||
private final String dbHost;
|
||||
|
||||
@JsonCreator
|
||||
public JdbcSplit(
|
||||
@JsonProperty("connectorId") String connectorId,
|
||||
@JsonProperty("catalogName") @Nullable String catalogName,
|
||||
@JsonProperty("schemaName") @Nullable String schemaName,
|
||||
@JsonProperty("tableName") String tableName,
|
||||
@JsonProperty("connectionUrl") String connectionUrl,
|
||||
@JsonProperty("connectionProperties") Map<String, String> connectionProperties,
|
||||
@JsonProperty("tupleDomain") TupleDomain<ColumnHandle> tupleDomain,
|
||||
@JsonProperty("splitPart") String splitPart,
|
||||
@JsonProperty("addresses") List<HostAddress> addresses,
|
||||
@JsonProperty("remotelyAccessible") boolean remotelyAccessible,
|
||||
@JsonProperty("baseTableName") String baseTableName,
|
||||
@JsonProperty("splitField") String splitField,
|
||||
@JsonProperty("beginIndex") String beginIndex,
|
||||
@JsonProperty("endIndex") String endIndex,
|
||||
@JsonProperty("timeStamp") long timeStamp,
|
||||
@JsonProperty("scanNodes") int scanNodes,
|
||||
@JsonProperty("isCalcStepEnable") boolean isCalcStepEnable,
|
||||
@JsonProperty("dbHost") String dbHost)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connector id is null");
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.tableName = checkNotNull(tableName, "table name is null");
|
||||
this.connectionUrl = checkNotNull(connectionUrl, "connectionUrl is null");
|
||||
this.connectionProperties = ImmutableMap.copyOf(checkNotNull(connectionProperties, "connectionProperties is null"));
|
||||
this.tupleDomain = checkNotNull(tupleDomain, "tupleDomain is null");
|
||||
this.splitPart = splitPart;
|
||||
this.remotelyAccessible = remotelyAccessible;
|
||||
this.addresses = checkNotNull(addresses, "host addresses is null");
|
||||
this.baseTableName = baseTableName;
|
||||
this.splitField = splitField;
|
||||
this.beginIndex = beginIndex;
|
||||
this.endIndex = endIndex;
|
||||
this.timeStamp = timeStamp;
|
||||
this.scanNodes = scanNodes;
|
||||
this.isCalcStepEnable = isCalcStepEnable;
|
||||
this.dbHost = dbHost;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getCatalogName()
|
||||
{
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getSchemaName()
|
||||
{
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getTableName()
|
||||
{
|
||||
return tableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectionUrl()
|
||||
{
|
||||
return connectionUrl;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public Map<String, String> getConnectionProperties()
|
||||
{
|
||||
return connectionProperties;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public TupleDomain<ColumnHandle> getTupleDomain()
|
||||
{
|
||||
return tupleDomain;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public boolean isRemotelyAccessible()
|
||||
{
|
||||
return remotelyAccessible;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public List<HostAddress> getAddresses()
|
||||
{
|
||||
return addresses;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getInfo()
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getSplitPart()
|
||||
{
|
||||
return splitPart;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getBaseTableName()
|
||||
{
|
||||
return baseTableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getSplitField()
|
||||
{
|
||||
return splitField;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getBeginIndex()
|
||||
{
|
||||
return beginIndex;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getEndIndex()
|
||||
{
|
||||
return endIndex;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getTimeStamp()
|
||||
{
|
||||
return timeStamp;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getScanNodes()
|
||||
{
|
||||
return scanNodes;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isCalcStepEnable()
|
||||
{
|
||||
return isCalcStepEnable;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getDbHost()
|
||||
{
|
||||
return dbHost;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorPartition;
|
||||
import com.facebook.presto.spi.ConnectorPartitionResult;
|
||||
import com.facebook.presto.spi.ConnectorSplit;
|
||||
import com.facebook.presto.spi.ConnectorSplitManager;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.ConnectorTableHandle;
|
||||
import com.facebook.presto.spi.FixedSplitSource;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.Types.checkType;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcSplitManager
|
||||
implements ConnectorSplitManager
|
||||
{
|
||||
private final String connectorId;
|
||||
private final JdbcClient jdbcClient;
|
||||
|
||||
@Inject
|
||||
public JdbcSplitManager(JdbcConnectorId connectorId, JdbcClient jdbcClient)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
|
||||
this.jdbcClient = checkNotNull(jdbcClient, "client is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain)
|
||||
{
|
||||
JdbcTableHandle handle = checkType(tableHandle, JdbcTableHandle.class, "tableHandle");
|
||||
return jdbcClient.getPartitions(handle, tupleDomain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorSplitSource getPartitionSplits(ConnectorTableHandle tableHandle, List<ConnectorPartition> partitions)
|
||||
{
|
||||
if (partitions.isEmpty()) {
|
||||
return new FixedSplitSource(connectorId, ImmutableList.<ConnectorSplit>of());
|
||||
}
|
||||
|
||||
checkArgument(partitions.size() == 1, "Expected one partition but got %s", partitions.size());
|
||||
JdbcPartition partition = checkType(partitions.get(0), JdbcPartition.class, "partition");
|
||||
|
||||
return jdbcClient.getPartitionSplits(partition);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ConnectorTableHandle;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public final class JdbcTableHandle
|
||||
implements ConnectorTableHandle
|
||||
{
|
||||
private final String connectorId;
|
||||
private final SchemaTableName schemaTableName;
|
||||
private final String catalogName;
|
||||
private final String schemaName;
|
||||
private final String tableName;
|
||||
|
||||
@JsonCreator
|
||||
public JdbcTableHandle(
|
||||
@JsonProperty("connectorId") String connectorId,
|
||||
@JsonProperty("schemaTableName") SchemaTableName schemaTableName,
|
||||
@JsonProperty("catalogName") @Nullable String catalogName,
|
||||
@JsonProperty("schemaName") @Nullable String schemaName,
|
||||
@JsonProperty("tableName") String tableName)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null");
|
||||
this.schemaTableName = checkNotNull(schemaTableName, "schemaTableName is null");
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.tableName = checkNotNull(tableName, "tableName is null");
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public SchemaTableName getSchemaTableName()
|
||||
{
|
||||
return schemaTableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getCatalogName()
|
||||
{
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Nullable
|
||||
public String getSchemaName()
|
||||
{
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getTableName()
|
||||
{
|
||||
return tableName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if ((obj == null) || (getClass() != obj.getClass())) {
|
||||
return false;
|
||||
}
|
||||
JdbcTableHandle o = (JdbcTableHandle) obj;
|
||||
return Objects.equals(this.connectorId, o.connectorId) &&
|
||||
Objects.equals(this.schemaTableName, o.schemaTableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(connectorId, schemaTableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return Joiner.on(":").useForNull("null").join(connectorId, schemaTableName, catalogName, schemaName, tableName);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,199 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.Domain;
|
||||
import com.facebook.presto.spi.Range;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.facebook.presto.spi.type.BigintType;
|
||||
import com.facebook.presto.spi.type.BooleanType;
|
||||
import com.facebook.presto.spi.type.DateType;
|
||||
import com.facebook.presto.spi.type.DoubleType;
|
||||
import com.facebook.presto.spi.type.TimestampType;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.spi.type.VarcharType;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import io.airlift.slice.Slice;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
|
||||
public class QueryBuilder
|
||||
{
|
||||
private final String quote;
|
||||
|
||||
public QueryBuilder(String quote)
|
||||
{
|
||||
this.quote = checkNotNull(quote, "quote is null");
|
||||
}
|
||||
|
||||
public String buildSql(int dbtype, String catalog, String schema, String table, List<JdbcColumnHandle> columns, TupleDomain<ColumnHandle> tupleDomain)
|
||||
{
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("SELECT ");
|
||||
Joiner.on(", ").appendTo(sql, transform(columns, column -> quote(column.getColumnName())));
|
||||
if (columns.isEmpty()) {
|
||||
sql.append("null");
|
||||
}
|
||||
|
||||
sql.append(" FROM ");
|
||||
if (!isNullOrEmpty(catalog)) {
|
||||
sql.append(quote(catalog)).append('.');
|
||||
}
|
||||
if (!isNullOrEmpty(schema)) {
|
||||
sql.append(quote(schema)).append('.');
|
||||
}
|
||||
sql.append(quote(table));
|
||||
if (dbtype == BaseJdbcClient.TYPE_SQLSERVER) {
|
||||
sql.append(" WITH(NOLOCK) ");
|
||||
}
|
||||
|
||||
List<String> clauses = toConjuncts(columns, tupleDomain);
|
||||
if (!clauses.isEmpty()) {
|
||||
sql.append(" WHERE ")
|
||||
.append(Joiner.on(" AND ").join(clauses));
|
||||
}
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private List<String> toConjuncts(List<JdbcColumnHandle> columns, TupleDomain<ColumnHandle> tupleDomain)
|
||||
{
|
||||
ImmutableList.Builder<String> builder = ImmutableList.builder();
|
||||
for (JdbcColumnHandle column : columns) {
|
||||
Type type = column.getColumnType();
|
||||
if (type.equals(BigintType.BIGINT) || type.equals(DoubleType.DOUBLE) || type.equals(BooleanType.BOOLEAN)
|
||||
|| type.equals(VarcharType.VARCHAR) || type.equals(DateType.DATE) || type.equals(TimestampType.TIMESTAMP)) {
|
||||
Domain domain = tupleDomain.getDomains().get(column);
|
||||
if (domain != null) {
|
||||
builder.add(toPredicate(column.getColumnName(), domain, type));
|
||||
}
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private String toPredicate(String columnName, Domain domain, Type columnType)
|
||||
{
|
||||
if (domain.getRanges().isNone() && domain.isNullAllowed()) {
|
||||
return quote(columnName) + " IS NULL";
|
||||
}
|
||||
|
||||
if (domain.getRanges().isAll() && !domain.isNullAllowed()) {
|
||||
return quote(columnName) + " IS NOT NULL";
|
||||
}
|
||||
|
||||
// Add disjuncts for ranges
|
||||
List<String> disjuncts = new ArrayList<>();
|
||||
List<Object> singleValues = new ArrayList<>();
|
||||
for (Range range : domain.getRanges()) {
|
||||
checkState(!range.isAll()); // Already checked
|
||||
if (range.isSingleValue()) {
|
||||
singleValues.add(range.getLow().getValue());
|
||||
}
|
||||
else {
|
||||
List<String> rangeConjuncts = new ArrayList<>();
|
||||
if (!range.getLow().isLowerUnbounded()) {
|
||||
switch (range.getLow().getBound()) {
|
||||
case ABOVE:
|
||||
rangeConjuncts.add(toPredicate(columnName, ">", range.getLow().getValue(), columnType));
|
||||
break;
|
||||
case EXACTLY:
|
||||
rangeConjuncts.add(toPredicate(columnName, ">=", range.getLow().getValue(), columnType));
|
||||
break;
|
||||
case BELOW:
|
||||
throw new IllegalArgumentException("Low Marker should never use BELOW bound: " + range);
|
||||
default:
|
||||
throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
|
||||
}
|
||||
}
|
||||
if (!range.getHigh().isUpperUnbounded()) {
|
||||
switch (range.getHigh().getBound()) {
|
||||
case ABOVE:
|
||||
throw new IllegalArgumentException("High Marker should never use ABOVE bound: " + range);
|
||||
case EXACTLY:
|
||||
rangeConjuncts.add(toPredicate(columnName, "<=", range.getHigh().getValue(), columnType));
|
||||
break;
|
||||
case BELOW:
|
||||
rangeConjuncts.add(toPredicate(columnName, "<", range.getHigh().getValue(), columnType));
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
|
||||
}
|
||||
}
|
||||
// If rangeConjuncts is null, then the range was ALL, which should already have been checked for
|
||||
checkState(!rangeConjuncts.isEmpty());
|
||||
disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")");
|
||||
}
|
||||
}
|
||||
|
||||
// Add back all of the possible single values either as an equality or an IN predicate
|
||||
if (singleValues.size() == 1) {
|
||||
disjuncts.add(toPredicate(columnName, "=", getOnlyElement(singleValues), columnType));
|
||||
}
|
||||
else if (singleValues.size() > 1) {
|
||||
ImmutableList.Builder<String> inListBuilder = ImmutableList.builder();
|
||||
singleValues.stream().forEach(value -> inListBuilder.add(encode(value, columnType)));
|
||||
disjuncts.add(quote(columnName) + " IN (" + Joiner.on(",").join(inListBuilder.build()) + ")");
|
||||
}
|
||||
|
||||
// Add nullability disjuncts
|
||||
checkState(!disjuncts.isEmpty());
|
||||
if (domain.isNullAllowed()) {
|
||||
disjuncts.add(quote(columnName) + " IS NULL");
|
||||
}
|
||||
|
||||
return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
|
||||
}
|
||||
|
||||
private String toPredicate(String columnName, String operator, Object value, Type columnType)
|
||||
{
|
||||
return quote(columnName) + " " + operator + " " + encode(value, columnType);
|
||||
}
|
||||
|
||||
private String quote(String name)
|
||||
{
|
||||
name = name.replace(quote, quote + quote);
|
||||
return quote + name + quote;
|
||||
}
|
||||
|
||||
private static String encode(Object value, Type columnType)
|
||||
{
|
||||
if (value instanceof Number || value instanceof Boolean) {
|
||||
if (columnType.equals(DateType.DATE)) {
|
||||
return "'" + new SimpleDateFormat("yyyy-MM-dd").format(new Date(86400000 * Long.parseLong(value.toString(), 10))) + "'";
|
||||
}
|
||||
else if (columnType.equals(TimestampType.TIMESTAMP)) {
|
||||
return "'" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date(Long.parseLong(value.toString(), 10))) + "'";
|
||||
}
|
||||
return value.toString();
|
||||
}
|
||||
else if (value instanceof Slice) {
|
||||
return "'" + ((Slice) value).toStringUtf8() + "'";
|
||||
}
|
||||
throw new UnsupportedOperationException("Can't handle type: " + value.getClass().getName());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public final class Types
|
||||
{
|
||||
private Types() {}
|
||||
|
||||
public static <A, B extends A> B checkType(A value, Class<B> target, String name)
|
||||
{
|
||||
checkNotNull(value, "%s is null", name);
|
||||
checkArgument(target.isInstance(value),
|
||||
"%s must be of type %s, not %s",
|
||||
name,
|
||||
target.getName(),
|
||||
value.getClass().getName());
|
||||
return target.cast(value);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.cache;
|
||||
|
||||
import io.airlift.configuration.Config;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class JdbcCacheConfig
|
||||
{
|
||||
public static final String DEFAULT_VALUE = "NA";
|
||||
private String cacheTableConfig = DEFAULT_VALUE;
|
||||
private String cacheTableClause;
|
||||
private Duration cacheRefreshInterval = new Duration(5, TimeUnit.MINUTES);
|
||||
private Duration cacheExpireInterval = new Duration(5, TimeUnit.MINUTES);
|
||||
private boolean jdbcCacheEnable = false;
|
||||
|
||||
public String getCacheTableConfig()
|
||||
{
|
||||
return cacheTableConfig;
|
||||
}
|
||||
|
||||
@Config("jdbc-cache-table-config")
|
||||
public JdbcCacheConfig setCacheTableConfig(String cacheTableConfig)
|
||||
{
|
||||
this.cacheTableConfig = cacheTableConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getCacheTableClause()
|
||||
{
|
||||
return cacheTableClause;
|
||||
}
|
||||
|
||||
@Config("jdbc-cache-table-clause")
|
||||
public JdbcCacheConfig setCacheTableClause(String cacheTableClause)
|
||||
{
|
||||
this.cacheTableClause = cacheTableClause;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Duration getCacheRefreshInterval()
|
||||
{
|
||||
return cacheRefreshInterval;
|
||||
}
|
||||
|
||||
@Config("jdbc-cache-refresh-interval")
|
||||
public JdbcCacheConfig setCacheRefreshInterval(Duration cacheRefreshInterval)
|
||||
{
|
||||
this.cacheRefreshInterval = cacheRefreshInterval;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Duration getCacheExpireInterval()
|
||||
{
|
||||
return cacheExpireInterval;
|
||||
}
|
||||
|
||||
@Config("jdbc-cache-expire-interval")
|
||||
public JdbcCacheConfig setCacheExpireInterval(Duration cacheExpireInterval)
|
||||
{
|
||||
this.cacheExpireInterval = cacheExpireInterval;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getJdbcCacheEnable()
|
||||
{
|
||||
return jdbcCacheEnable;
|
||||
}
|
||||
|
||||
@Config("jdbc-cache-enable")
|
||||
public JdbcCacheConfig setJdbcCacheEnable(boolean jdbcCacheEnable)
|
||||
{
|
||||
this.jdbcCacheEnable = jdbcCacheEnable;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.cache;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class JdbcCacheSplit
|
||||
{
|
||||
private final String connectorId;
|
||||
private final String catalogName;
|
||||
private final String schemaName;
|
||||
private final String tableName;
|
||||
private final String baseTableName;
|
||||
private final String connectionUrl;
|
||||
|
||||
public JdbcCacheSplit(String connectorId, String catalogName,
|
||||
String schemaName, String tableName, String connectionUrl, String baseTableName)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connector id is null");
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = schemaName;
|
||||
this.tableName = checkNotNull(tableName, "table name is null");
|
||||
this.connectionUrl = checkNotNull(connectionUrl, "connectionUrl is null");
|
||||
this.baseTableName = checkNotNull(baseTableName, "table name is null");
|
||||
}
|
||||
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
public String getCatalogName()
|
||||
{
|
||||
return catalogName == null ? "null" : catalogName;
|
||||
}
|
||||
|
||||
public String getSchemaName()
|
||||
{
|
||||
return schemaName == null ? "null" : schemaName;
|
||||
}
|
||||
|
||||
public String getTableName()
|
||||
{
|
||||
return tableName;
|
||||
}
|
||||
|
||||
public String getBaseTableName()
|
||||
{
|
||||
return baseTableName;
|
||||
}
|
||||
|
||||
public String getConnectionUrl()
|
||||
{
|
||||
return connectionUrl;
|
||||
}
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(getConnectorId(), getConnectionUrl(), getCatalogName(), getSchemaName(), getTableName());
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (obj instanceof JdbcCacheSplit) {
|
||||
JdbcCacheSplit other = (JdbcCacheSplit) obj;
|
||||
return this.getConnectorId().equals(other.getConnectorId())
|
||||
&& this.getConnectionUrl().equals(other.getConnectionUrl())
|
||||
&& this.getCatalogName().equals(other.getCatalogName())
|
||||
&& this.getSchemaName().equals(other.getSchemaName())
|
||||
&& this.getTableName().equals(other.getTableName());
|
||||
}
|
||||
else {
|
||||
return this.hashCode() == obj.hashCode();
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return getConnectorId() + ","
|
||||
+ getConnectionUrl() + ","
|
||||
+ getCatalogName() + ","
|
||||
+ getSchemaName() + ","
|
||||
+ getTableName();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.cache;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class JdbcJavaBean
|
||||
{
|
||||
private List<String> columns;
|
||||
private Object[] values;
|
||||
public JdbcJavaBean(List<String> columns)
|
||||
{
|
||||
this.columns = columns;
|
||||
this.values = new Object[columns.size()];
|
||||
}
|
||||
|
||||
public Object getFieldObjectValue(int index)
|
||||
{
|
||||
return values[index];
|
||||
}
|
||||
|
||||
public void setFieldObjectValue(int index, Object value)
|
||||
{
|
||||
values[index] = value;
|
||||
}
|
||||
|
||||
public List<String> getColumns()
|
||||
{
|
||||
return columns;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,227 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.cache;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.Driver;
|
||||
import java.sql.Statement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.util.JdbcUtil;
|
||||
import com.facebook.presto.spi.type.BigintType;
|
||||
import com.facebook.presto.spi.type.BooleanType;
|
||||
import com.facebook.presto.spi.type.DateType;
|
||||
import com.facebook.presto.spi.type.DoubleType;
|
||||
import com.facebook.presto.spi.type.TimeType;
|
||||
import com.facebook.presto.spi.type.TimestampType;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.spi.type.VarbinaryType;
|
||||
import com.facebook.presto.spi.type.VarcharType;
|
||||
import com.fasterxml.jackson.core.JsonParseException;
|
||||
import com.fasterxml.jackson.databind.JsonMappingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public class JdbcResultCache
|
||||
{
|
||||
private final LoadingCache<JdbcCacheSplit, List<JdbcJavaBean>> jdbcResultCache;
|
||||
private final String identifierQuote;
|
||||
private final Driver driver;
|
||||
private final Properties connectionProperties;
|
||||
private static final Logger log = Logger.get(JdbcResultCache.class);
|
||||
private List<String> tableList = new ArrayList<String>();
|
||||
private HashMap<String, List<String>> fieldList = new HashMap<String, List<String>>();
|
||||
private LinkedHashMap<String, String> cacheTableClauseMap;
|
||||
|
||||
public JdbcResultCache(String identifierQuote,
|
||||
Driver driver,
|
||||
Properties connectionProperties,
|
||||
JdbcCacheConfig cacheConfig)
|
||||
{
|
||||
this.identifierQuote = identifierQuote;
|
||||
this.driver = driver;
|
||||
this.connectionProperties = connectionProperties;
|
||||
long expiresAfterWrite = checkNotNull(cacheConfig.getCacheExpireInterval(), "cacheExpireInterval is null").toMillis();
|
||||
long refreshAfterWrite = checkNotNull(cacheConfig.getCacheRefreshInterval(), "cacheRefreshInterval is null").toMillis();
|
||||
analyseCacheTableAndField(cacheConfig.getCacheTableConfig(), cacheConfig.getCacheTableClause());
|
||||
jdbcResultCache = CacheBuilder
|
||||
.newBuilder()
|
||||
.expireAfterWrite(expiresAfterWrite, TimeUnit.MILLISECONDS)
|
||||
.refreshAfterWrite(refreshAfterWrite, TimeUnit.MILLISECONDS)
|
||||
.build(new CacheLoader<JdbcCacheSplit, List<JdbcJavaBean>>(){
|
||||
@Override
|
||||
public List<JdbcJavaBean> load(JdbcCacheSplit key) throws Exception
|
||||
{
|
||||
return loadTableDataSet(key);
|
||||
}
|
||||
});
|
||||
}
|
||||
private List<JdbcJavaBean> loadTableDataSet(JdbcCacheSplit key)
|
||||
{
|
||||
log.debug("loadTableDataSet key : " + key);
|
||||
List<JdbcJavaBean> list = new ArrayList<JdbcJavaBean>();
|
||||
try {
|
||||
Connection connection = getConnection(key.getConnectionUrl());
|
||||
HashMap<String, Type> types = getColumnTypes(key);
|
||||
String tableName = key.getBaseTableName();
|
||||
List<String> columns = fieldList.get(tableName);
|
||||
String columnPart = Joiner.on(",").join(columns);
|
||||
String sql = "SELECT " + columnPart + " FROM " +
|
||||
JdbcUtil.getTableName(identifierQuote, key.getCatalogName(), key.getSchemaName(), key.getTableName());
|
||||
if (cacheTableClauseMap != null && !isNullOrEmpty(cacheTableClauseMap.get(tableName))) {
|
||||
sql += " WHERE " + cacheTableClauseMap.get(tableName);
|
||||
}
|
||||
Statement statement = connection.createStatement();
|
||||
statement.setFetchSize(10_000);
|
||||
long startTime = System.currentTimeMillis();
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
log.debug("The connection url: %s ,ExecuteQuery: %s ,spend time : %s , thread id : %s", key.getConnectionUrl(), sql, (System.currentTimeMillis() - startTime), Thread.currentThread().getId());
|
||||
while (resultSet.next()) {
|
||||
JdbcJavaBean tableDataSet = new JdbcJavaBean(columns);
|
||||
for (int i = 1; i <= columns.size(); i++) {
|
||||
Type type = types.get(columns.get(i - 1));
|
||||
if (type.equals(BooleanType.BOOLEAN)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getBoolean(i));
|
||||
}
|
||||
else if (type.equals(BigintType.BIGINT)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getLong(i));
|
||||
}
|
||||
else if (type.equals(DateType.DATE)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getDate(i));
|
||||
}
|
||||
else if (type.equals(TimeType.TIME)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getTime(i));
|
||||
}
|
||||
else if (type.equals(TimestampType.TIMESTAMP)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getTimestamp(i));
|
||||
}
|
||||
else if (type.equals(DoubleType.DOUBLE)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getDouble(i));
|
||||
}
|
||||
else if (type.equals(VarcharType.VARCHAR)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getString(i));
|
||||
}
|
||||
else if (type.equals(VarbinaryType.VARBINARY)) {
|
||||
tableDataSet.setFieldObjectValue((i - 1), resultSet.getBytes(i));
|
||||
}
|
||||
}
|
||||
list.add(tableDataSet);
|
||||
}
|
||||
log.debug("The connection url: %s ,parse result: %s ,spend time : %s , thread id : %s", key.getConnectionUrl(), sql, (System.currentTimeMillis() - startTime), Thread.currentThread().getId());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
public List<JdbcJavaBean> getResult(JdbcCacheSplit key)
|
||||
{
|
||||
try {
|
||||
return jdbcResultCache.get(key);
|
||||
}
|
||||
catch (ExecutionException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
public HashMap<String, Type> getColumnTypes(JdbcCacheSplit key)
|
||||
{
|
||||
HashMap<String, Type> types = new HashMap<String, Type>();
|
||||
try (Connection connection = getConnection(key.getConnectionUrl())) {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
try (ResultSet resultSet = metadata.getColumns(key.getSchemaName(), key.getCatalogName(), key.getTableName(), null)) {
|
||||
while (resultSet.next()) {
|
||||
Type columnType = JdbcUtil.toPrestoType(resultSet.getInt("DATA_TYPE"));
|
||||
if (columnType != null) {
|
||||
String columnName = resultSet.getString("COLUMN_NAME").toLowerCase(ENGLISH);
|
||||
types.put(columnName, columnType);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
public Connection getConnection(String connectionURL)
|
||||
throws SQLException
|
||||
{
|
||||
Connection connection = driver.connect(connectionURL, connectionProperties);
|
||||
try {
|
||||
connection.setReadOnly(true);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
connection.close();
|
||||
throw e;
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
private void analyseCacheTableAndField(String cacheTableConfig, String cacheTableClause)
|
||||
{
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
try {
|
||||
// table name and column field
|
||||
List<LinkedHashMap<String, Object>> readValue = objectMapper.readValue(cacheTableConfig.toLowerCase(ENGLISH), ArrayList.class);
|
||||
for (LinkedHashMap<String, Object> map : readValue) {
|
||||
for (String t : map.keySet()) {
|
||||
tableList.add(t);
|
||||
ArrayList<String> object = (ArrayList<String>) map.get(t);
|
||||
fieldList.put(t, object);
|
||||
}
|
||||
}
|
||||
if (!isNullOrEmpty(cacheTableClause)) {
|
||||
// table where condition
|
||||
cacheTableClauseMap = (LinkedHashMap<String, String>) objectMapper.readValue(cacheTableClause, Map.class);
|
||||
}
|
||||
}
|
||||
catch (JsonParseException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
catch (JsonMappingException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isCacheTable(String tableName)
|
||||
{
|
||||
return tableList.contains(tableName);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,255 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import io.airlift.log.Logger;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import org.apache.commons.dbutils.DbUtils;
|
||||
import org.apache.commons.dbutils.QueryRunner;
|
||||
import org.apache.commons.dbutils.ResultSetHandler;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.JdbcSplit;
|
||||
import com.facebook.presto.plugin.jdbc.subtable.PdboTableInfo.DBType;
|
||||
import com.facebook.presto.plugin.jdbc.util.JdbcUtil;
|
||||
import com.facebook.presto.plugin.jdbc.util.PdboMetadata;
|
||||
import com.facebook.presto.server.PrestoServer;
|
||||
import com.mysql.jdbc.Driver;
|
||||
|
||||
public class JdbcLoadTread implements Runnable
|
||||
{
|
||||
private static final Logger log = Logger.get(JdbcLoadTread.class);
|
||||
|
||||
protected final String connectionUrl;
|
||||
protected final Properties connectionProperties;
|
||||
protected final String connectorId;
|
||||
protected final Duration jdbcReloadSubtableInterval;
|
||||
private long lastLoadSubTableTimeStamp = 0L;
|
||||
protected final Driver driver;
|
||||
private final boolean jdbcSubTableAllocator;
|
||||
|
||||
private final ConcurrentMap<PdboTableInfo, ArrayList<PdboSplit>> pdboTables = new ConcurrentHashMap<>();
|
||||
|
||||
public JdbcLoadTread(String connectionUrl,
|
||||
Properties connectionProperties,
|
||||
String connectorId,
|
||||
Duration jdbcReloadSubtableInterval) throws SQLException
|
||||
{
|
||||
this.connectionUrl = connectionUrl;
|
||||
this.connectionProperties = connectionProperties;
|
||||
this.connectorId = connectorId;
|
||||
this.jdbcReloadSubtableInterval = jdbcReloadSubtableInterval;
|
||||
this.driver = new Driver();
|
||||
this.jdbcSubTableAllocator = PrestoServer.isCoordinator();
|
||||
}
|
||||
|
||||
public void run()
|
||||
{
|
||||
while (jdbcSubTableAllocator) {
|
||||
try {
|
||||
if (lastLoadSubTableTimeStamp == 0) {
|
||||
loadPdboTableInfo();
|
||||
}
|
||||
else {
|
||||
Thread.sleep(jdbcReloadSubtableInterval.toMillis());
|
||||
long curTime = System.currentTimeMillis();
|
||||
loadPdboTableInfo();
|
||||
log.debug(connectorId + " load sub-table info spend time : " + (System.currentTimeMillis() - curTime) + " ms ");
|
||||
}
|
||||
lastLoadSubTableTimeStamp = System.currentTimeMillis();
|
||||
}
|
||||
catch (Exception e) {
|
||||
lastLoadSubTableTimeStamp = System.currentTimeMillis();
|
||||
log.error(e, connectorId + " Error reloading sub-table infomation : %s" , e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void loadPdboTableInfo()
|
||||
{
|
||||
String sql = PdboMetadata.getPdboTableInfoSQL();
|
||||
Connection conn = getConnection();
|
||||
QueryRunner runner = new QueryRunner();
|
||||
try {
|
||||
runner.query(conn, sql, new PdboTableResultHandle(), connectorId);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error(e, "loadPdboTableInfo Execute %s error : %s" , sql, e.getMessage());
|
||||
}
|
||||
finally {
|
||||
DbUtils.closeQuietly(conn);
|
||||
}
|
||||
}
|
||||
|
||||
public List<PdboSplit> getPDBOLogs(String connectorId, String schemaName, String tableName)
|
||||
{
|
||||
String sql = PdboMetadata.getPdboLogsSQL();
|
||||
Connection conn = getConnection();
|
||||
QueryRunner runner = new QueryRunner();
|
||||
List<PdboSplit> pdboSplits = null;
|
||||
try {
|
||||
pdboSplits = runner.query(conn, sql, new PdboLogsResultHandle(), connectorId, schemaName, tableName);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error(e, "getPDBOLogs Execute %s error : %s" , sql, e.getMessage());
|
||||
}
|
||||
finally {
|
||||
DbUtils.closeQuietly(conn);
|
||||
}
|
||||
return pdboSplits;
|
||||
}
|
||||
|
||||
private class PdboLogsResultHandle implements ResultSetHandler<List<PdboSplit>>
|
||||
{
|
||||
@Override
|
||||
public List<PdboSplit> handle(ResultSet rs) throws SQLException
|
||||
{
|
||||
List<PdboSplit> tables = new ArrayList<>();
|
||||
int scannodenumber = 0;
|
||||
//A.CONNECTORID,A.SCHEMANAME,A.TABLENAME,A.ROWS,A.BEGININDEX,A.ENDINDEX,B.DBTYPE,
|
||||
//C.DBHOST,C.DBPORT,C.CONNECTION_PROPERTIES,C.PRESTO_WORK_HOST,C.REMOTELYACCESSIBLE,C.SPLITFIELD,
|
||||
//C.SCANNODENUMBER,D.USERNAME,D.PASSWORD,A.CONTROL_SCAN_CONCURRENCY_ENABLED,A.SCAN_CONCURRENCY_COUNT
|
||||
while (rs.next()) {
|
||||
scannodenumber = rs.getInt(14);
|
||||
tables.add(new PdboSplit().setConnectorId(rs.getString(1)).
|
||||
setSchemaName(rs.getString(2)).
|
||||
setTableName(rs.getString(3)).
|
||||
setRows(rs.getLong(4)).
|
||||
setBeginIndex(rs.getLong(5)).
|
||||
setEndIndex(rs.getLong(6)).
|
||||
setDbHost(rs.getString(8)).
|
||||
setConnectionUrl(getConnectionURL(rs.getString(7), rs.getString(8), rs.getString(9), rs.getString(10))).
|
||||
setPrestoWorkHost(rs.getString(11)).
|
||||
setRemotelyAccessible(rs.getString(12)).
|
||||
setSplitField(rs.getString(13)).
|
||||
setScanNodes(rs.getInt(14)).
|
||||
setUsername(JdbcUtil.Base64Decode(rs.getString(15))).
|
||||
setPassword(JdbcUtil.Base64Decode(rs.getString(16))).
|
||||
setCalcStepEnable("Y").
|
||||
setControlScanConcurrencyEnabled(rs.getString(17)).
|
||||
setScanConcurrencyCount(rs.getInt(18)));
|
||||
}
|
||||
if (scannodenumber != tables.size()) {
|
||||
tables.clear();
|
||||
loadPdboTableInfo();
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
}
|
||||
|
||||
private class PdboTableResultHandle implements ResultSetHandler<String>
|
||||
{
|
||||
@Override
|
||||
public String handle(ResultSet rs) throws SQLException
|
||||
{
|
||||
pdboTables.clear();
|
||||
//CONNECTORID,PRESTO_SCHEMA,PRESTO_TABLE,DBTYPE,PDBOENABLE,CONTROL_SCAN_CONCURRENCY_ENABLED,SCAN_CONCURRENCY_COUNT
|
||||
//DBHOST,DBPORT,CONNECTION_PROPERTIES,SOURCE_SCHEMA,SOURCE_TABLE,SPLITFIELD,REMOTELYACCESSIBLE,PRESTO_WORK_HOST,SCANNODENUMBER,
|
||||
//FIELDMAXVALUE,FIELDMINVALUE,USERNAME,PASSWORD,"
|
||||
while (rs.next()) {
|
||||
PdboTableInfo table = new PdboTableInfo(rs.getString(1).toLowerCase(ENGLISH),
|
||||
rs.getString(2).toLowerCase(ENGLISH), rs.getString(3).toLowerCase(ENGLISH));
|
||||
table.setDbType(rs.getString(4));
|
||||
table.setCalcStepEnable(rs.getString(5));
|
||||
table.setControlScanConcurrencyEnabled(rs.getString(6));
|
||||
table.setScanConcurrencyCount(rs.getInt(7));
|
||||
String connectionUrl = getConnectionURL(table.getDbType(), rs.getString(8), rs.getString(9), rs.getString(10));
|
||||
PdboSplit pdboSplit = new PdboSplit().setSchemaName(rs.getString(11).toLowerCase(ENGLISH)).
|
||||
setTableName(rs.getString(12).toLowerCase(ENGLISH)).
|
||||
setDbHost(rs.getString(8)).
|
||||
setConnectionUrl(connectionUrl).
|
||||
setSplitField(rs.getString(13)).
|
||||
setRemotelyAccessible(rs.getString(14)).
|
||||
setPrestoWorkHost(rs.getString(15)).
|
||||
setScanNodes(rs.getInt(16)).
|
||||
setFieldMaxValue(rs.getLong(17)).
|
||||
setFieldMinValue(rs.getLong(18)).
|
||||
setUsername(JdbcUtil.Base64Decode(rs.getString(19))).
|
||||
setPassword(JdbcUtil.Base64Decode(rs.getString(20))).
|
||||
setCalcStepEnable(rs.getString(5)).
|
||||
setControlScanConcurrencyEnabled(rs.getString(6)).
|
||||
setScanConcurrencyCount(rs.getInt(7));
|
||||
ArrayList<PdboSplit> routeList = pdboTables.get(table);
|
||||
if (routeList == null) {
|
||||
routeList = new ArrayList<>();
|
||||
}
|
||||
routeList.add(pdboSplit);
|
||||
pdboTables.put(table, routeList);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getConnectionURL(String dbType, String dbHost, String dbPort, String connectionProperties)
|
||||
throws SQLException
|
||||
{
|
||||
String connectionUrl = "";
|
||||
if (dbType.equals(DBType.MYSQL.toString())) {
|
||||
connectionUrl = "jdbc:mysql://";
|
||||
}
|
||||
else if (dbType.equals(DBType.SQLSERVER.toString())) {
|
||||
connectionUrl = "jdbc:jtds:sqlserver://";
|
||||
}
|
||||
else if (dbType.equals(DBType.ORACLE.toString())) {
|
||||
connectionUrl = "jdbc:oracle:thin:@";
|
||||
}
|
||||
connectionUrl += dbHost + ":" + dbPort + connectionProperties;
|
||||
return connectionUrl;
|
||||
}
|
||||
|
||||
public ConcurrentMap<PdboTableInfo, ArrayList<PdboSplit>> getPdboTableInfo()
|
||||
{
|
||||
return pdboTables;
|
||||
}
|
||||
|
||||
public void commitPdboLogs(JdbcSplit split, long rowCount)
|
||||
{
|
||||
Connection conn = getConnection();
|
||||
QueryRunner runner = new QueryRunner();
|
||||
String insertSql = PdboMetadata.getInsertPdboLogSQL(split, rowCount, connectorId);
|
||||
String updateSql = PdboMetadata.getUpdatePdboHistoryLogSQL(split, connectorId);
|
||||
try {
|
||||
runner.update(conn, updateSql);
|
||||
runner.update(conn, insertSql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error(e, "insert sql : %s,update sql : %s commitPdboLogs error : %s", insertSql, updateSql, e.getMessage());
|
||||
}
|
||||
finally {
|
||||
DbUtils.closeQuietly(conn);
|
||||
}
|
||||
}
|
||||
|
||||
public Connection getConnection()
|
||||
{
|
||||
Connection conn = null;
|
||||
try {
|
||||
conn = driver.connect(connectionUrl, connectionProperties);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error("Connect pdbo db error : " + e.getMessage());
|
||||
}
|
||||
return conn;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import io.airlift.configuration.Config;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class JdbcSubTableConfig
|
||||
{
|
||||
public static final String DEFAULT_VALUE = "NA";
|
||||
|
||||
private String jdbcSubTableConnectionURL;
|
||||
private String jdbcSubTableConnectionUser;
|
||||
private String jdbcSubTableConnectionPassword;
|
||||
private Duration jdbcReloadSubtableInterval = new Duration(5, TimeUnit.MINUTES);
|
||||
private boolean jdbcSubTableEnable = false;
|
||||
|
||||
public String getJdbcSubTableConnectionURL()
|
||||
{
|
||||
return jdbcSubTableConnectionURL;
|
||||
}
|
||||
|
||||
@Config("jdbc-sub-table-connection-url")
|
||||
public JdbcSubTableConfig setJdbcSubTableConnectionURL(String connectionUrl)
|
||||
{
|
||||
this.jdbcSubTableConnectionURL = connectionUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getJdbcSubTableConnectionUser()
|
||||
{
|
||||
return jdbcSubTableConnectionUser;
|
||||
}
|
||||
|
||||
@Config("jdbc-sub-table-connection-user")
|
||||
public JdbcSubTableConfig setJdbcSubTableConnectionUser(String connectionUser)
|
||||
{
|
||||
this.jdbcSubTableConnectionUser = connectionUser;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getJdbcSubTableConnectionPassword()
|
||||
{
|
||||
return jdbcSubTableConnectionPassword;
|
||||
}
|
||||
|
||||
@Config("jdbc-sub-table-connection-password")
|
||||
public JdbcSubTableConfig setJdbcSubTableConnectionPassword(String connectionPassword)
|
||||
{
|
||||
this.jdbcSubTableConnectionPassword = connectionPassword;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Duration getJdbcReloadSubtableInterval()
|
||||
{
|
||||
return jdbcReloadSubtableInterval;
|
||||
}
|
||||
|
||||
@Config("jdbc-reload-subtable-interval")
|
||||
public JdbcSubTableConfig setJdbcReloadSubtableInterval(Duration jdbcReloadSubtableInterval)
|
||||
{
|
||||
this.jdbcReloadSubtableInterval = jdbcReloadSubtableInterval;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getJdbcSubTableEnable()
|
||||
{
|
||||
return jdbcSubTableEnable;
|
||||
}
|
||||
|
||||
@Config("jdbc-sub-table-enable")
|
||||
public JdbcSubTableConfig setJdbcSubTableEnable(boolean jdbcSubTableEnable)
|
||||
{
|
||||
this.jdbcSubTableEnable = jdbcSubTableEnable;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,297 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.Maps.fromProperties;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Driver;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.JdbcPartition;
|
||||
import com.facebook.presto.plugin.jdbc.JdbcSplit;
|
||||
import com.facebook.presto.plugin.jdbc.JdbcTableHandle;
|
||||
import com.facebook.presto.plugin.jdbc.util.JdbcUtil;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.HostAddress;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
public class JdbcSubTableManager
|
||||
{
|
||||
private static final Logger log = Logger.get(JdbcSubTableManager.class);
|
||||
|
||||
protected final String connectorId;
|
||||
protected final String identifierQuote;
|
||||
protected final Driver driver;
|
||||
protected final String defaultConnectionUrl;
|
||||
protected final Properties defaultConnectionProperties;
|
||||
protected final String jdbcSubTableConnectionUrl;
|
||||
protected final Properties jdbcSubTableConnectionProperties;
|
||||
|
||||
private JdbcLoadTread loadTread;
|
||||
|
||||
public JdbcSubTableManager(String connectorId,
|
||||
String identifierQuote,
|
||||
Driver driver,
|
||||
String defaultConnectionUrl,
|
||||
Properties defaultConnectionProperties,
|
||||
JdbcSubTableConfig config)
|
||||
{
|
||||
this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
|
||||
this.identifierQuote = checkNotNull(identifierQuote, "identifierQuote is null");
|
||||
this.driver = checkNotNull(driver, "driver is null");
|
||||
this.defaultConnectionUrl = defaultConnectionUrl;
|
||||
this.defaultConnectionProperties = defaultConnectionProperties;
|
||||
|
||||
checkNotNull(config, "config is null");
|
||||
jdbcSubTableConnectionUrl = config.getJdbcSubTableConnectionURL();
|
||||
jdbcSubTableConnectionProperties = new Properties();
|
||||
jdbcSubTableConnectionProperties.setProperty("user", config.getJdbcSubTableConnectionUser());
|
||||
jdbcSubTableConnectionProperties.setProperty("password", config.getJdbcSubTableConnectionPassword());
|
||||
|
||||
if (config.getJdbcSubTableEnable()) {
|
||||
try {
|
||||
loadTread = new JdbcLoadTread(config.getJdbcSubTableConnectionURL(), jdbcSubTableConnectionProperties,
|
||||
connectorId, config.getJdbcReloadSubtableInterval());
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error("Init JdbcLoadTread error", e);
|
||||
}
|
||||
Thread loadTableThread = new Thread(loadTread);
|
||||
loadTableThread.setName("LoadTableThread");
|
||||
loadTableThread.setDaemon(true);
|
||||
loadTableThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table splits
|
||||
* @param jdbcPartition
|
||||
* @return
|
||||
*/
|
||||
public ConnectorSplitSource getTableSplits(JdbcPartition jdbcPartition)
|
||||
{
|
||||
JdbcTableHandle jdbcTableHandle = jdbcPartition.getJdbcTableHandle();
|
||||
List<JdbcSplit> jdbcSplitsList = new ArrayList<JdbcSplit>();
|
||||
String schemaName = getSchemaName(jdbcTableHandle);
|
||||
PdboTableInfo key = new PdboTableInfo(connectorId, schemaName, jdbcTableHandle.getTableName().toLowerCase(ENGLISH));
|
||||
ArrayList<PdboSplit> pdboSplits = loadTread.getPdboTableInfo().get(key);
|
||||
if (JdbcUtil.checkListNullOrEmpty(pdboSplits)) {
|
||||
if (pdboSplits == null) {
|
||||
pdboSplits = new ArrayList<PdboSplit>();
|
||||
}
|
||||
PdboSplit config = new PdboSplit();
|
||||
config.setConnectionUrl(defaultConnectionUrl);
|
||||
config.setConnectorId(jdbcTableHandle.getCatalogName());
|
||||
config.setSchemaName(jdbcTableHandle.getSchemaName());
|
||||
config.setTableName(jdbcTableHandle.getTableName());
|
||||
config.setRemotelyAccessible("Y");
|
||||
config.setPrestoTableName(jdbcTableHandle.getTableName());
|
||||
pdboSplits.add(config);
|
||||
}
|
||||
long timeStamp = System.nanoTime();
|
||||
if (pdboSplits.get(0).isCalcStepEnable()) {
|
||||
jdbcSplitsList = getTableSplitsFromPdboLog(connectorId, schemaName, jdbcTableHandle.getTableName().toLowerCase(ENGLISH), jdbcPartition, timeStamp);
|
||||
}
|
||||
if (JdbcUtil.checkListNullOrEmpty(jdbcSplitsList)) {
|
||||
for (PdboSplit config : pdboSplits) {
|
||||
constructJdbcSplits(jdbcPartition, jdbcSplitsList, config, timeStamp);
|
||||
}
|
||||
}
|
||||
return new PdboSplitSource(connectorId, jdbcSplitsList, pdboSplits.get(0).getControlScanConcurrencyEnabled(), pdboSplits.get(0).getScanConcurrencyCount());
|
||||
}
|
||||
|
||||
private String getSchemaName(JdbcTableHandle jdbcTableHandle)
|
||||
{
|
||||
String schemaName = "";
|
||||
if (defaultConnectionUrl.indexOf("mysql") != -1) {
|
||||
schemaName = jdbcTableHandle.getCatalogName().toLowerCase(ENGLISH);
|
||||
}
|
||||
else {
|
||||
schemaName = jdbcTableHandle.getSchemaName().toLowerCase(ENGLISH);
|
||||
}
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
private void constructJdbcSplits(JdbcPartition jdbcPartition,
|
||||
List<JdbcSplit> splits, PdboSplit config, long timeStamp)
|
||||
{
|
||||
List<HostAddress> addresses = getSplitHost(config.getPrestoWorkHost());
|
||||
Properties connectionProperties = resetConnectionProperties(config.getUsername(), config.getPassword());
|
||||
int scanNodes = config.getScanNodes() <= 0 ? 1 : config.getScanNodes();
|
||||
if (scanNodes == 1 || (scanNodes > 1 && isNullOrEmpty(config.getSplitField()))) {
|
||||
addJdbcSplit(jdbcPartition, splits, addresses, new String[]{"", "", ""}, connectionProperties, timeStamp, scanNodes, config);
|
||||
}
|
||||
else {
|
||||
splitTable(jdbcPartition, splits, config, addresses, connectionProperties, scanNodes, timeStamp);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Splitting table by field or limit
|
||||
*/
|
||||
private void splitTable(JdbcPartition jdbcPartition,
|
||||
List<JdbcSplit> splits, PdboSplit config,
|
||||
List<HostAddress> addresses, Properties connectionProperties,
|
||||
int scanNodes, long timeStamp)
|
||||
{
|
||||
long tableTotalRecords = 0L;
|
||||
Long[] autoIncrementFieldMinAndMaxValue = new Long[2];
|
||||
autoIncrementFieldMinAndMaxValue = getSplitFieldMinAndMaxValue(config, connectionProperties);
|
||||
tableTotalRecords = autoIncrementFieldMinAndMaxValue[0] - autoIncrementFieldMinAndMaxValue[1];
|
||||
long targetChunkSize = (long) Math.ceil(tableTotalRecords * 1.0 / scanNodes);
|
||||
long chunkOffset = 0L;
|
||||
long autoIncrementOffset = 0L;
|
||||
while (chunkOffset < tableTotalRecords) {
|
||||
long chunkLength = Math.min(targetChunkSize, tableTotalRecords - chunkOffset);
|
||||
if (chunkOffset == 0) {
|
||||
autoIncrementOffset = autoIncrementFieldMinAndMaxValue[1] - 1;
|
||||
}
|
||||
String[] splitInfo = getSplitInfo(chunkOffset, autoIncrementOffset,
|
||||
autoIncrementFieldMinAndMaxValue, chunkLength, tableTotalRecords, config.getSplitField());
|
||||
addJdbcSplit(jdbcPartition, splits, addresses, splitInfo, connectionProperties, timeStamp, scanNodes, config);
|
||||
chunkOffset += chunkLength;
|
||||
autoIncrementOffset += chunkLength;
|
||||
}
|
||||
fillLastRecord(jdbcPartition, connectionProperties, splits, scanNodes, timeStamp, config.getFieldMaxValue(), config);
|
||||
}
|
||||
|
||||
/**
|
||||
* If the table split by field,the filter conditions will follow like this :
|
||||
* field > offset and field <= offset + chunkLength.
|
||||
* @return splitInfo[0] : splitPart; splitInfo[1] : beginIndex; splitInfo[2] : endIndex;
|
||||
*/
|
||||
private String[] getSplitInfo(long chunkOffset, long autoIncrementOffset,
|
||||
Long[] autoIncrementFieldMinAndMaxValue, long chunkLength, long tableTotalRecords, String splitField)
|
||||
{
|
||||
String[] splitInfo = new String[3];
|
||||
String splitPart = "";
|
||||
splitInfo[1] = String.valueOf(autoIncrementOffset);
|
||||
splitPart = splitField + " > " + autoIncrementOffset + " and " + splitField + " <= ";
|
||||
if ((chunkOffset + chunkLength) == tableTotalRecords) {
|
||||
splitPart += autoIncrementFieldMinAndMaxValue[0];
|
||||
splitInfo[2] = String.valueOf(autoIncrementFieldMinAndMaxValue[0]);
|
||||
}
|
||||
else {
|
||||
splitPart += (autoIncrementOffset + chunkLength);
|
||||
splitInfo[2] = String.valueOf(autoIncrementOffset + chunkLength);
|
||||
}
|
||||
splitInfo[0] = splitPart;
|
||||
return splitInfo;
|
||||
}
|
||||
|
||||
private void addJdbcSplit(JdbcPartition jdbcPartition,
|
||||
List<JdbcSplit> builder, List<HostAddress> addresses, String[] splitInfo,
|
||||
Properties connectionProperties, long timeStamp, int scanNodes, PdboSplit config)
|
||||
{
|
||||
builder.add(new JdbcSplit(connectorId, config.getConnectorId(), config.getSchemaName(), config.getTableName(),
|
||||
config.getConnectionUrl(), fromProperties(connectionProperties), jdbcPartition.getTupleDomain(),
|
||||
splitInfo[0], addresses, config.getRemotelyAccessible(), config.getPrestoTableName(),
|
||||
config.getSplitField(), splitInfo[1], splitInfo[2], timeStamp, scanNodes, config.isCalcStepEnable(), config.getDbHost()));
|
||||
}
|
||||
|
||||
protected Long[] getSplitFieldMinAndMaxValue(PdboSplit conf, Properties connectionProperties)
|
||||
{
|
||||
Long[] value = new Long[2];
|
||||
if (conf.getFieldMaxValue() > 0) {
|
||||
value[0] = conf.getFieldMaxValue();
|
||||
value[1] = conf.getFieldMinValue();
|
||||
return value;
|
||||
}
|
||||
String sql = "SELECT MAX(" + conf.getSplitField() + "),MIN(" + conf.getSplitField() + ") FROM "
|
||||
+ JdbcUtil.getTableName(identifierQuote, conf.getConnectorId(), conf.getSchemaName(), conf.getTableName());
|
||||
Connection connection = null;
|
||||
Statement stat = null;
|
||||
ResultSet rs = null;
|
||||
try {
|
||||
connection = driver.connect(conf.getConnectionUrl(), connectionProperties);
|
||||
stat = connection.createStatement();
|
||||
rs = stat.executeQuery(sql.toString());
|
||||
while (rs.next()) {
|
||||
value[0] = rs.getLong(1);
|
||||
value[1] = rs.getLong(2);
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error("SQL : " + sql + ",getSplitFieldMinAndMaxValue error : " + e.getMessage());
|
||||
return null;
|
||||
}
|
||||
finally {
|
||||
JdbcUtil.closeJdbcConnection(connection, stat, rs);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public List<JdbcSplit> getTableSplitsFromPdboLog(String catalogName, String schemaName, String tableName,
|
||||
JdbcPartition jdbcPartition, long timeStamp)
|
||||
{
|
||||
List<JdbcSplit> splits = new ArrayList<>();
|
||||
List<PdboSplit> pdboLogs = loadTread.getPDBOLogs(catalogName, schemaName, tableName);
|
||||
if (JdbcUtil.checkListNullOrEmpty(pdboLogs)) {
|
||||
return splits;
|
||||
}
|
||||
int scanNodes = pdboLogs.size();
|
||||
for (PdboSplit table : pdboLogs) {
|
||||
table.setConnectorId(null);
|
||||
List<HostAddress> addresses = getSplitHost(table.getPrestoWorkHost());
|
||||
Properties connectionProperties = resetConnectionProperties(table.getUsername(), table.getPassword());
|
||||
String splitPart = table.getSplitField() + " > " + table.getBeginIndex() + " and "
|
||||
+ table.getSplitField() + " <= " + table.getEndIndex();
|
||||
addJdbcSplit(jdbcPartition, splits, addresses,
|
||||
new String[]{splitPart, String.valueOf(table.getBeginIndex()), String.valueOf(table.getEndIndex())},
|
||||
connectionProperties, timeStamp, scanNodes, table);
|
||||
}
|
||||
PdboSplit lastRecord = pdboLogs.get(scanNodes - 1);
|
||||
fillLastRecord(jdbcPartition, resetConnectionProperties(lastRecord.getUsername(), lastRecord.getPassword()),
|
||||
splits, scanNodes, timeStamp, lastRecord.getEndIndex(), lastRecord);
|
||||
return splits;
|
||||
}
|
||||
|
||||
private void fillLastRecord(JdbcPartition jdbcPartition, Properties connectionProperties,
|
||||
List<JdbcSplit> splits, int scanNodes, long timeStamp, long endIndex, PdboSplit config)
|
||||
{
|
||||
String splitPart = config.getSplitField() + " > " + endIndex;
|
||||
addJdbcSplit(jdbcPartition, splits, getSplitHost(config.getPrestoWorkHost()),
|
||||
new String[]{splitPart, "", ""}, connectionProperties, timeStamp, scanNodes, config);
|
||||
}
|
||||
|
||||
private Properties resetConnectionProperties(String username, String password)
|
||||
{
|
||||
Properties connectionProperties = (Properties) defaultConnectionProperties.clone();
|
||||
if (!isNullOrEmpty(username) && !isNullOrEmpty(password)) {
|
||||
connectionProperties.setProperty("user", username);
|
||||
connectionProperties.setProperty("password", password);
|
||||
}
|
||||
return connectionProperties;
|
||||
}
|
||||
|
||||
private List<HostAddress> getSplitHost(String host)
|
||||
{
|
||||
return isNullOrEmpty(host) ? ImmutableList.of() : ImmutableList.of(HostAddress.fromString(host));
|
||||
}
|
||||
|
||||
public void commitPdboLogs(JdbcSplit split, long rowCount)
|
||||
{
|
||||
loadTread.commitPdboLogs(split, rowCount);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,284 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.util.JdbcUtil;
|
||||
|
||||
public class PdboSplit
|
||||
{
|
||||
private String connectorId;
|
||||
private String schemaName;
|
||||
private String tableName;
|
||||
private Long rows;
|
||||
private Long beginIndex;
|
||||
private Long endIndex;
|
||||
private String recordFlag;
|
||||
private String dbHost;
|
||||
private String connectionUrl;
|
||||
private String prestoWorkHost;
|
||||
private String remotelyAccessible;
|
||||
private String splitField;
|
||||
private String username;
|
||||
private String password;
|
||||
private Long timeStamp;
|
||||
private int scanNodes;
|
||||
private long fieldMaxValue;
|
||||
private long fieldMinValue;
|
||||
private String prestoTableName;
|
||||
private String calcStepEnable;
|
||||
private String controlScanConcurrencyEnabled;
|
||||
private int scanConcurrencyCount;
|
||||
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
public PdboSplit setConnectorId(String connectorId)
|
||||
{
|
||||
this.connectorId = connectorId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getSchemaName()
|
||||
{
|
||||
return schemaName;
|
||||
}
|
||||
|
||||
public PdboSplit setSchemaName(String schemaName)
|
||||
{
|
||||
this.schemaName = schemaName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getTableName()
|
||||
{
|
||||
return tableName;
|
||||
}
|
||||
|
||||
public PdboSplit setTableName(String tableName)
|
||||
{
|
||||
this.tableName = tableName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Long getRows()
|
||||
{
|
||||
return rows;
|
||||
}
|
||||
|
||||
public PdboSplit setRows(Long rows)
|
||||
{
|
||||
this.rows = rows;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Long getBeginIndex()
|
||||
{
|
||||
return beginIndex;
|
||||
}
|
||||
|
||||
public PdboSplit setBeginIndex(Long beginIndex)
|
||||
{
|
||||
this.beginIndex = beginIndex;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Long getEndIndex()
|
||||
{
|
||||
return endIndex;
|
||||
}
|
||||
|
||||
public PdboSplit setEndIndex(Long endIndex)
|
||||
{
|
||||
this.endIndex = endIndex;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getRecordFlag()
|
||||
{
|
||||
return recordFlag;
|
||||
}
|
||||
|
||||
public PdboSplit setRecordFlag(String recordFlag)
|
||||
{
|
||||
this.recordFlag = recordFlag;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getDbHost()
|
||||
{
|
||||
return (dbHost == null || dbHost.length() == 0) ? "default" : dbHost;
|
||||
}
|
||||
|
||||
public PdboSplit setDbHost(String dbHost)
|
||||
{
|
||||
this.dbHost = dbHost;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getConnectionUrl()
|
||||
{
|
||||
return connectionUrl;
|
||||
}
|
||||
|
||||
public PdboSplit setConnectionUrl(String connectionUrl)
|
||||
{
|
||||
this.connectionUrl = connectionUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getPrestoWorkHost()
|
||||
{
|
||||
return prestoWorkHost;
|
||||
}
|
||||
|
||||
public PdboSplit setPrestoWorkHost(String prestoWorkHost)
|
||||
{
|
||||
this.prestoWorkHost = prestoWorkHost;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getRemotelyAccessible()
|
||||
{
|
||||
return JdbcUtil.converStringToBoolean(remotelyAccessible, false);
|
||||
}
|
||||
|
||||
public PdboSplit setRemotelyAccessible(String remotelyAccessible)
|
||||
{
|
||||
this.remotelyAccessible = remotelyAccessible;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getUsername()
|
||||
{
|
||||
return username;
|
||||
}
|
||||
|
||||
public PdboSplit setUsername(String username)
|
||||
{
|
||||
this.username = username;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getPassword()
|
||||
{
|
||||
return password;
|
||||
}
|
||||
|
||||
public PdboSplit setPassword(String password)
|
||||
{
|
||||
this.password = password;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getSplitField()
|
||||
{
|
||||
return splitField;
|
||||
}
|
||||
|
||||
public PdboSplit setSplitField(String splitField)
|
||||
{
|
||||
this.splitField = splitField;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Long getTimeStamp()
|
||||
{
|
||||
return timeStamp;
|
||||
}
|
||||
|
||||
public PdboSplit setTimeStamp(Long timeStamp)
|
||||
{
|
||||
this.timeStamp = timeStamp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getScanNodes()
|
||||
{
|
||||
return scanNodes;
|
||||
}
|
||||
|
||||
public PdboSplit setScanNodes(int scanNodes)
|
||||
{
|
||||
this.scanNodes = scanNodes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long getFieldMaxValue()
|
||||
{
|
||||
return fieldMaxValue;
|
||||
}
|
||||
|
||||
public PdboSplit setFieldMaxValue(long fieldMaxValue)
|
||||
{
|
||||
this.fieldMaxValue = fieldMaxValue;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long getFieldMinValue()
|
||||
{
|
||||
return fieldMinValue;
|
||||
}
|
||||
|
||||
public PdboSplit setFieldMinValue(long fieldMinValue)
|
||||
{
|
||||
this.fieldMinValue = fieldMinValue;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getPrestoTableName()
|
||||
{
|
||||
return prestoTableName;
|
||||
}
|
||||
|
||||
public void setPrestoTableName(String prestoTableName)
|
||||
{
|
||||
this.prestoTableName = prestoTableName;
|
||||
}
|
||||
|
||||
public boolean isCalcStepEnable()
|
||||
{
|
||||
return JdbcUtil.converStringToBoolean(calcStepEnable, false);
|
||||
}
|
||||
|
||||
public PdboSplit setCalcStepEnable(String calcStepEnable)
|
||||
{
|
||||
this.calcStepEnable = calcStepEnable;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getControlScanConcurrencyEnabled()
|
||||
{
|
||||
return JdbcUtil.converStringToBoolean(controlScanConcurrencyEnabled, false);
|
||||
}
|
||||
|
||||
public PdboSplit setControlScanConcurrencyEnabled(
|
||||
String controlScanConcurrencyEnabled)
|
||||
{
|
||||
this.controlScanConcurrencyEnabled = controlScanConcurrencyEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getScanConcurrencyCount()
|
||||
{
|
||||
return scanConcurrencyCount;
|
||||
}
|
||||
|
||||
public PdboSplit setScanConcurrencyCount(int scanConcurrencyCount)
|
||||
{
|
||||
this.scanConcurrencyCount = scanConcurrencyCount;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.JdbcSplit;
|
||||
import com.facebook.presto.spi.ConnectorSplit;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import static java.util.concurrent.CompletableFuture.completedFuture;
|
||||
|
||||
public class PdboSplitSource
|
||||
implements ConnectorSplitSource
|
||||
{
|
||||
private final String dataSourceName;
|
||||
private final List<ConnectorSplit> splits;
|
||||
private int offset;
|
||||
private final boolean controlScanConcurrencyEnabled;
|
||||
private final int scanConcurrencyCount;
|
||||
|
||||
public PdboSplitSource(String dataSourceName, Iterable<? extends ConnectorSplit> splits, boolean controlScanConcurrencyEnabled, int scanConcurrencyCount)
|
||||
{
|
||||
this.dataSourceName = dataSourceName;
|
||||
if (splits == null) {
|
||||
throw new NullPointerException("splits is null");
|
||||
}
|
||||
List<ConnectorSplit> splitsList = new ArrayList<>();
|
||||
sortSplitByDbHost(splits, controlScanConcurrencyEnabled, splitsList);
|
||||
this.splits = Collections.unmodifiableList(splitsList);
|
||||
this.controlScanConcurrencyEnabled = controlScanConcurrencyEnabled;
|
||||
this.scanConcurrencyCount = scanConcurrencyCount;
|
||||
}
|
||||
|
||||
private void sortSplitByDbHost(Iterable<? extends ConnectorSplit> splits,
|
||||
boolean controlScanConcurrencyEnabled,
|
||||
List<ConnectorSplit> splitsList)
|
||||
{
|
||||
Map<String, List<ConnectorSplit>> map = new HashMap<String, List<ConnectorSplit>>();
|
||||
if (controlScanConcurrencyEnabled) {
|
||||
int splitSize = 0;
|
||||
for (ConnectorSplit split : splits) {
|
||||
if (split instanceof JdbcSplit) {
|
||||
JdbcSplit jdbcSplit = (JdbcSplit) split;
|
||||
List<ConnectorSplit> list = map.get(jdbcSplit.getDbHost());
|
||||
if (list == null) {
|
||||
list = new ArrayList<>();
|
||||
}
|
||||
list.add(split);
|
||||
map.put(jdbcSplit.getDbHost(), list);
|
||||
splitSize++;
|
||||
}
|
||||
}
|
||||
int loopCount = 0;
|
||||
while (loopCount < splitSize) {
|
||||
for (String dbHost : map.keySet()) {
|
||||
List<ConnectorSplit> list = map.get(dbHost);
|
||||
if (list == null || list.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
splitsList.add(list.get(0));
|
||||
loopCount++;
|
||||
list.remove(0);
|
||||
map.put(dbHost, list);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (ConnectorSplit split : splits) {
|
||||
splitsList.add(split);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDataSourceName()
|
||||
{
|
||||
return dataSourceName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<List<ConnectorSplit>> getNextBatch(int maxSize)
|
||||
{
|
||||
int remainingSplits = splits.size() - offset;
|
||||
int size = Math.min(remainingSplits, maxSize);
|
||||
List<ConnectorSplit> results = splits.subList(offset, offset + size);
|
||||
offset += size;
|
||||
return completedFuture(results);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isFinished()
|
||||
{
|
||||
return offset >= splits.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isControlScanConcurrencyEnabled()
|
||||
{
|
||||
return controlScanConcurrencyEnabled;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getScanConcurrencyCount()
|
||||
{
|
||||
return scanConcurrencyCount;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,141 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.subtable;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.util.JdbcUtil;
|
||||
|
||||
public class PdboTableInfo
|
||||
{
|
||||
enum DBType
|
||||
{
|
||||
MYSQL,
|
||||
SQLSERVER,
|
||||
ORACLE
|
||||
}
|
||||
private String tableId;
|
||||
private String dbType;
|
||||
private String connectorId;
|
||||
private String prestoSchema;
|
||||
private String prestoTable;
|
||||
private String calcStepEnable;
|
||||
private String controlScanConcurrencyEnabled;
|
||||
private int scanConcurrencyCount;
|
||||
|
||||
public PdboTableInfo(String connectorId, String prestoSchema, String prestoTable)
|
||||
{
|
||||
this.connectorId = connectorId;
|
||||
this.prestoSchema = prestoSchema;
|
||||
this.prestoTable = prestoTable;
|
||||
}
|
||||
public String getTableId()
|
||||
{
|
||||
return tableId;
|
||||
}
|
||||
|
||||
public void setTableId(String tableId)
|
||||
{
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
public String getDbType()
|
||||
{
|
||||
return dbType;
|
||||
}
|
||||
|
||||
public void setDbType(String dbType)
|
||||
{
|
||||
this.dbType = dbType;
|
||||
}
|
||||
|
||||
public String getConnectorId()
|
||||
{
|
||||
return connectorId;
|
||||
}
|
||||
|
||||
public void setConnectorId(String connectorId)
|
||||
{
|
||||
this.connectorId = connectorId;
|
||||
}
|
||||
|
||||
public String getPrestoSchema()
|
||||
{
|
||||
return prestoSchema;
|
||||
}
|
||||
|
||||
public void setPrestoSchema(String prestoSchema)
|
||||
{
|
||||
this.prestoSchema = prestoSchema;
|
||||
}
|
||||
|
||||
public String getPrestoTable()
|
||||
{
|
||||
return prestoTable;
|
||||
}
|
||||
|
||||
public void setPrestoTable(String prestoTable)
|
||||
{
|
||||
this.prestoTable = prestoTable;
|
||||
}
|
||||
|
||||
public boolean isCalcStepEnable()
|
||||
{
|
||||
return JdbcUtil.converStringToBoolean(calcStepEnable, false);
|
||||
}
|
||||
|
||||
public void setCalcStepEnable(String calcStepEnable)
|
||||
{
|
||||
this.calcStepEnable = calcStepEnable;
|
||||
}
|
||||
|
||||
public boolean getControlScanConcurrencyEnabled()
|
||||
{
|
||||
return JdbcUtil.converStringToBoolean(controlScanConcurrencyEnabled, false);
|
||||
}
|
||||
|
||||
public void setControlScanConcurrencyEnabled(
|
||||
String controlScanConcurrencyEnabled)
|
||||
{
|
||||
this.controlScanConcurrencyEnabled = controlScanConcurrencyEnabled;
|
||||
}
|
||||
|
||||
public int getScanConcurrencyCount()
|
||||
{
|
||||
return scanConcurrencyCount;
|
||||
}
|
||||
|
||||
public void setScanConcurrencyCount(int scanConcurrencyCount)
|
||||
{
|
||||
this.scanConcurrencyCount = scanConcurrencyCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(getConnectorId(), getPrestoSchema(), getPrestoTable());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return getConnectorId() + "-" + getPrestoSchema() + "-" + getPrestoTable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
return this.hashCode() == obj.hashCode();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.util;
|
||||
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Types;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.BooleanType.BOOLEAN;
|
||||
import static com.facebook.presto.spi.type.DateType.DATE;
|
||||
import static com.facebook.presto.spi.type.DoubleType.DOUBLE;
|
||||
import static com.facebook.presto.spi.type.TimeType.TIME;
|
||||
import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP;
|
||||
import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
|
||||
public final class JdbcUtil
|
||||
{
|
||||
private static final Logger log = Logger.get(JdbcUtil.class);
|
||||
|
||||
private JdbcUtil()
|
||||
{
|
||||
}
|
||||
/**
|
||||
* Get table name : catalog.schema.table
|
||||
* @param catalog
|
||||
* @param schema
|
||||
* @param table
|
||||
* @return
|
||||
*/
|
||||
public static String getTableName(String identifierQuote, String catalog, String schema, String table)
|
||||
{
|
||||
StringBuilder sql = new StringBuilder();
|
||||
if (!isNullOrEmpty(schema)) {
|
||||
sql.append(quoted(identifierQuote, schema)).append('.');
|
||||
}
|
||||
else {
|
||||
if (!isNullOrEmpty(catalog)) {
|
||||
sql.append(quoted(identifierQuote, catalog)).append('.');
|
||||
}
|
||||
}
|
||||
sql.append(quoted(identifierQuote, table));
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private static String quoted(String identifierQuote, String name)
|
||||
{
|
||||
name = name.replace(identifierQuote, identifierQuote + identifierQuote);
|
||||
return identifierQuote + name + identifierQuote;
|
||||
}
|
||||
|
||||
/**
|
||||
* close JDBC connection
|
||||
* @param connection
|
||||
* @param stat
|
||||
* @param rs
|
||||
*/
|
||||
public static void closeJdbcConnection(Connection connection, Statement stat, ResultSet rs)
|
||||
{
|
||||
try {
|
||||
if (rs != null) {
|
||||
rs.close();
|
||||
}
|
||||
if (stat != null) {
|
||||
stat.close();
|
||||
}
|
||||
if (connection != null) {
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
log.error("close connection error : " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static Type toPrestoType(int jdbcType)
|
||||
{
|
||||
switch (jdbcType) {
|
||||
case Types.BIT:
|
||||
case Types.BOOLEAN:
|
||||
return BOOLEAN;
|
||||
case Types.TINYINT:
|
||||
case Types.SMALLINT:
|
||||
case Types.INTEGER:
|
||||
case Types.BIGINT:
|
||||
return BIGINT;
|
||||
case Types.FLOAT:
|
||||
case Types.REAL:
|
||||
case Types.DOUBLE:
|
||||
case Types.NUMERIC:
|
||||
case Types.DECIMAL:
|
||||
return DOUBLE;
|
||||
case Types.CHAR:
|
||||
case Types.NCHAR:
|
||||
case Types.VARCHAR:
|
||||
case Types.NVARCHAR:
|
||||
case Types.LONGVARCHAR:
|
||||
case Types.LONGNVARCHAR:
|
||||
return VARCHAR;
|
||||
case Types.BINARY:
|
||||
case Types.VARBINARY:
|
||||
case Types.LONGVARBINARY:
|
||||
return VARBINARY;
|
||||
case Types.DATE:
|
||||
return DATE;
|
||||
case Types.TIME:
|
||||
return TIME;
|
||||
case Types.TIMESTAMP:
|
||||
return TIMESTAMP;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Properties toProperties(Map<String, String> map)
|
||||
{
|
||||
Properties properties = new Properties();
|
||||
for (Map.Entry<String, String> entry : map.entrySet()) {
|
||||
properties.setProperty(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
public static <E> boolean checkListNullOrEmpty(List<E> list)
|
||||
{
|
||||
return list == null || list.isEmpty();
|
||||
}
|
||||
|
||||
public static boolean converStringToBoolean(String fieldValue, boolean defaultValue)
|
||||
{
|
||||
if (fieldValue == null || "".equals(fieldValue)) {
|
||||
return defaultValue;
|
||||
}
|
||||
else if ("Y".equals(fieldValue)
|
||||
|| "y".equals(fieldValue)) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static String Base64Encode(String str)
|
||||
{
|
||||
str = str == null ? "" : str;
|
||||
return new String(Base64.getEncoder().encode(str.getBytes()), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static String Base64Decode(String str)
|
||||
{
|
||||
str = str == null ? "" : str;
|
||||
return new String(Base64.getDecoder().decode(str.getBytes()), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc.util;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.JdbcSplit;
|
||||
|
||||
public final class PdboMetadata
|
||||
{
|
||||
public static final String PDBO_DATABASE = "route_schema";
|
||||
public static final String PDBO_TABLE = PDBO_DATABASE + ".pdbo_table";
|
||||
public static final String PDBO_ROUTE = PDBO_DATABASE + ".pdbo_route";
|
||||
public static final String PDBO_LOG = PDBO_DATABASE + ".pdbo_log";
|
||||
public static final String DB_INFO = PDBO_DATABASE + ".db_info";
|
||||
|
||||
private PdboMetadata()
|
||||
{
|
||||
}
|
||||
|
||||
public static String getPdboLogsSQL()
|
||||
{
|
||||
return "SELECT A.CONNECTORID,A.SCHEMANAME,A.TABLENAME,A.ROWS,A.BEGININDEX,A.ENDINDEX,B.DBTYPE,"
|
||||
+ " C.DBHOST,C.DBPORT,C.CONNECTION_PROPERTIES,C.PRESTO_WORK_HOST,C.REMOTELYACCESSIBLE,C.SPLITFIELD, "
|
||||
+ " C.SCANNODENUMBER,D.USERNAME,D.PASSWORD,A.CONTROL_SCAN_CONCURRENCY_ENABLED,A.SCAN_CONCURRENCY_COUNT "
|
||||
+ " FROM " + PDBO_LOG + " A INNER JOIN " + PDBO_TABLE + " B "
|
||||
+ " ON A.CONNECTORID = B.CONNECTORID AND A.SCHEMANAME = B.PRESTO_SCHEMA AND A.TABLENAME = B.PRESTO_TABLE "
|
||||
+ " INNER JOIN " + PDBO_ROUTE + " C ON B.TABLEID = C.TABLEID "
|
||||
+ " LEFT JOIN " + DB_INFO + " D ON C.UID = D.UID "
|
||||
+ " WHERE A.CONNECTORID = ? "
|
||||
+ " AND A.SCHEMANAME = ? "
|
||||
+ " AND A.TABLENAME = ? "
|
||||
+ " AND A.RECORDFLAG = 'finish' "
|
||||
+ " AND B.CALC_STEP_ENABLE = 'Y' "
|
||||
+ " ORDER BY A.BEGININDEX";
|
||||
}
|
||||
|
||||
public static String getPdboTableInfoSQL()
|
||||
{
|
||||
return "SELECT CONNECTORID,PRESTO_SCHEMA,PRESTO_TABLE,DBTYPE,CALC_STEP_ENABLE,CONTROL_SCAN_CONCURRENCY_ENABLED,SCAN_CONCURRENCY_COUNT,"
|
||||
+ " B.DBHOST,B.DBPORT,CONNECTION_PROPERTIES,SOURCE_SCHEMA,SOURCE_TABLE,SPLITFIELD,REMOTELYACCESSIBLE,PRESTO_WORK_HOST,SCANNODENUMBER,"
|
||||
+ " FIELDMAXVALUE,FIELDMINVALUE,USERNAME,PASSWORD"
|
||||
+ " FROM " + PDBO_TABLE + " A INNER JOIN " + PDBO_ROUTE + " B ON A.TABLEID = B.TABLEID"
|
||||
+ " LEFT JOIN " + DB_INFO + " C ON B.UID = C.UID"
|
||||
+ " WHERE CONNECTORID = ? ";
|
||||
}
|
||||
|
||||
public static String getInsertPdboLogSQL(JdbcSplit split, long rowCount, String connectorId)
|
||||
{
|
||||
return "INSERT INTO " + PDBO_LOG
|
||||
+ " (CONNECTORID,SCHEMANAME,TABLENAME,ROWS,BEGININDEX,ENDINDEX,RECORDFLAG,SCANNODES,TIMESTAMP) VALUES "
|
||||
+ "('" + connectorId + "',"
|
||||
+ "'" + split.getSchemaName() + "',"
|
||||
+ "'" + split.getTableName() + "',"
|
||||
+ rowCount + ","
|
||||
+ split.getBeginIndex() + ","
|
||||
+ split.getEndIndex() + ","
|
||||
+ "'new',"
|
||||
+ split.getScanNodes() + ","
|
||||
+ split.getTimeStamp() + ")";
|
||||
}
|
||||
|
||||
public static String getUpdatePdboHistoryLogSQL(JdbcSplit split, String connectorId)
|
||||
{
|
||||
return "UPDATE " + PDBO_LOG + " SET RECORDFLAG='runhistory' "
|
||||
+ "WHERE RECORDFLAG='new' AND CONNECTORID='" + connectorId
|
||||
+ "' AND SCHEMANAME='" + split.getSchemaName() + "' AND TABLENAME='"
|
||||
+ split.getTableName() + "'" + " AND timestamp < " + split.getTimeStamp();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.Session;
|
||||
import com.facebook.presto.tests.DistributedQueryRunner;
|
||||
import com.facebook.presto.tpch.TpchPlugin;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.airlift.tpch.TpchTable;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY;
|
||||
import static com.facebook.presto.tests.QueryAssertions.copyTpchTables;
|
||||
import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME;
|
||||
import static io.airlift.testing.Closeables.closeAllSuppress;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public final class JdbcQueryRunner
|
||||
{
|
||||
private JdbcQueryRunner()
|
||||
{
|
||||
}
|
||||
|
||||
private static final String TPCH_SCHEMA = "tpch";
|
||||
|
||||
public static DistributedQueryRunner createJdbcQueryRunner(TpchTable<?>... tables)
|
||||
throws Exception
|
||||
{
|
||||
return createJdbcQueryRunner(ImmutableList.copyOf(tables));
|
||||
}
|
||||
|
||||
public static DistributedQueryRunner createJdbcQueryRunner(Iterable<TpchTable<?>> tables)
|
||||
throws Exception
|
||||
{
|
||||
DistributedQueryRunner queryRunner = null;
|
||||
try {
|
||||
queryRunner = new DistributedQueryRunner(createSession(), 3);
|
||||
|
||||
queryRunner.installPlugin(new TpchPlugin());
|
||||
queryRunner.createCatalog("tpch", "tpch");
|
||||
|
||||
Map<String, String> properties = TestingH2JdbcModule.createProperties();
|
||||
createSchema(properties, "tpch");
|
||||
|
||||
queryRunner.installPlugin(new JdbcPlugin("base-jdbc", new TestingH2JdbcModule()));
|
||||
queryRunner.createCatalog("jdbc", "base-jdbc", properties);
|
||||
|
||||
copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables);
|
||||
|
||||
return queryRunner;
|
||||
}
|
||||
catch (Throwable e) {
|
||||
closeAllSuppress(e, queryRunner);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private static void createSchema(Map<String, String> properties, String schema)
|
||||
throws SQLException
|
||||
{
|
||||
try (Connection connection = DriverManager.getConnection(properties.get("connection-url"));
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("CREATE SCHEMA " + schema);
|
||||
}
|
||||
}
|
||||
|
||||
public static Session createSession()
|
||||
{
|
||||
return Session.builder()
|
||||
.setUser("user")
|
||||
.setSource("test")
|
||||
.setCatalog("jdbc")
|
||||
.setSchema(TPCH_SCHEMA)
|
||||
.setTimeZoneKey(UTC_KEY)
|
||||
.setLocale(ENGLISH)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.type.StandardTypes;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.fasterxml.jackson.databind.DeserializationContext;
|
||||
import com.fasterxml.jackson.databind.JsonDeserializer;
|
||||
import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.airlift.json.JsonCodec;
|
||||
import io.airlift.json.JsonCodecFactory;
|
||||
import io.airlift.json.ObjectMapperProvider;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
final class MetadataUtil
|
||||
{
|
||||
private MetadataUtil() {}
|
||||
|
||||
public static final JsonCodec<JdbcColumnHandle> COLUMN_CODEC;
|
||||
public static final JsonCodec<JdbcTableHandle> TABLE_CODEC;
|
||||
public static final JsonCodec<JdbcOutputTableHandle> OUTPUT_TABLE_CODEC;
|
||||
|
||||
static {
|
||||
ObjectMapperProvider provider = new ObjectMapperProvider();
|
||||
provider.setJsonDeserializers(ImmutableMap.<Class<?>, JsonDeserializer<?>>of(Type.class, new TestingTypeDeserializer()));
|
||||
JsonCodecFactory codecFactory = new JsonCodecFactory(provider);
|
||||
COLUMN_CODEC = codecFactory.jsonCodec(JdbcColumnHandle.class);
|
||||
TABLE_CODEC = codecFactory.jsonCodec(JdbcTableHandle.class);
|
||||
OUTPUT_TABLE_CODEC = codecFactory.jsonCodec(JdbcOutputTableHandle.class);
|
||||
}
|
||||
|
||||
public static final class TestingTypeDeserializer
|
||||
extends FromStringDeserializer<Type>
|
||||
{
|
||||
private final Map<String, Type> types = ImmutableMap.<String, Type>of(
|
||||
StandardTypes.BIGINT, BIGINT,
|
||||
StandardTypes.VARCHAR, VARCHAR);
|
||||
|
||||
public TestingTypeDeserializer()
|
||||
{
|
||||
super(Type.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Type _deserialize(String value, DeserializationContext context)
|
||||
{
|
||||
Type type = types.get(value.toLowerCase(ENGLISH));
|
||||
checkArgument(type != null, "Unknown type %s", value);
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
||||
public static <T> void assertJsonRoundTrip(JsonCodec<T> codec, T object)
|
||||
{
|
||||
String json = codec.toJson(object);
|
||||
T copy = codec.fromJson(json);
|
||||
assertEquals(copy, object);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.airlift.configuration.testing.ConfigAssertions;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class TestBaseJdbcConfig
|
||||
{
|
||||
@Test
|
||||
public void testDefaults()
|
||||
{
|
||||
ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(BaseJdbcConfig.class)
|
||||
.setConnectionUrl(null)
|
||||
.setConnectionUser(null)
|
||||
.setConnectionPassword(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExplicitPropertyMappings()
|
||||
{
|
||||
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
|
||||
.put("connection-url", "jdbc:h2:mem:config")
|
||||
.put("connection-user", "user")
|
||||
.put("connection-password", "password")
|
||||
.build();
|
||||
|
||||
BaseJdbcConfig expected = new BaseJdbcConfig()
|
||||
.setConnectionUrl("jdbc:h2:mem:config")
|
||||
.setConnectionUser("user")
|
||||
.setConnectionPassword("password");
|
||||
|
||||
ConfigAssertions.assertFullMapping(properties, expected);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.TestingDatabase.CONNECTOR_ID;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
@Test
|
||||
public class TestJdbcClient
|
||||
{
|
||||
private TestingDatabase database;
|
||||
private String catalogName;
|
||||
private JdbcClient jdbcClient;
|
||||
|
||||
@BeforeClass
|
||||
public void setUp()
|
||||
throws Exception
|
||||
{
|
||||
database = new TestingDatabase();
|
||||
catalogName = database.getConnection().getCatalog();
|
||||
jdbcClient = database.getJdbcClient();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public void tearDown()
|
||||
throws Exception
|
||||
{
|
||||
database.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetadata()
|
||||
throws Exception
|
||||
{
|
||||
assertTrue(jdbcClient.getSchemaNames().containsAll(ImmutableSet.of("example", "tpch")));
|
||||
assertEquals(jdbcClient.getTableNames("example"), ImmutableList.of(new SchemaTableName("example", "numbers")));
|
||||
assertEquals(jdbcClient.getTableNames("tpch"), ImmutableList.of(
|
||||
new SchemaTableName("tpch", "lineitem"),
|
||||
new SchemaTableName("tpch", "orders")));
|
||||
|
||||
SchemaTableName schemaTableName = new SchemaTableName("example", "numbers");
|
||||
JdbcTableHandle table = jdbcClient.getTableHandle(schemaTableName);
|
||||
assertNotNull(table, "table is null");
|
||||
assertEquals(table.getCatalogName(), catalogName.toUpperCase(ENGLISH));
|
||||
assertEquals(table.getSchemaName(), "EXAMPLE");
|
||||
assertEquals(table.getTableName(), "NUMBERS");
|
||||
assertEquals(table.getSchemaTableName(), schemaTableName);
|
||||
assertEquals(jdbcClient.getColumns(table), ImmutableList.of(
|
||||
new JdbcColumnHandle(CONNECTOR_ID, "TEXT", VARCHAR),
|
||||
new JdbcColumnHandle(CONNECTOR_ID, "VALUE", BIGINT)));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import io.airlift.testing.EquivalenceTester;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.COLUMN_CODEC;
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.assertJsonRoundTrip;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
|
||||
public class TestJdbcColumnHandle
|
||||
{
|
||||
@Test
|
||||
public void testJsonRoundTrip()
|
||||
{
|
||||
assertJsonRoundTrip(COLUMN_CODEC, new JdbcColumnHandle("connectorId", "columnName", VARCHAR));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEquivalence()
|
||||
{
|
||||
EquivalenceTester.equivalenceTester()
|
||||
.addEquivalentGroup(
|
||||
new JdbcColumnHandle("connectorId", "columnName", VARCHAR),
|
||||
new JdbcColumnHandle("connectorId", "columnName", VARCHAR),
|
||||
new JdbcColumnHandle("connectorId", "columnName", BIGINT),
|
||||
new JdbcColumnHandle("connectorId", "columnName", VARCHAR))
|
||||
.addEquivalentGroup(
|
||||
new JdbcColumnHandle("connectorIdX", "columnName", VARCHAR),
|
||||
new JdbcColumnHandle("connectorIdX", "columnName", VARCHAR),
|
||||
new JdbcColumnHandle("connectorIdX", "columnName", BIGINT),
|
||||
new JdbcColumnHandle("connectorIdX", "columnName", VARCHAR))
|
||||
.addEquivalentGroup(
|
||||
new JdbcColumnHandle("connectorId", "columnNameX", VARCHAR),
|
||||
new JdbcColumnHandle("connectorId", "columnNameX", VARCHAR),
|
||||
new JdbcColumnHandle("connectorId", "columnNameX", BIGINT),
|
||||
new JdbcColumnHandle("connectorId", "columnNameX", VARCHAR))
|
||||
.check();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class TestJdbcConnectorFactory
|
||||
{
|
||||
@Test
|
||||
public void test()
|
||||
throws Exception
|
||||
{
|
||||
JdbcConnectorFactory connectorFactory = new JdbcConnectorFactory(
|
||||
"test",
|
||||
new TestingH2JdbcModule(),
|
||||
ImmutableMap.<String, String>of(),
|
||||
getClass().getClassLoader());
|
||||
|
||||
connectorFactory.create("test", TestingH2JdbcModule.createProperties());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.tests.AbstractTestQueries;
|
||||
import io.airlift.tpch.TpchTable;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.JdbcQueryRunner.createJdbcQueryRunner;
|
||||
|
||||
public class TestJdbcDistributedQueries
|
||||
extends AbstractTestQueries
|
||||
{
|
||||
public TestJdbcDistributedQueries()
|
||||
throws Exception
|
||||
{
|
||||
super(createJdbcQueryRunner(TpchTable.getTables()));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.HostAddress;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.TestingDatabase.CONNECTOR_ID;
|
||||
import static org.testng.Assert.assertFalse;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
public class TestJdbcHandleResolver
|
||||
{
|
||||
private static final JdbcHandleResolver RESOLVER = new JdbcHandleResolver(new JdbcConnectorId(CONNECTOR_ID));
|
||||
|
||||
@Test
|
||||
public void testCanHandle()
|
||||
throws Exception
|
||||
{
|
||||
assertTrue(RESOLVER.canHandle(createTableHandle(CONNECTOR_ID)));
|
||||
assertFalse(RESOLVER.canHandle(createTableHandle("unknown")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCanHandleRecordSet()
|
||||
{
|
||||
assertTrue(RESOLVER.canHandle(createSplit(CONNECTOR_ID)));
|
||||
assertFalse(RESOLVER.canHandle(createSplit("unknown")));
|
||||
}
|
||||
|
||||
private static JdbcTableHandle createTableHandle(String connectorId)
|
||||
{
|
||||
return new JdbcTableHandle(connectorId, new SchemaTableName("schema", "table"), "catalog", "schema", "table");
|
||||
}
|
||||
|
||||
private static JdbcSplit createSplit(String connectorId)
|
||||
{
|
||||
List<HostAddress> address = ImmutableList.of();
|
||||
return new JdbcSplit(connectorId, "catalog", "schema", "table", "connectionUrl", ImmutableMap.<String, String>of(), TupleDomain.<ColumnHandle>all(),
|
||||
"", address, true, "", "", "", "", System.nanoTime(), 1, false, "");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.JdbcQueryRunner.createJdbcQueryRunner;
|
||||
import static io.airlift.tpch.TpchTable.ORDERS;
|
||||
|
||||
public class TestJdbcIntegrationSmokeTest
|
||||
extends AbstractTestIntegrationSmokeTest
|
||||
{
|
||||
public TestJdbcIntegrationSmokeTest()
|
||||
throws Exception
|
||||
{
|
||||
super(createJdbcQueryRunner(ORDERS));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,190 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnMetadata;
|
||||
import com.facebook.presto.spi.ConnectorSession;
|
||||
import com.facebook.presto.spi.ConnectorTableMetadata;
|
||||
import com.facebook.presto.spi.PrestoException;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.TableNotFoundException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.TestingDatabase.CONNECTOR_ID;
|
||||
import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND;
|
||||
import static com.facebook.presto.spi.StandardErrorCode.PERMISSION_DENIED;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNull;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
import static org.testng.Assert.fail;
|
||||
|
||||
@Test(singleThreaded = true)
|
||||
public class TestJdbcMetadata
|
||||
{
|
||||
private static final ConnectorSession SESSION = new ConnectorSession("user", UTC_KEY, ENGLISH, System.currentTimeMillis(), null);
|
||||
|
||||
private TestingDatabase database;
|
||||
private JdbcMetadata metadata;
|
||||
private JdbcTableHandle tableHandle;
|
||||
|
||||
@BeforeMethod
|
||||
public void setUp()
|
||||
throws Exception
|
||||
{
|
||||
database = new TestingDatabase();
|
||||
metadata = new JdbcMetadata(new JdbcConnectorId(CONNECTOR_ID), database.getJdbcClient(), new JdbcMetadataConfig());
|
||||
tableHandle = metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers"));
|
||||
}
|
||||
|
||||
@AfterMethod(alwaysRun = true)
|
||||
public void tearDown()
|
||||
throws Exception
|
||||
{
|
||||
database.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListSchemaNames()
|
||||
{
|
||||
assertTrue(metadata.listSchemaNames(SESSION).containsAll(ImmutableSet.of("example", "tpch")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTableHandle()
|
||||
{
|
||||
JdbcTableHandle tableHandle = metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers"));
|
||||
assertEquals(metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers")), tableHandle);
|
||||
assertNull(metadata.getTableHandle(SESSION, new SchemaTableName("example", "unknown")));
|
||||
assertNull(metadata.getTableHandle(SESSION, new SchemaTableName("unknown", "numbers")));
|
||||
assertNull(metadata.getTableHandle(SESSION, new SchemaTableName("unknown", "unknown")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetColumnHandles()
|
||||
{
|
||||
// known table
|
||||
assertEquals(metadata.getColumnHandles(tableHandle), ImmutableMap.of(
|
||||
"text", new JdbcColumnHandle(CONNECTOR_ID, "TEXT", VARCHAR),
|
||||
"value", new JdbcColumnHandle(CONNECTOR_ID, "VALUE", BIGINT)));
|
||||
|
||||
// unknown table
|
||||
unknownTableColumnHandle(new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName("unknown", "unknown"), "unknown", "unknown", "unknown"));
|
||||
unknownTableColumnHandle(new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName("example", "numbers"), null, "example", "unknown"));
|
||||
}
|
||||
|
||||
private void unknownTableColumnHandle(JdbcTableHandle tableHandle)
|
||||
{
|
||||
try {
|
||||
metadata.getColumnHandles(tableHandle);
|
||||
fail("Expected getColumnHandle of unknown table to throw a TableNotFoundException");
|
||||
}
|
||||
catch (TableNotFoundException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getTableMetadata()
|
||||
{
|
||||
// known table
|
||||
ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(tableHandle);
|
||||
assertEquals(tableMetadata.getTable(), new SchemaTableName("example", "numbers"));
|
||||
assertEquals(tableMetadata.getColumns(), ImmutableList.of(
|
||||
new ColumnMetadata("text", VARCHAR, false),
|
||||
new ColumnMetadata("value", BIGINT, false)));
|
||||
|
||||
// unknown tables should produce null
|
||||
unknownTableMetadata(new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName("u", "numbers"), null, "unknown", "unknown"));
|
||||
unknownTableMetadata(new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName("example", "numbers"), null, "example", "unknown"));
|
||||
unknownTableMetadata(new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName("example", "numbers"), null, "unknown", "numbers"));
|
||||
}
|
||||
|
||||
private void unknownTableMetadata(JdbcTableHandle tableHandle)
|
||||
{
|
||||
try {
|
||||
metadata.getTableMetadata(tableHandle);
|
||||
fail("Expected getTableMetadata of unknown table to throw a TableNotFoundException");
|
||||
}
|
||||
catch (TableNotFoundException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListTables()
|
||||
{
|
||||
// all schemas
|
||||
assertEquals(ImmutableSet.copyOf(metadata.listTables(SESSION, null)), ImmutableSet.of(
|
||||
new SchemaTableName("example", "numbers"),
|
||||
new SchemaTableName("tpch", "orders"),
|
||||
new SchemaTableName("tpch", "lineitem")));
|
||||
|
||||
// specific schema
|
||||
assertEquals(ImmutableSet.copyOf(metadata.listTables(SESSION, "example")), ImmutableSet.of(
|
||||
new SchemaTableName("example", "numbers")));
|
||||
assertEquals(ImmutableSet.copyOf(metadata.listTables(SESSION, "tpch")), ImmutableSet.of(
|
||||
new SchemaTableName("tpch", "orders"),
|
||||
new SchemaTableName("tpch", "lineitem")));
|
||||
|
||||
// unknown schema
|
||||
assertEquals(ImmutableSet.copyOf(metadata.listTables(SESSION, "unknown")), ImmutableSet.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getColumnMetadata()
|
||||
{
|
||||
assertEquals(
|
||||
metadata.getColumnMetadata(tableHandle, new JdbcColumnHandle(CONNECTOR_ID, "text", VARCHAR)),
|
||||
new ColumnMetadata("text", VARCHAR, false));
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = PrestoException.class)
|
||||
public void testCreateTable()
|
||||
{
|
||||
metadata.createTable(SESSION, new ConnectorTableMetadata(
|
||||
new SchemaTableName("example", "foo"),
|
||||
ImmutableList.of(new ColumnMetadata("text", VARCHAR, false))));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDropTableTable()
|
||||
{
|
||||
try {
|
||||
metadata.dropTable(tableHandle);
|
||||
fail("expected exception");
|
||||
}
|
||||
catch (PrestoException e) {
|
||||
assertEquals(e.getErrorCode(), PERMISSION_DENIED.toErrorCode());
|
||||
}
|
||||
|
||||
JdbcMetadataConfig config = new JdbcMetadataConfig().setAllowDropTable(true);
|
||||
metadata = new JdbcMetadata(new JdbcConnectorId(CONNECTOR_ID), database.getJdbcClient(), config);
|
||||
metadata.dropTable(tableHandle);
|
||||
|
||||
try {
|
||||
metadata.getTableMetadata(tableHandle);
|
||||
fail("expected exception");
|
||||
}
|
||||
catch (PrestoException e) {
|
||||
assertEquals(e.getErrorCode(), NOT_FOUND.toErrorCode());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
|
||||
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
|
||||
import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
|
||||
|
||||
public class TestJdbcMetadataConfig
|
||||
{
|
||||
@Test
|
||||
public void testDefaults()
|
||||
{
|
||||
assertRecordedDefaults(recordDefaults(JdbcMetadataConfig.class)
|
||||
.setAllowDropTable(false));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExplicitPropertyMappings()
|
||||
{
|
||||
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
|
||||
.put("allow-drop-table", "true")
|
||||
.build();
|
||||
|
||||
JdbcMetadataConfig expected = new JdbcMetadataConfig()
|
||||
.setAllowDropTable(true);
|
||||
|
||||
assertFullMapping(properties, expected);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.OUTPUT_TABLE_CODEC;
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.assertJsonRoundTrip;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
|
||||
public class TestJdbcOutputTableHandle
|
||||
{
|
||||
@Test
|
||||
public void testJsonRoundTrip()
|
||||
{
|
||||
JdbcOutputTableHandle handle = new JdbcOutputTableHandle(
|
||||
"connectorId",
|
||||
"catalog",
|
||||
"schema",
|
||||
"table",
|
||||
ImmutableList.of("abc", "xyz"),
|
||||
ImmutableList.<Type>of(VARCHAR, VARCHAR),
|
||||
"test",
|
||||
"tmp_table",
|
||||
"jdbc:junk",
|
||||
ImmutableMap.of("user", "test"));
|
||||
|
||||
assertJsonRoundTrip(OUTPUT_TABLE_CODEC, handle);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,154 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.RecordCursor;
|
||||
import com.facebook.presto.spi.RecordSet;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertFalse;
|
||||
|
||||
@Test
|
||||
public class TestJdbcRecordSet
|
||||
{
|
||||
private TestingDatabase database;
|
||||
private JdbcClient jdbcClient;
|
||||
private JdbcSplit split;
|
||||
private Map<String, JdbcColumnHandle> columnHandles;
|
||||
|
||||
@BeforeClass
|
||||
public void setUp()
|
||||
throws Exception
|
||||
{
|
||||
database = new TestingDatabase();
|
||||
jdbcClient = database.getJdbcClient();
|
||||
split = database.getSplit("example", "numbers");
|
||||
columnHandles = database.getColumnHandles("example", "numbers");
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public void tearDown()
|
||||
throws Exception
|
||||
{
|
||||
database.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetColumnTypes()
|
||||
throws Exception
|
||||
{
|
||||
RecordSet recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
new JdbcColumnHandle("test", "text", VARCHAR),
|
||||
new JdbcColumnHandle("test", "value", BIGINT)));
|
||||
assertEquals(recordSet.getColumnTypes(), ImmutableList.of(VARCHAR, BIGINT));
|
||||
|
||||
recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
new JdbcColumnHandle("test", "value", BIGINT),
|
||||
new JdbcColumnHandle("test", "text", VARCHAR)));
|
||||
assertEquals(recordSet.getColumnTypes(), ImmutableList.of(BIGINT, VARCHAR));
|
||||
|
||||
recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
new JdbcColumnHandle("test", "value", BIGINT),
|
||||
new JdbcColumnHandle("test", "value", BIGINT),
|
||||
new JdbcColumnHandle("test", "text", VARCHAR)));
|
||||
assertEquals(recordSet.getColumnTypes(), ImmutableList.of(BIGINT, BIGINT, VARCHAR));
|
||||
|
||||
recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.<JdbcColumnHandle>of());
|
||||
assertEquals(recordSet.getColumnTypes(), ImmutableList.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCursorSimple()
|
||||
throws Exception
|
||||
{
|
||||
RecordSet recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
columnHandles.get("text"),
|
||||
columnHandles.get("value")));
|
||||
|
||||
try (RecordCursor cursor = recordSet.cursor()) {
|
||||
assertEquals(cursor.getType(0), VARCHAR);
|
||||
assertEquals(cursor.getType(1), BIGINT);
|
||||
|
||||
Map<String, Long> data = new LinkedHashMap<>();
|
||||
while (cursor.advanceNextPosition()) {
|
||||
data.put(cursor.getSlice(0).toStringUtf8(), cursor.getLong(1));
|
||||
assertFalse(cursor.isNull(0));
|
||||
assertFalse(cursor.isNull(1));
|
||||
}
|
||||
|
||||
assertEquals(data, ImmutableMap.<String, Long>builder()
|
||||
.put("one", 1L)
|
||||
.put("two", 2L)
|
||||
.put("three", 3L)
|
||||
.put("ten", 10L)
|
||||
.put("eleven", 11L)
|
||||
.put("twelve", 12L)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCursorMixedOrder()
|
||||
throws Exception
|
||||
{
|
||||
RecordSet recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
columnHandles.get("value"),
|
||||
columnHandles.get("value"),
|
||||
columnHandles.get("text")));
|
||||
|
||||
try (RecordCursor cursor = recordSet.cursor()) {
|
||||
assertEquals(cursor.getType(0), BIGINT);
|
||||
assertEquals(cursor.getType(1), BIGINT);
|
||||
assertEquals(cursor.getType(2), VARCHAR);
|
||||
|
||||
Map<String, Long> data = new LinkedHashMap<>();
|
||||
while (cursor.advanceNextPosition()) {
|
||||
assertEquals(cursor.getLong(0), cursor.getLong(1));
|
||||
data.put(cursor.getSlice(2).toStringUtf8(), cursor.getLong(0));
|
||||
}
|
||||
|
||||
assertEquals(data, ImmutableMap.<String, Long>builder()
|
||||
.put("one", 1L)
|
||||
.put("two", 2L)
|
||||
.put("three", 3L)
|
||||
.put("ten", 10L)
|
||||
.put("eleven", 11L)
|
||||
.put("twelve", 12L)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIdempotentClose()
|
||||
{
|
||||
RecordSet recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of(
|
||||
columnHandles.get("value"),
|
||||
columnHandles.get("value"),
|
||||
columnHandles.get("text")));
|
||||
|
||||
RecordCursor cursor = recordSet.cursor();
|
||||
cursor.close();
|
||||
cursor.close();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,191 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorPartitionResult;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.Domain;
|
||||
import com.facebook.presto.spi.Range;
|
||||
import com.facebook.presto.spi.RecordCursor;
|
||||
import com.facebook.presto.spi.RecordSet;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.SortedRangeSet;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static io.airlift.concurrent.MoreFutures.getFutureValue;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
|
||||
@Test
|
||||
public class TestJdbcRecordSetProvider
|
||||
{
|
||||
private TestingDatabase database;
|
||||
private JdbcClient jdbcClient;
|
||||
private JdbcSplit split;
|
||||
|
||||
private JdbcTableHandle table;
|
||||
private JdbcColumnHandle textColumn;
|
||||
private JdbcColumnHandle valueColumn;
|
||||
|
||||
@BeforeClass
|
||||
public void setUp()
|
||||
throws Exception
|
||||
{
|
||||
database = new TestingDatabase();
|
||||
jdbcClient = database.getJdbcClient();
|
||||
split = database.getSplit("example", "numbers");
|
||||
|
||||
table = jdbcClient.getTableHandle(new SchemaTableName("example", "numbers"));
|
||||
|
||||
Map<String, JdbcColumnHandle> columns = database.getColumnHandles("example", "numbers");
|
||||
textColumn = columns.get("text");
|
||||
valueColumn = columns.get("value");
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public void tearDown()
|
||||
throws Exception
|
||||
{
|
||||
database.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRecordSet()
|
||||
throws Exception
|
||||
{
|
||||
JdbcRecordSetProvider recordSetProvider = new JdbcRecordSetProvider(jdbcClient);
|
||||
RecordSet recordSet = recordSetProvider.getRecordSet(split, ImmutableList.of(textColumn, valueColumn));
|
||||
assertNotNull(recordSet, "recordSet is null");
|
||||
|
||||
RecordCursor cursor = recordSet.cursor();
|
||||
assertNotNull(cursor, "cursor is null");
|
||||
|
||||
Map<String, Long> data = new LinkedHashMap<>();
|
||||
while (cursor.advanceNextPosition()) {
|
||||
data.put(cursor.getSlice(0).toStringUtf8(), cursor.getLong(1));
|
||||
}
|
||||
assertEquals(data, ImmutableMap.<String, Long>builder()
|
||||
.put("one", 1L)
|
||||
.put("two", 2L)
|
||||
.put("three", 3L)
|
||||
.put("ten", 10L)
|
||||
.put("eleven", 11L)
|
||||
.put("twelve", 12L)
|
||||
.build());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTupleDomain()
|
||||
throws Exception
|
||||
{
|
||||
// single value
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.singleValue("foo"))
|
||||
));
|
||||
|
||||
// multiple values (string)
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue("foo"), Domain.singleValue("bar"))))
|
||||
));
|
||||
|
||||
// inequality (string)
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(Range.greaterThan("foo")), false))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(Range.greaterThan("foo")), false))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(Range.lessThanOrEqual("foo")), false))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(Range.lessThan("foo")), false))
|
||||
));
|
||||
|
||||
// is null
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.onlyNull(String.class))
|
||||
));
|
||||
|
||||
// not null
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.notNull(String.class))
|
||||
));
|
||||
|
||||
// specific value or null
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue("foo"), Domain.onlyNull(String.class))))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(Range.range("bar", true, "foo", true)), false))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(SortedRangeSet.of(
|
||||
Range.range("bar", true, "foo", true),
|
||||
Range.range("hello", false, "world", false)),
|
||||
false
|
||||
))
|
||||
));
|
||||
|
||||
getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
|
||||
ImmutableMap.<ColumnHandle, Domain>of(
|
||||
textColumn,
|
||||
Domain.create(SortedRangeSet.of(
|
||||
Range.range("bar", true, "foo", true),
|
||||
Range.range("hello", false, "world", false),
|
||||
Range.equal("apple"),
|
||||
Range.equal("banana"),
|
||||
Range.equal("zoo")),
|
||||
false
|
||||
),
|
||||
|
||||
valueColumn,
|
||||
Domain.create(SortedRangeSet.of(
|
||||
Range.range(1, true, 5, true),
|
||||
Range.range(10, false, 20, false)),
|
||||
true
|
||||
)
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
private RecordCursor getCursor(JdbcTableHandle jdbcTableHandle, List<JdbcColumnHandle> columns, TupleDomain<ColumnHandle> domain)
|
||||
throws InterruptedException
|
||||
{
|
||||
ConnectorPartitionResult partitions = jdbcClient.getPartitions(jdbcTableHandle, domain);
|
||||
ConnectorSplitSource splits = jdbcClient.getPartitionSplits((JdbcPartition) getOnlyElement(partitions.getPartitions()));
|
||||
JdbcSplit split = (JdbcSplit) getOnlyElement(getFutureValue(splits.getNextBatch(1000)));
|
||||
|
||||
JdbcRecordSetProvider recordSetProvider = new JdbcRecordSetProvider(jdbcClient);
|
||||
RecordSet recordSet = recordSetProvider.getRecordSet(split, columns);
|
||||
|
||||
return recordSet.cursor();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.HostAddress;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.airlift.json.JsonCodec;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static io.airlift.json.JsonCodec.jsonCodec;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
public class TestJdbcSplit
|
||||
{
|
||||
List<HostAddress> address = ImmutableList.of();
|
||||
private final JdbcSplit split = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.<String, String>of(), TupleDomain.<ColumnHandle>all(),
|
||||
"", address, true, "", "", "", "", System.nanoTime(), 1, false, "");
|
||||
|
||||
@Test
|
||||
public void testAddresses()
|
||||
{
|
||||
// split uses "example" scheme so no addresses are available and is not remotely accessible
|
||||
assertEquals(split.getAddresses(), ImmutableList.of());
|
||||
assertEquals(split.isRemotelyAccessible(), true);
|
||||
|
||||
JdbcSplit jdbcSplit = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.<String, String>of(), TupleDomain.<ColumnHandle>all(),
|
||||
"", address, true, "", "", "", "", System.nanoTime(), 1, false, "");
|
||||
assertEquals(jdbcSplit.getAddresses(), ImmutableList.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJsonRoundTrip()
|
||||
{
|
||||
JsonCodec<JdbcSplit> codec = jsonCodec(JdbcSplit.class);
|
||||
String json = codec.toJson(split);
|
||||
JdbcSplit copy = codec.fromJson(json);
|
||||
assertEquals(copy.getConnectorId(), split.getConnectorId());
|
||||
assertEquals(copy.getSchemaName(), split.getSchemaName());
|
||||
assertEquals(copy.getTableName(), split.getTableName());
|
||||
|
||||
assertEquals(copy.getAddresses(), ImmutableList.of());
|
||||
assertEquals(copy.isRemotelyAccessible(), true);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import io.airlift.testing.EquivalenceTester;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.TABLE_CODEC;
|
||||
import static com.facebook.presto.plugin.jdbc.MetadataUtil.assertJsonRoundTrip;
|
||||
|
||||
public class TestJdbcTableHandle
|
||||
{
|
||||
@Test
|
||||
public void testJsonRoundTrip()
|
||||
{
|
||||
assertJsonRoundTrip(TABLE_CODEC, new JdbcTableHandle("connectorId", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTable"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEquivalence()
|
||||
{
|
||||
EquivalenceTester.equivalenceTester()
|
||||
.addEquivalentGroup(
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schema", "table"), "jdbcCatalogX", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchemaX", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTableX"))
|
||||
.addEquivalentGroup(
|
||||
new JdbcTableHandle("connectorIdX", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorIdX", new SchemaTableName("schema", "table"), "jdbcCatalogX", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorIdX", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchemaX", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorIdX", new SchemaTableName("schema", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTableX"))
|
||||
.addEquivalentGroup(
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schemaX", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schemaX", "table"), "jdbcCatalogX", "jdbcSchema", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schemaX", "table"), "jdbcCatalog", "jdbcSchemaX", "jdbcTable"),
|
||||
new JdbcTableHandle("connectorId", new SchemaTableName("schemaX", "table"), "jdbcCatalog", "jdbcSchema", "jdbcTableX"))
|
||||
.check();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcCacheConfig;
|
||||
import com.facebook.presto.plugin.jdbc.subtable.JdbcSubTableConfig;
|
||||
import com.facebook.presto.spi.ColumnHandle;
|
||||
import com.facebook.presto.spi.ConnectorPartitionResult;
|
||||
import com.facebook.presto.spi.ConnectorSplitSource;
|
||||
import com.facebook.presto.spi.SchemaTableName;
|
||||
import com.facebook.presto.spi.TupleDomain;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.h2.Driver;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static io.airlift.concurrent.MoreFutures.getFutureValue;
|
||||
|
||||
final class TestingDatabase
|
||||
implements AutoCloseable
|
||||
{
|
||||
public static final String CONNECTOR_ID = "test";
|
||||
|
||||
private final Connection connection;
|
||||
private final JdbcClient jdbcClient;
|
||||
|
||||
public TestingDatabase()
|
||||
throws SQLException
|
||||
{
|
||||
String connectionUrl = "jdbc:h2:mem:test" + System.nanoTime();
|
||||
jdbcClient = new BaseJdbcClient(
|
||||
new JdbcConnectorId(CONNECTOR_ID),
|
||||
new BaseJdbcConfig().setConnectionUrl(connectionUrl),
|
||||
"\"",
|
||||
new Driver(),
|
||||
new JdbcSubTableConfig(),
|
||||
new JdbcCacheConfig());
|
||||
|
||||
connection = DriverManager.getConnection(connectionUrl);
|
||||
connection.createStatement().execute("CREATE SCHEMA example");
|
||||
|
||||
connection.createStatement().execute("CREATE TABLE example.numbers(text varchar primary key, value bigint)");
|
||||
connection.createStatement().execute("INSERT INTO example.numbers(text, value) VALUES " +
|
||||
"('one', 1)," +
|
||||
"('two', 2)," +
|
||||
"('three', 3)," +
|
||||
"('ten', 10)," +
|
||||
"('eleven', 11)," +
|
||||
"('twelve', 12)" +
|
||||
"");
|
||||
connection.createStatement().execute("CREATE SCHEMA tpch");
|
||||
connection.createStatement().execute("CREATE TABLE tpch.orders(orderkey bigint primary key, custkey bigint)");
|
||||
connection.createStatement().execute("CREATE TABLE tpch.lineitem(orderkey bigint primary key, partkey bigint)");
|
||||
|
||||
connection.commit();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
throws SQLException
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
|
||||
public Connection getConnection()
|
||||
{
|
||||
return connection;
|
||||
}
|
||||
|
||||
public JdbcClient getJdbcClient()
|
||||
{
|
||||
return jdbcClient;
|
||||
}
|
||||
|
||||
public JdbcSplit getSplit(String schemaName, String tableName)
|
||||
throws InterruptedException
|
||||
{
|
||||
JdbcTableHandle jdbcTableHandle = jdbcClient.getTableHandle(new SchemaTableName(schemaName, tableName));
|
||||
ConnectorPartitionResult partitions = jdbcClient.getPartitions(jdbcTableHandle, TupleDomain.<ColumnHandle>all());
|
||||
ConnectorSplitSource splits = jdbcClient.getPartitionSplits((JdbcPartition) getOnlyElement(partitions.getPartitions()));
|
||||
return (JdbcSplit) getOnlyElement(getFutureValue(splits.getNextBatch(1000)));
|
||||
}
|
||||
|
||||
public Map<String, JdbcColumnHandle> getColumnHandles(String schemaName, String tableName)
|
||||
{
|
||||
JdbcTableHandle tableHandle = jdbcClient.getTableHandle(new SchemaTableName(schemaName, tableName));
|
||||
List<JdbcColumnHandle> columns = jdbcClient.getColumns(tableHandle);
|
||||
checkArgument(columns != null, "table not found: %s.%s", schemaName, tableName);
|
||||
|
||||
ImmutableMap.Builder<String, JdbcColumnHandle> columnHandles = ImmutableMap.builder();
|
||||
for (JdbcColumnHandle column : columns) {
|
||||
columnHandles.put(column.getColumnMetadata().getName(), column);
|
||||
}
|
||||
return columnHandles.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.plugin.jdbc;
|
||||
|
||||
import com.facebook.presto.plugin.jdbc.cache.JdbcCacheConfig;
|
||||
import com.facebook.presto.plugin.jdbc.subtable.JdbcSubTableConfig;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.inject.Binder;
|
||||
import com.google.inject.Module;
|
||||
import com.google.inject.Provides;
|
||||
import org.h2.Driver;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static io.airlift.configuration.ConfigBinder.configBinder;
|
||||
import static java.lang.String.format;
|
||||
|
||||
class TestingH2JdbcModule
|
||||
implements Module
|
||||
{
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
configBinder(binder).bindConfig(BaseJdbcConfig.class);
|
||||
}
|
||||
|
||||
@Provides
|
||||
public JdbcClient provideJdbcClient(JdbcConnectorId id, BaseJdbcConfig config)
|
||||
{
|
||||
return new BaseJdbcClient(id, config, "\"", new Driver(),
|
||||
new JdbcSubTableConfig(),
|
||||
new JdbcCacheConfig());
|
||||
}
|
||||
|
||||
public static Map<String, String> createProperties()
|
||||
{
|
||||
return ImmutableMap.<String, String>builder()
|
||||
.put("connection-url", format("jdbc:h2:mem:test%s;DB_CLOSE_DELAY=-1", System.nanoTime()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,127 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>0.107</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-benchmark-driver</artifactId>
|
||||
<name>presto-benchmark-driver</name>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
<main-class>com.facebook.presto.benchmark.driver.PrestoBenchmarkDriver</main-class>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>airline</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>discovery</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>http-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>units</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log-manager</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.inject</groupId>
|
||||
<artifactId>javax.inject</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- for testing -->
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<shadedArtifactAttached>true</shadedArtifactAttached>
|
||||
<shadedClassifierName>executable</shadedClassifierName>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<manifestEntries>
|
||||
<Main-Class>${main-class}</Main-Class>
|
||||
</manifestEntries>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.skife.maven</groupId>
|
||||
<artifactId>really-executable-jar-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<flags>-Xmx1G</flags>
|
||||
<classifier>executable</classifier>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>really-executable-jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.facebook.presto.client.ClientSession;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.HostAndPort;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class BenchmarkDriver
|
||||
implements Closeable
|
||||
{
|
||||
private final ClientSession clientSession;
|
||||
private final List<BenchmarkQuery> queries;
|
||||
private final BenchmarkResultsStore resultsStore;
|
||||
private final BenchmarkQueryRunner queryRunner;
|
||||
|
||||
public BenchmarkDriver(BenchmarkResultsStore resultsStore,
|
||||
ClientSession clientSession,
|
||||
Iterable<BenchmarkQuery> queries,
|
||||
int warm,
|
||||
int runs,
|
||||
boolean debug,
|
||||
int maxFailures,
|
||||
Optional<HostAndPort> socksProxy)
|
||||
{
|
||||
this.resultsStore = checkNotNull(resultsStore, "resultsStore is null");
|
||||
this.clientSession = checkNotNull(clientSession, "clientSession is null");
|
||||
this.queries = ImmutableList.copyOf(checkNotNull(queries, "queries is null"));
|
||||
|
||||
queryRunner = new BenchmarkQueryRunner(warm, runs, debug, maxFailures, clientSession.getServer(), socksProxy);
|
||||
}
|
||||
|
||||
public void run(Suite suite)
|
||||
throws Exception
|
||||
{
|
||||
// select queries to run
|
||||
List<BenchmarkQuery> queries = suite.selectQueries(this.queries);
|
||||
if (queries.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
ClientSession session = ClientSession.withSessionProperties(clientSession, suite.getSessionProperties());
|
||||
|
||||
// select schemas to use
|
||||
List<BenchmarkSchema> benchmarkSchemas;
|
||||
if (!suite.getSchemaNameTemplates().isEmpty()) {
|
||||
List<String> schemas = queryRunner.getSchemas(session);
|
||||
benchmarkSchemas = suite.selectSchemas(schemas);
|
||||
}
|
||||
else {
|
||||
benchmarkSchemas = ImmutableList.of(new BenchmarkSchema(session.getSchema()));
|
||||
}
|
||||
if (benchmarkSchemas.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (BenchmarkSchema benchmarkSchema : benchmarkSchemas) {
|
||||
for (BenchmarkQuery benchmarkQuery : queries) {
|
||||
session = ClientSession.withCatalogAndSchema(session, session.getCatalog(), benchmarkSchema.getName());
|
||||
BenchmarkQueryResult result = queryRunner.execute(suite, session, benchmarkQuery);
|
||||
|
||||
resultsStore.store(benchmarkSchema, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
queryRunner.close();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public class BenchmarkDriverExecutionException
|
||||
extends RuntimeException
|
||||
{
|
||||
public BenchmarkDriverExecutionException(String message, RuntimeException cause)
|
||||
{
|
||||
super(requireNonNull(message, "message is null"), requireNonNull(cause, "cause is null"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,217 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.facebook.presto.client.ClientSession;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.net.HostAndPort;
|
||||
import io.airlift.command.Option;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.CharsetEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static java.nio.charset.StandardCharsets.US_ASCII;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public class BenchmarkDriverOptions
|
||||
{
|
||||
@Option(name = "--server", title = "server", description = "Presto server location (default: localhost:8080)")
|
||||
public String server = "localhost:8080";
|
||||
|
||||
@Option(name = "--user", title = "user", description = "Username")
|
||||
public String user = System.getProperty("user.name");
|
||||
|
||||
@Option(name = "--catalog", title = "catalog", description = "Default catalog (default: default)")
|
||||
public String catalog = "default";
|
||||
|
||||
@Option(name = "--schema", title = "schema", description = "Default schema (default: default)")
|
||||
public String schema = "default";
|
||||
|
||||
@Option(name = "--suite", title = "suite", description = "Suite to execute")
|
||||
public List<String> suites = new ArrayList<>();
|
||||
|
||||
@Option(name = "--suite-config", title = "suite-config", description = "Suites configuration file (default: suite.json)")
|
||||
public String suiteConfigFile = "suite.json";
|
||||
|
||||
@Option(name = "--sql", title = "sql", description = "Directory containing sql files (default: sql)")
|
||||
public String sqlTemplateDir = "sql";
|
||||
|
||||
@Option(name = "--query", title = "query", description = "Queries to execute")
|
||||
public List<String> queries = new ArrayList<>();
|
||||
|
||||
@Option(name = "--debug", title = "debug", description = "Enable debug information (default: false)")
|
||||
public boolean debug;
|
||||
|
||||
@Option(name = "--session", title = "session", description = "Session property (property can be used multiple times; format is key=value)")
|
||||
public final List<ClientSessionProperty> sessionProperties = new ArrayList<>();
|
||||
|
||||
@Option(name = "--runs", title = "runs", description = "Number of times to run each query (default: 3)")
|
||||
public int runs = 3;
|
||||
|
||||
@Option(name = "--warm", title = "warm", description = "Number of times to run each query for a warm-up (default: 1)")
|
||||
public int warm = 1;
|
||||
|
||||
@Option(name = "--max-failures", title = "max failures", description = "Max number of consecutive failures before benchmark fails")
|
||||
public int maxFailures = 10;
|
||||
|
||||
@Option(name = "--socks", title = "socks", description = "Socks proxy to use")
|
||||
public HostAndPort socksProxy;
|
||||
|
||||
public ClientSession getClientSession()
|
||||
{
|
||||
return new ClientSession(
|
||||
parseServer(server),
|
||||
user,
|
||||
"presto-benchmark",
|
||||
catalog,
|
||||
schema,
|
||||
TimeZone.getDefault().getID(),
|
||||
Locale.getDefault(),
|
||||
toProperties(this.sessionProperties),
|
||||
debug);
|
||||
}
|
||||
|
||||
private static URI parseServer(String server)
|
||||
{
|
||||
server = server.toLowerCase(ENGLISH);
|
||||
if (server.startsWith("http://") || server.startsWith("https://")) {
|
||||
return URI.create(server);
|
||||
}
|
||||
|
||||
HostAndPort host = HostAndPort.fromString(server);
|
||||
try {
|
||||
return new URI("http", null, host.getHostText(), host.getPortOrDefault(80), null, null, null);
|
||||
}
|
||||
catch (URISyntaxException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, String> toProperties(List<ClientSessionProperty> sessionProperties)
|
||||
{
|
||||
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
|
||||
for (ClientSessionProperty sessionProperty : sessionProperties) {
|
||||
String name = sessionProperty.getName();
|
||||
if (sessionProperty.getCatalog().isPresent()) {
|
||||
name = sessionProperty.getCatalog().get() + "." + name;
|
||||
}
|
||||
builder.put(name, sessionProperty.getValue());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
public static final class ClientSessionProperty
|
||||
{
|
||||
private static final Splitter NAME_VALUE_SPLITTER = Splitter.on('=').limit(2);
|
||||
private static final Splitter NAME_SPLITTER = Splitter.on('.');
|
||||
private final Optional<String> catalog;
|
||||
private final String name;
|
||||
private final String value;
|
||||
|
||||
public ClientSessionProperty(String property)
|
||||
{
|
||||
List<String> nameValue = NAME_VALUE_SPLITTER.splitToList(property);
|
||||
checkArgument(nameValue.size() == 2, "Session property: %s", property);
|
||||
|
||||
List<String> nameParts = NAME_SPLITTER.splitToList(nameValue.get(0));
|
||||
checkArgument(nameParts.size() == 1 || nameParts.size() == 2, "Invalid session property: %s", property);
|
||||
if (nameParts.size() == 1) {
|
||||
catalog = Optional.empty();
|
||||
name = nameParts.get(0);
|
||||
}
|
||||
else {
|
||||
catalog = Optional.of(nameParts.get(0));
|
||||
name = nameParts.get(1);
|
||||
}
|
||||
value = nameValue.get(1);
|
||||
|
||||
verifyProperty(catalog, name, value);
|
||||
}
|
||||
|
||||
public ClientSessionProperty(Optional<String> catalog, String name, String value)
|
||||
{
|
||||
this.catalog = checkNotNull(catalog, "catalog is null");
|
||||
this.name = checkNotNull(name, "name is null");
|
||||
this.value = checkNotNull(value, "value is null");
|
||||
|
||||
verifyProperty(catalog, name, value);
|
||||
}
|
||||
|
||||
private static void verifyProperty(Optional<String> catalog, String name, String value)
|
||||
{
|
||||
checkArgument(!catalog.isPresent() || !catalog.get().isEmpty(), "Invalid session property: %s.%s:%s", catalog, name, value);
|
||||
checkArgument(!name.isEmpty(), "Session property name is empty");
|
||||
|
||||
CharsetEncoder charsetEncoder = US_ASCII.newEncoder();
|
||||
checkArgument(catalog.orElse("").indexOf('=') < 0, "Session property catalog must not contain '=': %s", name);
|
||||
checkArgument(charsetEncoder.canEncode(catalog.orElse("")), "Session property catalog is not US_ASCII: %s", name);
|
||||
checkArgument(name.indexOf('=') < 0, "Session property name must not contain '=': %s", name);
|
||||
checkArgument(charsetEncoder.canEncode(name), "Session property name is not US_ASCII: %s", name);
|
||||
checkArgument(charsetEncoder.canEncode(value), "Session property value is not US_ASCII: %s", value);
|
||||
}
|
||||
|
||||
public Optional<String> getCatalog()
|
||||
{
|
||||
return catalog;
|
||||
}
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getValue()
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return (catalog.isPresent() ? catalog.get() + '.' : "") + name + '=' + value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(catalog, name, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (obj == null || getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
ClientSessionProperty other = (ClientSessionProperty) obj;
|
||||
return Objects.equals(this.catalog, other.catalog) &&
|
||||
Objects.equals(this.name, other.name) &&
|
||||
Objects.equals(this.value, other.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.base.Splitter.MapSplitter;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Files;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public final class BenchmarkQuery
|
||||
{
|
||||
private static final Splitter SECTION_SPLITTER = Splitter.on(Pattern.compile("\n\\s*=+\\s*\n", Pattern.MULTILINE)).limit(2).trimResults();
|
||||
private static final MapSplitter TAGS_SPLITTER = Splitter.on("\n").trimResults().withKeyValueSeparator('=');
|
||||
|
||||
private final String name;
|
||||
private final Map<String, String> tags;
|
||||
private final String sql;
|
||||
|
||||
public BenchmarkQuery(File file)
|
||||
throws IOException
|
||||
{
|
||||
checkNotNull(file, "file is null");
|
||||
|
||||
name = Files.getNameWithoutExtension(file.getName());
|
||||
|
||||
// file can have 2 sections separated by a line of equals signs
|
||||
String text = Files.toString(file, StandardCharsets.UTF_8);
|
||||
List<String> sections = SECTION_SPLITTER.splitToList(text);
|
||||
if (sections.size() == 2) {
|
||||
this.tags = ImmutableMap.copyOf(TAGS_SPLITTER.split(sections.get(0)));
|
||||
this.sql = sections.get(1);
|
||||
}
|
||||
else {
|
||||
// no tags
|
||||
this.tags = ImmutableMap.of();
|
||||
this.sql = sections.get(0);
|
||||
}
|
||||
}
|
||||
|
||||
public static Function<BenchmarkQuery, String> queryNameGetter()
|
||||
{
|
||||
return query -> query.getName();
|
||||
}
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
public Map<String, String> getTags()
|
||||
{
|
||||
return tags;
|
||||
}
|
||||
|
||||
public String getSql()
|
||||
{
|
||||
return sql;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return MoreObjects.toStringHelper(this)
|
||||
.add("name", name)
|
||||
.add("tags", tags)
|
||||
.add("sql", sql)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.MoreObjects;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
|
||||
public class BenchmarkQueryResult
|
||||
{
|
||||
public enum Status
|
||||
{
|
||||
PASS, FAIL
|
||||
}
|
||||
|
||||
private static final Stat FAIL_STAT = new Stat(new double[0]);
|
||||
|
||||
public static BenchmarkQueryResult passResult(Suite suite, BenchmarkQuery benchmarkQuery, Stat wallTimeNanos, Stat processCpuTimeNanos, Stat queryCpuTimeNanos)
|
||||
{
|
||||
return new BenchmarkQueryResult(suite, benchmarkQuery, Status.PASS, Optional.empty(), wallTimeNanos, processCpuTimeNanos, queryCpuTimeNanos);
|
||||
}
|
||||
|
||||
public static BenchmarkQueryResult failResult(Suite suite, BenchmarkQuery benchmarkQuery, String errorMessage)
|
||||
{
|
||||
return new BenchmarkQueryResult(suite, benchmarkQuery, Status.FAIL, Optional.of(errorMessage), FAIL_STAT, FAIL_STAT, FAIL_STAT);
|
||||
}
|
||||
|
||||
private final Suite suite;
|
||||
private final BenchmarkQuery benchmarkQuery;
|
||||
private final Status status;
|
||||
private final Optional<String> errorMessage;
|
||||
private final Stat wallTimeNanos;
|
||||
private final Stat processCpuTimeNanos;
|
||||
private final Stat queryCpuTimeNanos;
|
||||
|
||||
private BenchmarkQueryResult(
|
||||
Suite suite,
|
||||
BenchmarkQuery benchmarkQuery,
|
||||
Status status,
|
||||
Optional<String> errorMessage,
|
||||
Stat wallTimeNanos,
|
||||
Stat processCpuTimeNanos,
|
||||
Stat queryCpuTimeNanos)
|
||||
{
|
||||
this.suite = checkNotNull(suite, "suite is null");
|
||||
this.benchmarkQuery = checkNotNull(benchmarkQuery, "benchmarkQuery is null");
|
||||
this.status = checkNotNull(status, "status is null");
|
||||
this.errorMessage = requireNonNull(errorMessage, "errorMessage is null");
|
||||
this.wallTimeNanos = checkNotNull(wallTimeNanos, "wallTimeNanos is null");
|
||||
this.processCpuTimeNanos = checkNotNull(processCpuTimeNanos, "processCpuTimeNanos is null");
|
||||
this.queryCpuTimeNanos = checkNotNull(queryCpuTimeNanos, "queryCpuTimeNanos is null");
|
||||
}
|
||||
|
||||
public Suite getSuite()
|
||||
{
|
||||
return suite;
|
||||
}
|
||||
|
||||
public BenchmarkQuery getBenchmarkQuery()
|
||||
{
|
||||
return benchmarkQuery;
|
||||
}
|
||||
|
||||
public Status getStatus()
|
||||
{
|
||||
return status;
|
||||
}
|
||||
|
||||
public Optional<String> getErrorMessage()
|
||||
{
|
||||
return errorMessage;
|
||||
}
|
||||
|
||||
public Stat getWallTimeNanos()
|
||||
{
|
||||
return wallTimeNanos;
|
||||
}
|
||||
|
||||
public Stat getProcessCpuTimeNanos()
|
||||
{
|
||||
return processCpuTimeNanos;
|
||||
}
|
||||
|
||||
public Stat getQueryCpuTimeNanos()
|
||||
{
|
||||
return queryCpuTimeNanos;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return MoreObjects.toStringHelper(this)
|
||||
.add("suite", suite.getName())
|
||||
.add("benchmarkQuery", benchmarkQuery.getName())
|
||||
.add("status", status)
|
||||
.add("wallTimeMedian", new Duration(wallTimeNanos.getMedian(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("wallTimeMean", new Duration(wallTimeNanos.getMean(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("wallTimeStd", new Duration(wallTimeNanos.getStandardDeviation(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("processCpuTimeMedian", new Duration(processCpuTimeNanos.getMedian(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("processCpuTimeMean", new Duration(processCpuTimeNanos.getMean(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("processCpuTimeStd", new Duration(processCpuTimeNanos.getStandardDeviation(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("queryCpuTimeMedian", new Duration(queryCpuTimeNanos.getMedian(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("queryCpuTimeMean", new Duration(queryCpuTimeNanos.getMean(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("queryCpuTimeStd", new Duration(queryCpuTimeNanos.getStandardDeviation(), NANOSECONDS).convertToMostSuccinctTimeUnit())
|
||||
.add("error", errorMessage)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,279 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.facebook.presto.client.ClientSession;
|
||||
import com.facebook.presto.client.QueryError;
|
||||
import com.facebook.presto.client.QueryResults;
|
||||
import com.facebook.presto.client.StatementClient;
|
||||
import com.facebook.presto.client.StatementStats;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.HostAndPort;
|
||||
import io.airlift.discovery.client.ServiceDescriptor;
|
||||
import io.airlift.discovery.client.ServiceDescriptorsRepresentation;
|
||||
import io.airlift.http.client.HttpClient;
|
||||
import io.airlift.http.client.HttpClientConfig;
|
||||
import io.airlift.http.client.JsonResponseHandler;
|
||||
import io.airlift.http.client.Request;
|
||||
import io.airlift.http.client.jetty.JettyHttpClient;
|
||||
import io.airlift.json.JsonCodec;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.facebook.presto.benchmark.driver.BenchmarkQueryResult.failResult;
|
||||
import static com.facebook.presto.benchmark.driver.BenchmarkQueryResult.passResult;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom;
|
||||
import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler;
|
||||
import static io.airlift.http.client.Request.Builder.prepareGet;
|
||||
import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler;
|
||||
import static io.airlift.json.JsonCodec.jsonCodec;
|
||||
import static java.lang.Long.parseLong;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
|
||||
public class BenchmarkQueryRunner
|
||||
implements Closeable
|
||||
{
|
||||
private final int warm;
|
||||
private final int runs;
|
||||
private final boolean debug;
|
||||
private final int maxFailures;
|
||||
|
||||
private final HttpClient httpClient;
|
||||
private final List<URI> nodes;
|
||||
private final JsonCodec<QueryResults> queryResultsCodec;
|
||||
|
||||
private int failures;
|
||||
|
||||
public BenchmarkQueryRunner(int warm, int runs, boolean debug, int maxFailures, URI serverUri, Optional<HostAndPort> socksProxy)
|
||||
{
|
||||
checkArgument(warm >= 0, "warm is negative");
|
||||
this.warm = warm;
|
||||
|
||||
checkArgument(runs >= 1, "runs must be at least 1");
|
||||
this.runs = runs;
|
||||
|
||||
checkArgument(maxFailures >= 0, "maxFailures must be at least 0");
|
||||
this.maxFailures = maxFailures;
|
||||
|
||||
this.debug = debug;
|
||||
|
||||
this.queryResultsCodec = jsonCodec(QueryResults.class);
|
||||
|
||||
checkNotNull(socksProxy, "socksProxy is null");
|
||||
HttpClientConfig httpClientConfig = new HttpClientConfig();
|
||||
if (socksProxy.isPresent()) {
|
||||
httpClientConfig.setSocksProxy(socksProxy.get());
|
||||
}
|
||||
|
||||
this.httpClient = new JettyHttpClient(httpClientConfig.setConnectTimeout(new Duration(10, TimeUnit.SECONDS)));
|
||||
|
||||
nodes = getAllNodes(checkNotNull(serverUri, "serverUri is null"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("AssignmentToForLoopParameter")
|
||||
public BenchmarkQueryResult execute(Suite suite, ClientSession session, BenchmarkQuery query)
|
||||
{
|
||||
failures = 0;
|
||||
for (int i = 0; i < warm; ) {
|
||||
try {
|
||||
execute(session, query.getName(), query.getSql());
|
||||
i++;
|
||||
failures = 0;
|
||||
}
|
||||
catch (BenchmarkDriverExecutionException e) {
|
||||
return failResult(suite, query, e.getCause().getMessage());
|
||||
}
|
||||
catch (Exception e) {
|
||||
handleFailure(e);
|
||||
}
|
||||
}
|
||||
|
||||
double[] wallTimeNanos = new double[runs];
|
||||
double[] processCpuTimeNanos = new double[runs];
|
||||
double[] queryCpuTimeNanos = new double[runs];
|
||||
for (int i = 0; i < runs; ) {
|
||||
try {
|
||||
long startCpuTime = getTotalCpuTime();
|
||||
long startWallTime = System.nanoTime();
|
||||
|
||||
StatementStats statementStats = execute(session, query.getName(), query.getSql());
|
||||
|
||||
long endWallTime = System.nanoTime();
|
||||
long endCpuTime = getTotalCpuTime();
|
||||
|
||||
wallTimeNanos[i] = endWallTime - startWallTime;
|
||||
processCpuTimeNanos[i] = endCpuTime - startCpuTime;
|
||||
queryCpuTimeNanos[i] = MILLISECONDS.toNanos(statementStats.getCpuTimeMillis());
|
||||
|
||||
i++;
|
||||
failures = 0;
|
||||
}
|
||||
catch (BenchmarkDriverExecutionException e) {
|
||||
return failResult(suite, query, e.getCause().getMessage());
|
||||
}
|
||||
catch (Exception e) {
|
||||
handleFailure(e);
|
||||
}
|
||||
}
|
||||
|
||||
return passResult(
|
||||
suite,
|
||||
query,
|
||||
new Stat(wallTimeNanos),
|
||||
new Stat(processCpuTimeNanos),
|
||||
new Stat(queryCpuTimeNanos));
|
||||
}
|
||||
|
||||
public List<String> getSchemas(ClientSession session)
|
||||
{
|
||||
failures = 0;
|
||||
while (true) {
|
||||
// start query
|
||||
StatementClient client = new StatementClient(httpClient, queryResultsCodec, session, "show schemas");
|
||||
|
||||
// read query output
|
||||
ImmutableList.Builder<String> schemas = ImmutableList.builder();
|
||||
while (client.isValid() && client.advance()) {
|
||||
// we do not process the output
|
||||
Iterable<List<Object>> data = client.current().getData();
|
||||
if (data != null) {
|
||||
for (List<Object> objects : data) {
|
||||
schemas.add(objects.get(0).toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verify final state
|
||||
if (client.isClosed()) {
|
||||
throw new IllegalStateException("Query aborted by user");
|
||||
}
|
||||
|
||||
if (client.isGone()) {
|
||||
throw new IllegalStateException("Query is gone (server restarted?)");
|
||||
}
|
||||
|
||||
QueryError resultsError = client.finalResults().getError();
|
||||
if (resultsError != null) {
|
||||
RuntimeException cause = null;
|
||||
if (resultsError.getFailureInfo() != null) {
|
||||
cause = resultsError.getFailureInfo().toException();
|
||||
}
|
||||
handleFailure(cause);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
return schemas.build();
|
||||
}
|
||||
}
|
||||
|
||||
private StatementStats execute(ClientSession session, String name, String query)
|
||||
{
|
||||
// start query
|
||||
StatementClient client = new StatementClient(httpClient, queryResultsCodec, session, query);
|
||||
|
||||
// read query output
|
||||
while (client.isValid() && client.advance()) {
|
||||
// we do not process the output
|
||||
}
|
||||
|
||||
// verify final state
|
||||
if (client.isClosed()) {
|
||||
throw new IllegalStateException("Query aborted by user");
|
||||
}
|
||||
|
||||
if (client.isGone()) {
|
||||
throw new IllegalStateException("Query is gone (server restarted?)");
|
||||
}
|
||||
|
||||
QueryError resultsError = client.finalResults().getError();
|
||||
if (resultsError != null) {
|
||||
RuntimeException cause = null;
|
||||
if (resultsError.getFailureInfo() != null) {
|
||||
cause = resultsError.getFailureInfo().toException();
|
||||
}
|
||||
|
||||
throw new BenchmarkDriverExecutionException(format("Query %s failed: %s", name, resultsError.getMessage()), cause);
|
||||
}
|
||||
|
||||
return client.finalResults().getStats();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
httpClient.close();
|
||||
}
|
||||
|
||||
@SuppressWarnings("CallToPrintStackTrace")
|
||||
public void handleFailure(Exception e)
|
||||
{
|
||||
if (debug) {
|
||||
if (e == null) {
|
||||
e = new RuntimeException("Unknown error");
|
||||
}
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
failures++;
|
||||
|
||||
if (failures > maxFailures) {
|
||||
throw new RuntimeException("To many consecutive failures");
|
||||
}
|
||||
|
||||
try {
|
||||
TimeUnit.SECONDS.sleep(5);
|
||||
}
|
||||
catch (InterruptedException interruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw Throwables.propagate(interruptedException);
|
||||
}
|
||||
}
|
||||
|
||||
private long getTotalCpuTime()
|
||||
{
|
||||
long totalCpuTime = 0;
|
||||
for (URI server : nodes) {
|
||||
URI addressUri = uriBuilderFrom(server).replacePath("/v1/jmx/mbean/java.lang:type=OperatingSystem/ProcessCpuTime").build();
|
||||
String data = httpClient.execute(prepareGet().setUri(addressUri).build(), createStringResponseHandler()).getBody();
|
||||
totalCpuTime += parseLong(data.trim());
|
||||
}
|
||||
return TimeUnit.NANOSECONDS.toNanos(totalCpuTime);
|
||||
}
|
||||
|
||||
private List<URI> getAllNodes(URI server)
|
||||
{
|
||||
Request request = prepareGet().setUri(uriBuilderFrom(server).replacePath("/v1/service/presto").build()).build();
|
||||
JsonResponseHandler<ServiceDescriptorsRepresentation> responseHandler = createJsonResponseHandler(jsonCodec(ServiceDescriptorsRepresentation.class));
|
||||
ServiceDescriptorsRepresentation serviceDescriptors = httpClient.execute(request, responseHandler);
|
||||
|
||||
ImmutableList.Builder<URI> addresses = ImmutableList.builder();
|
||||
for (ServiceDescriptor serviceDescriptor : serviceDescriptors.getServiceDescriptors()) {
|
||||
String httpUri = serviceDescriptor.getProperties().get("http");
|
||||
if (httpUri != null) {
|
||||
addresses.add(URI.create(httpUri));
|
||||
}
|
||||
}
|
||||
return addresses.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import static com.google.common.base.CharMatcher.anyOf;
|
||||
import static com.google.common.base.Functions.forMap;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.collect.Iterables.getFirst;
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
|
||||
public class BenchmarkResultsPrinter
|
||||
implements BenchmarkResultsStore
|
||||
{
|
||||
private final List<String> tagNames;
|
||||
|
||||
public BenchmarkResultsPrinter(Iterable<Suite> suites, Iterable<BenchmarkQuery> queries)
|
||||
{
|
||||
this(getSelectedQueryTagNames(suites, queries));
|
||||
}
|
||||
|
||||
private static List<String> getSelectedQueryTagNames(Iterable<Suite> suites, Iterable<BenchmarkQuery> queries)
|
||||
{
|
||||
SortedSet<String> tags = new TreeSet<>();
|
||||
for (Suite suite : suites) {
|
||||
for (BenchmarkQuery query : suite.selectQueries(queries)) {
|
||||
tags.addAll(query.getTags().keySet());
|
||||
}
|
||||
|
||||
for (RegexTemplate regexTemplate : suite.getSchemaNameTemplates()) {
|
||||
tags.addAll(regexTemplate.getFieldNames());
|
||||
}
|
||||
}
|
||||
return ImmutableList.copyOf(tags);
|
||||
}
|
||||
|
||||
public BenchmarkResultsPrinter(List<String> tagNames)
|
||||
{
|
||||
this.tagNames = checkNotNull(tagNames, "tagNames is null");
|
||||
|
||||
// print header row
|
||||
printRow(ImmutableList.builder()
|
||||
.add("suite")
|
||||
.add("query")
|
||||
.addAll(tagNames)
|
||||
.add("wallTimeP50")
|
||||
.add("wallTimeMean")
|
||||
.add("wallTimeStd")
|
||||
.add("processCpuTimeP50")
|
||||
.add("processCpuTimeMean")
|
||||
.add("processCpuTimeStd")
|
||||
.add("queryCpuTimeP50")
|
||||
.add("queryCpuTimeMean")
|
||||
.add("queryCpuTimeStd")
|
||||
.add("status")
|
||||
.add("error")
|
||||
.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void store(BenchmarkSchema benchmarkSchema, BenchmarkQueryResult result)
|
||||
{
|
||||
Map<String, String> tags = new LinkedHashMap<>();
|
||||
tags.putAll(result.getBenchmarkQuery().getTags());
|
||||
tags.putAll(benchmarkSchema.getTags());
|
||||
|
||||
// only print first line of error message
|
||||
Optional<String> errorMessage = result.getErrorMessage().map(error -> getFirst(Splitter.on(anyOf("\r\n")).trimResults().split(error), ""));
|
||||
|
||||
printRow(ImmutableList.builder()
|
||||
.add(result.getSuite().getName())
|
||||
.add(result.getBenchmarkQuery().getName())
|
||||
.addAll(transform(tagNames, forMap(tags, "")))
|
||||
.add(NANOSECONDS.toMillis((long) result.getWallTimeNanos().getMedian()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getWallTimeNanos().getMean()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getWallTimeNanos().getStandardDeviation()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getProcessCpuTimeNanos().getMedian()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getProcessCpuTimeNanos().getMean()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getProcessCpuTimeNanos().getStandardDeviation()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getQueryCpuTimeNanos().getMedian()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getQueryCpuTimeNanos().getMean()))
|
||||
.add(NANOSECONDS.toMillis((long) result.getQueryCpuTimeNanos().getStandardDeviation()))
|
||||
.add(result.getStatus().toString().toLowerCase())
|
||||
.add(errorMessage.orElse(""))
|
||||
.build());
|
||||
}
|
||||
|
||||
@SuppressWarnings("UseOfSystemOutOrSystemErr")
|
||||
private static void printRow(Iterable<?> values)
|
||||
{
|
||||
System.out.println(Joiner.on('\t').join(values));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
public interface BenchmarkResultsStore
|
||||
{
|
||||
void store(BenchmarkSchema benchmarkSchema, BenchmarkQueryResult result);
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class BenchmarkSchema
|
||||
{
|
||||
private final String name;
|
||||
private final Map<String, String> tags;
|
||||
|
||||
public BenchmarkSchema(String name)
|
||||
{
|
||||
this(name, ImmutableMap.<String, String>of());
|
||||
}
|
||||
|
||||
public BenchmarkSchema(String name, Map<String, String> tags)
|
||||
{
|
||||
this.name = name;
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
public Map<String, String> getTags()
|
||||
{
|
||||
return tags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return MoreObjects.toStringHelper(this)
|
||||
.add("name", name)
|
||||
.add("tags", tags)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.facebook.presto.client.ClientSession;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.airlift.command.Command;
|
||||
import io.airlift.command.HelpOption;
|
||||
import io.airlift.log.Level;
|
||||
import io.airlift.log.Logging;
|
||||
import io.airlift.log.LoggingConfiguration;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.google.common.io.ByteStreams.nullOutputStream;
|
||||
import static io.airlift.command.SingleCommand.singleCommand;
|
||||
import static java.util.function.Function.identity;
|
||||
|
||||
@Command(name = "presto-benchmark", description = "Presto benchmark driver")
|
||||
public class PrestoBenchmarkDriver
|
||||
{
|
||||
@Inject
|
||||
public HelpOption helpOption;
|
||||
|
||||
@Inject
|
||||
public BenchmarkDriverOptions benchmarkDriverOptions = new BenchmarkDriverOptions();
|
||||
|
||||
public static void main(String[] args)
|
||||
throws Exception
|
||||
{
|
||||
new PrestoBenchmarkDriver().run(args);
|
||||
}
|
||||
|
||||
protected void run(String[] args)
|
||||
throws Exception
|
||||
{
|
||||
PrestoBenchmarkDriver prestoBenchmarkDriver = singleCommand(PrestoBenchmarkDriver.class).parse(args);
|
||||
|
||||
if (prestoBenchmarkDriver.helpOption.showHelpIfRequested()) {
|
||||
return;
|
||||
}
|
||||
|
||||
BenchmarkDriverOptions driverOptions = prestoBenchmarkDriver.benchmarkDriverOptions;
|
||||
|
||||
initializeLogging(driverOptions.debug);
|
||||
|
||||
// select suites
|
||||
List<Suite> suites = Suite.readSuites(new File(driverOptions.suiteConfigFile));
|
||||
if (!driverOptions.suites.isEmpty()) {
|
||||
suites = suites.stream()
|
||||
.filter(suite -> driverOptions.suites.contains(suite.getName()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
suites = ImmutableList.copyOf(suites);
|
||||
|
||||
// load queries
|
||||
File queriesDir = new File(driverOptions.sqlTemplateDir);
|
||||
List<BenchmarkQuery> allQueries = readQueries(queriesDir);
|
||||
|
||||
// select queries to run
|
||||
Set<BenchmarkQuery> queries;
|
||||
if (driverOptions.queries.isEmpty()) {
|
||||
queries = suites.stream()
|
||||
.map(suite -> suite.selectQueries(allQueries))
|
||||
.flatMap(List::stream)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
else {
|
||||
queries = driverOptions.queries.stream()
|
||||
.map(Pattern::compile)
|
||||
.map(pattern -> allQueries.stream().filter(query -> pattern.matcher(query.getName()).matches()))
|
||||
.flatMap(identity())
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
// create results store
|
||||
BenchmarkResultsStore resultsStore = getResultsStore(suites, queries);
|
||||
|
||||
// create session
|
||||
ClientSession session = driverOptions.getClientSession();
|
||||
|
||||
try (BenchmarkDriver benchmarkDriver = new BenchmarkDriver(
|
||||
resultsStore,
|
||||
session,
|
||||
queries,
|
||||
driverOptions.warm,
|
||||
driverOptions.runs,
|
||||
driverOptions.debug,
|
||||
driverOptions.maxFailures,
|
||||
Optional.ofNullable(driverOptions.socksProxy))) {
|
||||
for (Suite suite : suites) {
|
||||
benchmarkDriver.run(suite);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected BenchmarkResultsStore getResultsStore(List<Suite> suites, Set<BenchmarkQuery> queries)
|
||||
{
|
||||
return new BenchmarkResultsPrinter(suites, queries);
|
||||
}
|
||||
|
||||
private static List<BenchmarkQuery> readQueries(File queriesDir)
|
||||
throws IOException
|
||||
{
|
||||
File[] files = queriesDir.listFiles();
|
||||
if (files == null) {
|
||||
return ImmutableList.of();
|
||||
}
|
||||
Arrays.sort(files);
|
||||
|
||||
ImmutableList.Builder<BenchmarkQuery> queries = ImmutableList.builder();
|
||||
for (File file : files) {
|
||||
String fileName = file.getName();
|
||||
if (fileName.endsWith(".sql")) {
|
||||
queries.add(new BenchmarkQuery(file));
|
||||
}
|
||||
}
|
||||
return queries.build();
|
||||
}
|
||||
|
||||
@SuppressWarnings("UseOfSystemOutOrSystemErr")
|
||||
public static void initializeLogging(boolean debug)
|
||||
{
|
||||
// unhook out and err while initializing logging or logger will print to them
|
||||
PrintStream out = System.out;
|
||||
PrintStream err = System.err;
|
||||
try {
|
||||
if (debug) {
|
||||
Logging logging = Logging.initialize();
|
||||
logging.configure(new LoggingConfiguration());
|
||||
logging.setLevel("com.facebook.presto", Level.DEBUG);
|
||||
}
|
||||
else {
|
||||
System.setOut(new PrintStream(nullOutputStream()));
|
||||
System.setErr(new PrintStream(nullOutputStream()));
|
||||
|
||||
Logging logging = Logging.initialize();
|
||||
logging.configure(new LoggingConfiguration());
|
||||
logging.disableConsole();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
finally {
|
||||
System.setOut(out);
|
||||
System.setErr(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableBiMap;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class RegexTemplate
|
||||
{
|
||||
private static final Method NAMED_GROUPS_METHOD;
|
||||
|
||||
static {
|
||||
try {
|
||||
NAMED_GROUPS_METHOD = Pattern.class.getDeclaredMethod("namedGroups");
|
||||
NAMED_GROUPS_METHOD.setAccessible(true);
|
||||
}
|
||||
catch (NoSuchMethodException e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
}
|
||||
|
||||
private final String template;
|
||||
private final Pattern pattern;
|
||||
private final List<String> fieldNames;
|
||||
|
||||
public RegexTemplate(String template)
|
||||
{
|
||||
this.template = checkNotNull(template, "template is null");
|
||||
|
||||
try {
|
||||
this.pattern = Pattern.compile(template);
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new IllegalArgumentException("Invalid template: " + template, e);
|
||||
}
|
||||
|
||||
Map<String, Integer> namedGroups;
|
||||
try {
|
||||
namedGroups = (Map<String, Integer>) NAMED_GROUPS_METHOD.invoke(pattern);
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw Throwables.propagate(e);
|
||||
}
|
||||
ImmutableSortedMap<Integer, String> sortedGroups = ImmutableSortedMap.copyOf(ImmutableBiMap.copyOf(namedGroups).inverse());
|
||||
this.fieldNames = ImmutableList.copyOf(sortedGroups.values());
|
||||
}
|
||||
|
||||
public List<String> getFieldNames()
|
||||
{
|
||||
return fieldNames;
|
||||
}
|
||||
|
||||
public Optional<Map<String, String>> parse(String value)
|
||||
{
|
||||
checkNotNull(value, "value is null");
|
||||
|
||||
Matcher matcher = pattern.matcher(value);
|
||||
if (!matcher.matches()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
ImmutableMap.Builder<String, String> fieldsBuilder = ImmutableMap.builder();
|
||||
for (int index = 0; index < fieldNames.size(); index++) {
|
||||
String fieldName = fieldNames.get(index);
|
||||
String fieldValue = matcher.group(index + 1);
|
||||
if (fieldValue != null) {
|
||||
fieldsBuilder.put(fieldName, fieldValue);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, String> fields = fieldsBuilder.build();
|
||||
return Optional.of(fields);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return template;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.base.MoreObjects;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Median;
|
||||
|
||||
public class Stat
|
||||
{
|
||||
private final double mean;
|
||||
private final double standardDeviation;
|
||||
private final double median;
|
||||
|
||||
public Stat(double[] values)
|
||||
{
|
||||
mean = new Mean().evaluate(values);
|
||||
standardDeviation = new StandardDeviation().evaluate(values);
|
||||
median = new Median().evaluate(values);
|
||||
}
|
||||
|
||||
public double getMean()
|
||||
{
|
||||
return mean;
|
||||
}
|
||||
|
||||
public double getStandardDeviation()
|
||||
{
|
||||
return standardDeviation;
|
||||
}
|
||||
|
||||
public double getMedian()
|
||||
{
|
||||
return median;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return MoreObjects.toStringHelper(this)
|
||||
.add("mean", mean)
|
||||
.add("standardDeviation", standardDeviation)
|
||||
.add("median", median)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static io.airlift.json.JsonCodec.mapJsonCodec;
|
||||
|
||||
public class Suite
|
||||
{
|
||||
private final String name;
|
||||
private final Map<String, String> sessionProperties;
|
||||
private final List<RegexTemplate> schemaNameTemplates;
|
||||
private final List<Pattern> queryNamePatterns;
|
||||
|
||||
public Suite(String name, Map<String, String> sessionProperties, Iterable<RegexTemplate> schemaNameTemplates, Iterable<Pattern> queryNamePatterns)
|
||||
{
|
||||
this.name = checkNotNull(name, "name is null");
|
||||
this.sessionProperties = sessionProperties == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(sessionProperties);
|
||||
this.schemaNameTemplates = ImmutableList.copyOf(checkNotNull(schemaNameTemplates, "schemaNameTemplates is null"));
|
||||
this.queryNamePatterns = ImmutableList.copyOf(checkNotNull(queryNamePatterns, "queryNamePatterns is null"));
|
||||
}
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
public Map<String, String> getSessionProperties()
|
||||
{
|
||||
return sessionProperties;
|
||||
}
|
||||
|
||||
public List<RegexTemplate> getSchemaNameTemplates()
|
||||
{
|
||||
return schemaNameTemplates;
|
||||
}
|
||||
|
||||
public List<Pattern> getQueryNamePatterns()
|
||||
{
|
||||
return queryNamePatterns;
|
||||
}
|
||||
|
||||
public List<BenchmarkSchema> selectSchemas(Iterable<String> schemas)
|
||||
{
|
||||
if (schemaNameTemplates.isEmpty()) {
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
ImmutableList.Builder<BenchmarkSchema> benchmarkSchemas = ImmutableList.builder();
|
||||
for (RegexTemplate schemaNameTemplate : schemaNameTemplates) {
|
||||
for (String schema : schemas) {
|
||||
Optional<Map<String, String>> tags = schemaNameTemplate.parse(schema);
|
||||
if (tags.isPresent()) {
|
||||
benchmarkSchemas.add(new BenchmarkSchema(schema, tags.get()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return benchmarkSchemas.build();
|
||||
}
|
||||
|
||||
public List<BenchmarkQuery> selectQueries(Iterable<BenchmarkQuery> queries)
|
||||
{
|
||||
if (getQueryNamePatterns().isEmpty()) {
|
||||
return ImmutableList.copyOf(queries);
|
||||
}
|
||||
|
||||
List<BenchmarkQuery> filteredQueries = StreamSupport.stream(queries.spliterator(), false)
|
||||
.filter(query -> getQueryNamePatterns().stream().anyMatch(pattern -> pattern.matcher(query.getName()).matches()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
return ImmutableList.copyOf(filteredQueries);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return MoreObjects.toStringHelper(this)
|
||||
.add("name", name)
|
||||
.add("sessionProperties", sessionProperties)
|
||||
.add("queryNamePatterns", queryNamePatterns)
|
||||
.toString();
|
||||
}
|
||||
|
||||
public static List<Suite> readSuites(File file)
|
||||
throws IOException
|
||||
{
|
||||
checkNotNull(file, "file is null");
|
||||
checkArgument(file.canRead(), "Can not read file: %s" + file);
|
||||
byte[] json = Files.readAllBytes(file.toPath());
|
||||
Map<String, OptionsJson> options = mapJsonCodec(String.class, OptionsJson.class).fromJson(json);
|
||||
ImmutableList.Builder<Suite> runOptions = ImmutableList.builder();
|
||||
for (Entry<String, OptionsJson> entry : options.entrySet()) {
|
||||
runOptions.add(entry.getValue().toSuite(entry.getKey()));
|
||||
}
|
||||
return runOptions.build();
|
||||
}
|
||||
|
||||
public static class OptionsJson
|
||||
{
|
||||
private final List<String> schema;
|
||||
private final Map<String, String> session;
|
||||
private final List<String> query;
|
||||
|
||||
@JsonCreator
|
||||
public OptionsJson(
|
||||
@JsonProperty("schema") List<String> schema,
|
||||
@JsonProperty("session") Map<String, String> session,
|
||||
@JsonProperty("query") List<String> query)
|
||||
{
|
||||
this.schema = checkNotNull(ImmutableList.copyOf(schema), "schema is null");
|
||||
this.session = checkNotNull(ImmutableMap.copyOf(session), "session is null");
|
||||
this.query = checkNotNull(query, "query is null");
|
||||
}
|
||||
|
||||
public Suite toSuite(String name)
|
||||
{
|
||||
ImmutableList.Builder<Pattern> queryNameTemplates = ImmutableList.builder();
|
||||
for (String q : query) {
|
||||
queryNameTemplates.add(Pattern.compile(q));
|
||||
}
|
||||
ImmutableList.Builder<RegexTemplate> schemaNameTemplates = ImmutableList.builder();
|
||||
for (String s : schema) {
|
||||
schemaNameTemplates.add(new RegexTemplate(s));
|
||||
}
|
||||
return new Suite(name, session, schemaNameTemplates.build(), queryNameTemplates.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark.driver;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
public class RegexTemplateTest
|
||||
{
|
||||
@Test
|
||||
public void test()
|
||||
throws Exception
|
||||
{
|
||||
RegexTemplate regexTemplate = new RegexTemplate("tpch_sf(?<scale>.*?)_(?<format>.*?)_(?<compression>.*?)");
|
||||
|
||||
assertEquals(regexTemplate.getFieldNames(), ImmutableList.of("scale", "format", "compression"));
|
||||
assertEquals(regexTemplate.parse("tpch_sf100_orc_zlib"), Optional.of(ImmutableMap.of("scale", "100", "format", "orc", "compression", "zlib")));
|
||||
assertEquals(regexTemplate.parse("foo_tpch_sf100_orc_zlib"), Optional.empty());
|
||||
assertEquals(regexTemplate.parse("tpch_sf100_orc"), Optional.empty());
|
||||
assertEquals(regexTemplate.parse(""), Optional.empty());
|
||||
|
||||
regexTemplate = new RegexTemplate("tpch_sf(?<scale>.*?)_(?<format>.*?)_(?<compression>.*?)\\.sql");
|
||||
assertEquals(regexTemplate.parse("tpch_sf100_orc_zlib.sql"), Optional.of(ImmutableMap.of("scale", "100", "format", "orc", "compression", "zlib")));
|
||||
assertEquals(regexTemplate.parse("tpch_sf100_orc_zlibXsql"), Optional.empty());
|
||||
assertEquals(regexTemplate.parse("tpch_sf100_orc_zlib.sqlFoo"), Optional.empty());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
{
|
||||
"legacy_readers": {
|
||||
"schema": [ "(?<scale>tiny)", "sf(?<scale>.)" ],
|
||||
"session": {
|
||||
"enable_optimized_readers": "false"
|
||||
},
|
||||
"query": ["sum_quantity.*", "sum_discount.*"]
|
||||
},
|
||||
"optimized_readers": {
|
||||
"schema": [ "(?<scale>tiny)", "sf(?<scale>.)" ],
|
||||
"session": {
|
||||
"enable_optimized_readers": "true"
|
||||
},
|
||||
"query": ["sum_quantity.*", "sum_discount.*"]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
format=generate
|
||||
scale=1
|
||||
=====
|
||||
select count(discount)
|
||||
from tpch_sf1_rcbinary_lineitem
|
||||
@ -0,0 +1,4 @@
|
||||
format=generate
|
||||
=====
|
||||
select count(discount)
|
||||
from lineitem
|
||||
@ -0,0 +1,4 @@
|
||||
format=generate
|
||||
=====
|
||||
select count(quantity)
|
||||
from lineitem
|
||||
@ -0,0 +1,95 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<version>0.107</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-benchmark</artifactId>
|
||||
<name>presto-benchmark</name>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-spi</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-main</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.facebook.presto</groupId>
|
||||
<artifactId>presto-tpch</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>units</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jetbrains</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- for benchmark run -->
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>bootstrap</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.ws.rs</groupId>
|
||||
<artifactId>javax.ws.rs-api</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- for testing -->
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@ -0,0 +1,140 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import io.airlift.units.DataSize;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.benchmark.FormatUtils.formatCount;
|
||||
import static com.facebook.presto.benchmark.FormatUtils.formatCountRate;
|
||||
import static com.facebook.presto.benchmark.FormatUtils.formatDataRate;
|
||||
import static com.facebook.presto.benchmark.FormatUtils.formatDataSize;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static io.airlift.units.DataSize.Unit.BYTE;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
|
||||
public abstract class AbstractBenchmark
|
||||
{
|
||||
private final String benchmarkName;
|
||||
private final int warmupIterations;
|
||||
private final int measuredIterations;
|
||||
|
||||
protected AbstractBenchmark(String benchmarkName, int warmupIterations, int measuredIterations)
|
||||
{
|
||||
checkNotNull(benchmarkName, "benchmarkName is null");
|
||||
checkArgument(warmupIterations >= 0, "warmupIterations must not be negative");
|
||||
checkArgument(measuredIterations >= 0, "measuredIterations must not be negative");
|
||||
|
||||
this.benchmarkName = benchmarkName;
|
||||
this.warmupIterations = warmupIterations;
|
||||
this.measuredIterations = measuredIterations;
|
||||
}
|
||||
|
||||
public String getBenchmarkName()
|
||||
{
|
||||
return benchmarkName;
|
||||
}
|
||||
|
||||
protected int getWarmupIterations()
|
||||
{
|
||||
return warmupIterations;
|
||||
}
|
||||
|
||||
protected int getMeasuredIterations()
|
||||
{
|
||||
return measuredIterations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize any state necessary to run benchmark. This is run once at start up.
|
||||
*/
|
||||
protected void setUp()
|
||||
{
|
||||
// Default: no-op
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the benchmark and returns the result metrics
|
||||
*/
|
||||
protected abstract Map<String, Long> runOnce();
|
||||
|
||||
/**
|
||||
* Clean up any state from the benchmark. This is run once after all the iterations are complete.
|
||||
*/
|
||||
protected void tearDown()
|
||||
{
|
||||
// Default: no-op
|
||||
}
|
||||
|
||||
public void runBenchmark()
|
||||
{
|
||||
runBenchmark(null);
|
||||
}
|
||||
|
||||
public void runBenchmark(@Nullable BenchmarkResultHook benchmarkResultHook)
|
||||
{
|
||||
AverageBenchmarkResults averageBenchmarkResults = new AverageBenchmarkResults();
|
||||
setUp();
|
||||
try {
|
||||
for (int i = 0; i < warmupIterations; i++) {
|
||||
runOnce();
|
||||
}
|
||||
for (int i = 0; i < measuredIterations; i++) {
|
||||
Map<String, Long> results = runOnce();
|
||||
if (benchmarkResultHook != null) {
|
||||
benchmarkResultHook.addResults(results);
|
||||
}
|
||||
averageBenchmarkResults.addResults(results);
|
||||
}
|
||||
}
|
||||
catch (Throwable t) {
|
||||
throw new RuntimeException("Exception in " + getBenchmarkName(), t);
|
||||
}
|
||||
finally {
|
||||
tearDown();
|
||||
}
|
||||
if (benchmarkResultHook != null) {
|
||||
benchmarkResultHook.finished();
|
||||
}
|
||||
|
||||
Map<String, Double> resultsAvg = averageBenchmarkResults.getAverageResultsValues();
|
||||
Duration cpuNanos = new Duration(resultsAvg.get("cpu_nanos"), NANOSECONDS);
|
||||
|
||||
long inputRows = resultsAvg.get("input_rows").longValue();
|
||||
DataSize inputBytes = new DataSize(resultsAvg.get("input_bytes"), BYTE);
|
||||
|
||||
long outputRows = resultsAvg.get("output_rows").longValue();
|
||||
DataSize outputBytes = new DataSize(resultsAvg.get("output_bytes"), BYTE);
|
||||
|
||||
System.out.printf("%35s :: %8.3f cpu ms :: in %5s, %6s, %8s, %8s :: out %5s, %6s, %8s, %8s%n",
|
||||
getBenchmarkName(),
|
||||
cpuNanos.getValue(MILLISECONDS),
|
||||
|
||||
formatCount(inputRows),
|
||||
formatDataSize(inputBytes, true),
|
||||
formatCountRate(inputRows, cpuNanos, true),
|
||||
formatDataRate(inputBytes, cpuNanos, true),
|
||||
|
||||
formatCount(outputRows),
|
||||
formatDataSize(outputBytes, true),
|
||||
formatCountRate(outputRows, cpuNanos, true),
|
||||
formatDataRate(outputBytes, cpuNanos, true));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.Session;
|
||||
import com.facebook.presto.execution.TaskId;
|
||||
import com.facebook.presto.execution.TaskStateMachine;
|
||||
import com.facebook.presto.memory.MemoryPool;
|
||||
import com.facebook.presto.memory.MemoryPoolId;
|
||||
import com.facebook.presto.memory.QueryContext;
|
||||
import com.facebook.presto.operator.Driver;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.operator.TaskContext;
|
||||
import com.facebook.presto.operator.TaskStats;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.util.CpuTimer;
|
||||
import com.facebook.presto.util.CpuTimer.CpuDuration;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.airlift.units.DataSize;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static io.airlift.units.DataSize.Unit.BYTE;
|
||||
import static io.airlift.units.DataSize.Unit.GIGABYTE;
|
||||
import static io.airlift.units.DataSize.Unit.MEGABYTE;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
|
||||
/**
|
||||
* Abstract template for benchmarks that want to test the performance of an Operator.
|
||||
*/
|
||||
public abstract class AbstractOperatorBenchmark
|
||||
extends AbstractBenchmark
|
||||
{
|
||||
protected final LocalQueryRunner localQueryRunner;
|
||||
|
||||
protected AbstractOperatorBenchmark(
|
||||
LocalQueryRunner localQueryRunner,
|
||||
String benchmarkName,
|
||||
int warmupIterations,
|
||||
int measuredIterations)
|
||||
{
|
||||
super(benchmarkName, warmupIterations, measuredIterations);
|
||||
this.localQueryRunner = checkNotNull(localQueryRunner, "localQueryRunner is null");
|
||||
}
|
||||
|
||||
protected OperatorFactory createTableScanOperator(int operatorId, String tableName, String... columnNames)
|
||||
{
|
||||
return localQueryRunner.createTableScanOperator(operatorId, tableName, columnNames);
|
||||
}
|
||||
|
||||
protected OperatorFactory createHashProjectOperator(int operatorId, List<Type> types)
|
||||
{
|
||||
return localQueryRunner.createHashProjectOperator(operatorId, types);
|
||||
}
|
||||
|
||||
protected abstract List<Driver> createDrivers(TaskContext taskContext);
|
||||
|
||||
protected void execute(TaskContext taskContext)
|
||||
{
|
||||
List<Driver> drivers = createDrivers(taskContext);
|
||||
|
||||
boolean done = false;
|
||||
while (!done) {
|
||||
boolean processed = false;
|
||||
for (Driver driver : drivers) {
|
||||
if (!driver.isFinished()) {
|
||||
driver.process();
|
||||
processed = true;
|
||||
}
|
||||
}
|
||||
done = !processed;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Long> runOnce()
|
||||
{
|
||||
Session session = Session.builder()
|
||||
.setUser("user")
|
||||
.setSource("source")
|
||||
.setCatalog("catalog")
|
||||
.setSchema("schema")
|
||||
.setTimeZoneKey(UTC_KEY)
|
||||
.setLocale(ENGLISH)
|
||||
.setSystemProperties(ImmutableMap.of("optimizer.optimize-hash-generation", "true"))
|
||||
.build();
|
||||
ExecutorService executor = localQueryRunner.getExecutor();
|
||||
MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE), false);
|
||||
TaskContext taskContext = new QueryContext(false, new DataSize(256, MEGABYTE), memoryPool, executor)
|
||||
.addTaskContext(new TaskStateMachine(new TaskId("query", "stage", "task"), executor),
|
||||
session,
|
||||
new DataSize(256, MEGABYTE),
|
||||
new DataSize(1, MEGABYTE),
|
||||
false,
|
||||
false);
|
||||
|
||||
CpuTimer cpuTimer = new CpuTimer();
|
||||
execute(taskContext);
|
||||
CpuDuration executionTime = cpuTimer.elapsedTime();
|
||||
|
||||
TaskStats taskStats = taskContext.getTaskStats();
|
||||
long inputRows = taskStats.getRawInputPositions();
|
||||
long inputBytes = taskStats.getRawInputDataSize().toBytes();
|
||||
long outputRows = taskStats.getOutputPositions();
|
||||
long outputBytes = taskStats.getOutputDataSize().toBytes();
|
||||
|
||||
double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
|
||||
|
||||
return ImmutableMap.<String, Long>builder()
|
||||
// legacy computed values
|
||||
.put("elapsed_millis", executionTime.getWall().toMillis())
|
||||
.put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS)))
|
||||
.put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS)))
|
||||
.put("input_megabytes", (long) inputMegaBytes)
|
||||
.put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS)))
|
||||
|
||||
.put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS))
|
||||
.put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS))
|
||||
.put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS))
|
||||
.put("input_rows", inputRows)
|
||||
.put("input_bytes", inputBytes)
|
||||
.put("output_rows", outputRows)
|
||||
.put("output_bytes", outputBytes)
|
||||
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.Driver;
|
||||
import com.facebook.presto.operator.DriverContext;
|
||||
import com.facebook.presto.operator.DriverFactory;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.operator.TaskContext;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.testing.NullOutputOperator.NullOutputOperatorFactory;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class AbstractSimpleOperatorBenchmark
|
||||
extends AbstractOperatorBenchmark
|
||||
{
|
||||
protected AbstractSimpleOperatorBenchmark(
|
||||
LocalQueryRunner localQueryRunner,
|
||||
String benchmarkName,
|
||||
int warmupIterations,
|
||||
int measuredIterations)
|
||||
{
|
||||
super(localQueryRunner, benchmarkName, warmupIterations, measuredIterations);
|
||||
}
|
||||
|
||||
protected abstract List<? extends OperatorFactory> createOperatorFactories();
|
||||
|
||||
protected DriverFactory createDriverFactory()
|
||||
{
|
||||
List<OperatorFactory> operatorFactories = new ArrayList<>(createOperatorFactories());
|
||||
|
||||
operatorFactories.add(new NullOutputOperatorFactory(999, Iterables.getLast(operatorFactories).getTypes()));
|
||||
|
||||
return new DriverFactory(true, true, operatorFactories);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Driver> createDrivers(TaskContext taskContext)
|
||||
{
|
||||
DriverFactory driverFactory = createDriverFactory();
|
||||
DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();
|
||||
Driver driver = driverFactory.createDriver(driverContext);
|
||||
return ImmutableList.of(driver);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.Driver;
|
||||
import com.facebook.presto.operator.TaskContext;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.testing.NullOutputOperator.NullOutputFactory;
|
||||
import org.intellij.lang.annotations.Language;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public abstract class AbstractSqlBenchmark
|
||||
extends AbstractOperatorBenchmark
|
||||
{
|
||||
@Language("SQL")
|
||||
private final String query;
|
||||
|
||||
protected AbstractSqlBenchmark(
|
||||
LocalQueryRunner localQueryRunner,
|
||||
String benchmarkName,
|
||||
int warmupIterations,
|
||||
int measuredIterations,
|
||||
@Language("SQL") String query)
|
||||
{
|
||||
super(localQueryRunner, benchmarkName, warmupIterations, measuredIterations);
|
||||
this.query = query;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Driver> createDrivers(TaskContext taskContext)
|
||||
{
|
||||
return localQueryRunner.createDrivers(query, new NullOutputFactory(), taskContext);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
|
||||
public abstract class ArrayComparisonBenchmark
|
||||
{
|
||||
public static void main(String... args)
|
||||
{
|
||||
LocalQueryRunner localQueryRunner = createLocalQueryRunner();
|
||||
new ArrayEqualsBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
new ArrayLessThanBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
new ArrayGreaterThanBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
new ArrayNotEqualBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
new ArrayLessThanOrEqualBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
new ArrayGreaterThanOrEqualBenchmark(localQueryRunner).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
|
||||
public static class ArrayEqualsBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayEqualsBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_equals", 5, 50, "SELECT COUNT_IF(ARRAY [orderkey, orderkey + 1, orderkey + 2] = ARRAY[orderkey + 1, orderkey + 2, orderkey + 3]) FROM orders");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ArrayLessThanBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayLessThanBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_less_than", 5, 50, "SELECT COUNT_IF(ARRAY [quantity, quantity + 5] < ARRAY[quantity, quantity + 5, quantity + 3]) FROM lineitem");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ArrayGreaterThanBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayGreaterThanBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_greater_than", 5, 50, "SELECT COUNT_IF(ARRAY [quantity, quantity + 6] > ARRAY[quantity, quantity + 5, quantity + 3]) FROM lineitem");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ArrayNotEqualBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayNotEqualBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_not_equal", 5, 50, "SELECT COUNT_IF(ARRAY [orderkey, orderkey + 1, orderkey + 2] != ARRAY[orderkey + 1, orderkey + 2, orderkey + 3]) FROM orders");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ArrayLessThanOrEqualBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayLessThanOrEqualBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_less_than_or_equal", 5, 50, "SELECT COUNT_IF(ARRAY [quantity, quantity + 5, quantity + 2] <= ARRAY[quantity, quantity + 5, quantity + 3]) FROM lineitem");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ArrayGreaterThanOrEqualBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public ArrayGreaterThanOrEqualBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "array_greater_than_or_equal", 5, 50, "SELECT COUNT_IF(ARRAY [quantity, quantity + 6] >= ARRAY[quantity, quantity + 5, quantity + 3]) FROM lineitem");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class AverageBenchmarkResults
|
||||
implements BenchmarkResultHook
|
||||
{
|
||||
private final Map<String, Long> resultsSum = new LinkedHashMap<>();
|
||||
private int resultsCount;
|
||||
|
||||
@Override
|
||||
public BenchmarkResultHook addResults(Map<String, Long> results)
|
||||
{
|
||||
checkNotNull(results, "results is null");
|
||||
for (Entry<String, Long> entry : results.entrySet()) {
|
||||
Long currentSum = resultsSum.get(entry.getKey());
|
||||
if (currentSum == null) {
|
||||
currentSum = 0L;
|
||||
}
|
||||
resultsSum.put(entry.getKey(), currentSum + entry.getValue());
|
||||
}
|
||||
resultsCount++;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public Map<String, Double> getAverageResultsValues()
|
||||
{
|
||||
return Maps.transformValues(resultsSum, input -> 1.0 * input / resultsCount);
|
||||
}
|
||||
|
||||
public Map<String, String> getAverageResultsStrings()
|
||||
{
|
||||
return Maps.transformValues(resultsSum, input -> String.format("%,3.2f", 1.0 * input / resultsCount));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finished()
|
||||
{
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.Session;
|
||||
import com.facebook.presto.metadata.InMemoryNodeManager;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.tpch.TpchConnectorFactory;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import static com.facebook.presto.Session.SessionBuilder;
|
||||
import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY;
|
||||
import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public final class BenchmarkQueryRunner
|
||||
{
|
||||
private BenchmarkQueryRunner()
|
||||
{
|
||||
}
|
||||
|
||||
public static LocalQueryRunner createLocalQueryRunnerHashEnabled()
|
||||
{
|
||||
return createLocalQueryRunner(true);
|
||||
}
|
||||
|
||||
public static LocalQueryRunner createLocalQueryRunner()
|
||||
{
|
||||
return createLocalQueryRunner(false);
|
||||
}
|
||||
public static LocalQueryRunner createLocalQueryRunner(boolean hashingEnabled)
|
||||
{
|
||||
SessionBuilder sessionBuilder = Session
|
||||
.builder()
|
||||
.setUser("user")
|
||||
.setSource("test")
|
||||
.setCatalog("tpch")
|
||||
.setSchema(TINY_SCHEMA_NAME)
|
||||
.setTimeZoneKey(UTC_KEY)
|
||||
.setLocale(ENGLISH);
|
||||
|
||||
if (hashingEnabled) {
|
||||
sessionBuilder.setSystemProperties(ImmutableMap.of("optimizer.optimize_hash_generation", "true"));
|
||||
}
|
||||
|
||||
Session session = sessionBuilder.build();
|
||||
LocalQueryRunner localQueryRunner = new LocalQueryRunner(session);
|
||||
|
||||
// add tpch
|
||||
InMemoryNodeManager nodeManager = localQueryRunner.getNodeManager();
|
||||
localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(nodeManager, 1), ImmutableMap.<String, String>of());
|
||||
|
||||
return localQueryRunner;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public interface BenchmarkResultHook
|
||||
{
|
||||
BenchmarkResultHook addResults(Map<String, Long> results);
|
||||
|
||||
void finished();
|
||||
}
|
||||
@ -0,0 +1,179 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.io.Files;
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.facebook.presto.testing.LocalQueryRunner.createHashEnabledQueryRunner;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
public class BenchmarkSuite
|
||||
{
|
||||
private static final Logger LOGGER = Logger.get(BenchmarkSuite.class);
|
||||
|
||||
public static List<AbstractBenchmark> createBenchmarks(LocalQueryRunner localQueryRunner, LocalQueryRunner hashEnabledLocalQueryRunner)
|
||||
{
|
||||
return ImmutableList.<AbstractBenchmark>of(
|
||||
// hand built benchmarks
|
||||
new CountAggregationBenchmark(localQueryRunner),
|
||||
new DoubleSumAggregationBenchmark(localQueryRunner),
|
||||
new HashAggregationBenchmark(localQueryRunner),
|
||||
new PredicateFilterBenchmark(localQueryRunner),
|
||||
new RawStreamingBenchmark(localQueryRunner),
|
||||
new Top100Benchmark(localQueryRunner),
|
||||
new OrderByBenchmark(localQueryRunner),
|
||||
new HashBuildBenchmark(localQueryRunner),
|
||||
new HashJoinBenchmark(localQueryRunner),
|
||||
new HashBuildAndJoinBenchmark(localQueryRunner),
|
||||
new HashBuildAndJoinBenchmark(hashEnabledLocalQueryRunner),
|
||||
new HandTpchQuery1(localQueryRunner),
|
||||
new HandTpchQuery6(localQueryRunner),
|
||||
|
||||
// sql benchmarks
|
||||
new GroupBySumWithArithmeticSqlBenchmark(localQueryRunner),
|
||||
new CountAggregationSqlBenchmark(localQueryRunner),
|
||||
new SqlDoubleSumAggregationBenchmark(localQueryRunner),
|
||||
new CountWithFilterSqlBenchmark(localQueryRunner),
|
||||
new GroupByAggregationSqlBenchmark(localQueryRunner),
|
||||
new PredicateFilterSqlBenchmark(localQueryRunner),
|
||||
new RawStreamingSqlBenchmark(localQueryRunner),
|
||||
new Top100SqlBenchmark(localQueryRunner),
|
||||
new SqlHashJoinBenchmark(localQueryRunner),
|
||||
new SqlJoinWithPredicateBenchmark(localQueryRunner),
|
||||
new LongMaxAggregationSqlBenchmark(localQueryRunner),
|
||||
new VarBinaryMaxAggregationSqlBenchmark(localQueryRunner),
|
||||
new SqlDistinctMultipleFields(localQueryRunner),
|
||||
new SqlDistinctSingleField(localQueryRunner),
|
||||
new SqlTpchQuery1(localQueryRunner),
|
||||
new SqlTpchQuery6(localQueryRunner),
|
||||
new SqlLikeBenchmark(localQueryRunner),
|
||||
new SqlInBenchmark(localQueryRunner),
|
||||
new SqlSemiJoinInPredicateBenchmark(localQueryRunner),
|
||||
new SqlRegexpLikeBenchmark(localQueryRunner),
|
||||
new SqlApproximatePercentileBenchmark(localQueryRunner),
|
||||
new SqlBetweenBenchmark(localQueryRunner),
|
||||
|
||||
// statistics benchmarks
|
||||
new StatisticsBenchmark.LongVarianceBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.LongVariancePopBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.DoubleVarianceBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.DoubleVariancePopBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.LongStdDevBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.LongStdDevPopBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.DoubleStdDevBenchmark(localQueryRunner),
|
||||
new StatisticsBenchmark.DoubleStdDevPopBenchmark(localQueryRunner),
|
||||
|
||||
// array comparison benchmarks
|
||||
new ArrayComparisonBenchmark.ArrayEqualsBenchmark(localQueryRunner),
|
||||
new ArrayComparisonBenchmark.ArrayLessThanBenchmark(localQueryRunner),
|
||||
new ArrayComparisonBenchmark.ArrayGreaterThanBenchmark(localQueryRunner),
|
||||
new ArrayComparisonBenchmark.ArrayNotEqualBenchmark(localQueryRunner),
|
||||
new ArrayComparisonBenchmark.ArrayLessThanOrEqualBenchmark(localQueryRunner),
|
||||
new ArrayComparisonBenchmark.ArrayGreaterThanOrEqualBenchmark(localQueryRunner),
|
||||
|
||||
new SqlApproximateCountDistinctLongBenchmark(localQueryRunner),
|
||||
new SqlApproximateCountDistinctDoubleBenchmark(localQueryRunner),
|
||||
new SqlApproximateCountDistinctVarBinaryBenchmark(localQueryRunner)
|
||||
);
|
||||
}
|
||||
|
||||
private final LocalQueryRunner localQueryRunner;
|
||||
private final LocalQueryRunner hashEnabledLocalQueryRunner;
|
||||
private final String outputDirectory;
|
||||
|
||||
public BenchmarkSuite(LocalQueryRunner localQueryRunner, String outputDirectory)
|
||||
{
|
||||
this.localQueryRunner = localQueryRunner;
|
||||
this.hashEnabledLocalQueryRunner = createHashEnabledQueryRunner(localQueryRunner);
|
||||
this.outputDirectory = checkNotNull(outputDirectory, "outputDirectory is null");
|
||||
}
|
||||
|
||||
private static File createOutputFile(String fileName)
|
||||
throws IOException
|
||||
{
|
||||
File outputFile = new File(fileName);
|
||||
Files.createParentDirs(outputFile);
|
||||
return outputFile;
|
||||
}
|
||||
|
||||
public void runAllBenchmarks()
|
||||
throws IOException
|
||||
{
|
||||
List<AbstractBenchmark> benchmarks = createBenchmarks(localQueryRunner, hashEnabledLocalQueryRunner);
|
||||
|
||||
LOGGER.info("=== Pre-running all benchmarks for JVM warmup ===");
|
||||
for (AbstractBenchmark benchmark : benchmarks) {
|
||||
benchmark.runBenchmark();
|
||||
}
|
||||
|
||||
LOGGER.info("=== Actually running benchmarks for metrics ===");
|
||||
for (AbstractBenchmark benchmark : benchmarks) {
|
||||
try (OutputStream jsonOut = new FileOutputStream(createOutputFile(String.format("%s/json/%s.json", outputDirectory, benchmark.getBenchmarkName())));
|
||||
OutputStream jsonAvgOut = new FileOutputStream(createOutputFile(String.format("%s/json-avg/%s.json", outputDirectory, benchmark.getBenchmarkName())));
|
||||
OutputStream csvOut = new FileOutputStream(createOutputFile(String.format("%s/csv/%s.csv", outputDirectory, benchmark.getBenchmarkName())));
|
||||
OutputStream odsOut = new FileOutputStream(createOutputFile(String.format("%s/ods/%s.json", outputDirectory, benchmark.getBenchmarkName())))) {
|
||||
benchmark.runBenchmark(
|
||||
new ForwardingBenchmarkResultWriter(
|
||||
ImmutableList.of(
|
||||
new JsonBenchmarkResultWriter(jsonOut),
|
||||
new JsonAvgBenchmarkResultWriter(jsonAvgOut),
|
||||
new SimpleLineBenchmarkResultWriter(csvOut),
|
||||
new OdsBenchmarkResultWriter("presto.benchmark." + benchmark.getBenchmarkName(), odsOut)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class ForwardingBenchmarkResultWriter
|
||||
implements BenchmarkResultHook
|
||||
{
|
||||
private final List<BenchmarkResultHook> benchmarkResultHooks;
|
||||
|
||||
private ForwardingBenchmarkResultWriter(List<BenchmarkResultHook> benchmarkResultHooks)
|
||||
{
|
||||
checkNotNull(benchmarkResultHooks, "benchmarkResultWriters is null");
|
||||
this.benchmarkResultHooks = ImmutableList.copyOf(benchmarkResultHooks);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BenchmarkResultHook addResults(Map<String, Long> results)
|
||||
{
|
||||
checkNotNull(results, "results is null");
|
||||
for (BenchmarkResultHook benchmarkResultHook : benchmarkResultHooks) {
|
||||
benchmarkResultHook.addResults(results);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finished()
|
||||
{
|
||||
for (BenchmarkResultHook benchmarkResultHook : benchmarkResultHooks) {
|
||||
benchmarkResultHook.finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
import static com.facebook.presto.operator.aggregation.CountAggregation.COUNT;
|
||||
|
||||
public class CountAggregationBenchmark
|
||||
extends AbstractSimpleOperatorBenchmark
|
||||
{
|
||||
public CountAggregationBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "count_agg", 10, 100);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<? extends OperatorFactory> createOperatorFactories()
|
||||
{
|
||||
OperatorFactory tableScanOperator = createTableScanOperator(0, "orders", "orderkey");
|
||||
AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty(), Optional.empty(), 1.0)));
|
||||
return ImmutableList.of(tableScanOperator, aggregationOperator);
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new CountAggregationBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
|
||||
public class CountAggregationSqlBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public CountAggregationSqlBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "sql_count_agg", 10, 100, "select count(*) from orders");
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new CountAggregationSqlBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
|
||||
public class CountWithFilterSqlBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public CountWithFilterSqlBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "sql_count_with_filter", 10, 100, "SELECT count(*) from orders where orderstatus = 'F'");
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new CountWithFilterSqlBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
import static com.facebook.presto.operator.aggregation.DoubleSumAggregation.DOUBLE_SUM;
|
||||
|
||||
public class DoubleSumAggregationBenchmark
|
||||
extends AbstractSimpleOperatorBenchmark
|
||||
{
|
||||
public DoubleSumAggregationBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "double_sum_agg", 10, 100);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<? extends OperatorFactory> createOperatorFactories()
|
||||
{
|
||||
OperatorFactory tableScanOperator = createTableScanOperator(0, "orders", "totalprice");
|
||||
AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, Step.SINGLE, ImmutableList.of(DOUBLE_SUM.bind(ImmutableList.of(0), Optional.empty(), Optional.empty(), 1.0)));
|
||||
return ImmutableList.of(tableScanOperator, aggregationOperator);
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new DoubleSumAggregationBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,147 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import io.airlift.units.DataSize;
|
||||
import io.airlift.units.Duration;
|
||||
|
||||
import java.math.RoundingMode;
|
||||
import java.text.DecimalFormat;
|
||||
|
||||
import static io.airlift.units.DataSize.Unit.BYTE;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
|
||||
// TODO: these should be in airlift
|
||||
final class FormatUtils
|
||||
{
|
||||
private FormatUtils() {}
|
||||
|
||||
public static String formatCount(long count)
|
||||
{
|
||||
double fractional = count;
|
||||
String unit = "";
|
||||
if (fractional > 1000) {
|
||||
fractional /= 1000;
|
||||
unit = "K";
|
||||
}
|
||||
if (fractional > 1000) {
|
||||
fractional /= 1000;
|
||||
unit = "M";
|
||||
}
|
||||
if (fractional > 1000) {
|
||||
fractional /= 1000;
|
||||
unit = "B";
|
||||
}
|
||||
if (fractional > 1000) {
|
||||
fractional /= 1000;
|
||||
unit = "T";
|
||||
}
|
||||
if (fractional > 1000) {
|
||||
fractional /= 1000;
|
||||
unit = "Q";
|
||||
}
|
||||
|
||||
return format("%s%s", getFormat(fractional).format(fractional), unit);
|
||||
}
|
||||
|
||||
public static String formatCountRate(double count, Duration duration, boolean longForm)
|
||||
{
|
||||
double rate = count / duration.getValue(SECONDS);
|
||||
if (Double.isNaN(rate) || Double.isInfinite(rate)) {
|
||||
rate = 0;
|
||||
}
|
||||
|
||||
String rateString = formatCount((long) rate);
|
||||
if (longForm) {
|
||||
if (rateString.endsWith(" ")) {
|
||||
rateString = rateString.substring(0, rateString.length() - 1);
|
||||
}
|
||||
rateString += "/s";
|
||||
}
|
||||
return rateString;
|
||||
}
|
||||
|
||||
public static String formatDataSize(DataSize size, boolean longForm)
|
||||
{
|
||||
double fractional = size.toBytes();
|
||||
String unit = null;
|
||||
if (fractional >= 1024) {
|
||||
fractional /= 1024;
|
||||
unit = "K";
|
||||
}
|
||||
if (fractional >= 1024) {
|
||||
fractional /= 1024;
|
||||
unit = "M";
|
||||
}
|
||||
if (fractional >= 1024) {
|
||||
fractional /= 1024;
|
||||
unit = "G";
|
||||
}
|
||||
if (fractional >= 1024) {
|
||||
fractional /= 1024;
|
||||
unit = "T";
|
||||
}
|
||||
if (fractional >= 1024) {
|
||||
fractional /= 1024;
|
||||
unit = "P";
|
||||
}
|
||||
|
||||
if (unit == null) {
|
||||
unit = "B";
|
||||
}
|
||||
else if (longForm) {
|
||||
unit += "B";
|
||||
}
|
||||
|
||||
return format("%s%s", getFormat(fractional).format(fractional), unit);
|
||||
}
|
||||
|
||||
public static String formatDataRate(DataSize dataSize, Duration duration, boolean longForm)
|
||||
{
|
||||
double rate = dataSize.toBytes() / duration.getValue(SECONDS);
|
||||
if (Double.isNaN(rate) || Double.isInfinite(rate)) {
|
||||
rate = 0;
|
||||
}
|
||||
|
||||
String rateString = formatDataSize(new DataSize(rate, BYTE), false);
|
||||
if (longForm) {
|
||||
if (!rateString.endsWith("B")) {
|
||||
rateString += "B";
|
||||
}
|
||||
rateString += "/s";
|
||||
}
|
||||
return rateString;
|
||||
}
|
||||
|
||||
public static DecimalFormat getFormat(double value)
|
||||
{
|
||||
DecimalFormat format;
|
||||
if (value < 10) {
|
||||
// show up to two decimals to get 3 significant digits
|
||||
format = new DecimalFormat("#.##");
|
||||
}
|
||||
else if (value < 100) {
|
||||
// show up to one decimal to get 3 significant digits
|
||||
format = new DecimalFormat("#.#");
|
||||
}
|
||||
else {
|
||||
// show no decimals -- we have enough digits in the integer part
|
||||
format = new DecimalFormat("#");
|
||||
}
|
||||
|
||||
format.setRoundingMode(RoundingMode.HALF_UP);
|
||||
return format;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
|
||||
public class GroupByAggregationSqlBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public GroupByAggregationSqlBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "sql_groupby_agg", 15, 100, "select orderstatus, sum(totalprice) from orders group by orderstatus");
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new GroupByAggregationSqlBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
|
||||
public class GroupBySumWithArithmeticSqlBenchmark
|
||||
extends AbstractSqlBenchmark
|
||||
{
|
||||
public GroupBySumWithArithmeticSqlBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner,
|
||||
"sql_groupby_agg_with_arithmetic",
|
||||
1,
|
||||
4,
|
||||
"select linestatus, sum(orderkey - partkey) from lineitem group by linestatus");
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new GroupBySumWithArithmeticSqlBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.benchmark.HandTpchQuery1.TpchQuery1Operator.TpchQuery1OperatorFactory;
|
||||
import com.facebook.presto.operator.DriverContext;
|
||||
import com.facebook.presto.operator.HashAggregationOperator.HashAggregationOperatorFactory;
|
||||
import com.facebook.presto.operator.Operator;
|
||||
import com.facebook.presto.operator.OperatorContext;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.spi.Page;
|
||||
import com.facebook.presto.spi.PageBuilder;
|
||||
import com.facebook.presto.spi.block.Block;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.util.DateTimeUtils;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.primitives.Ints;
|
||||
import io.airlift.units.DataSize;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
import static com.facebook.presto.operator.aggregation.AverageAggregations.DOUBLE_AVERAGE;
|
||||
import static com.facebook.presto.operator.aggregation.AverageAggregations.LONG_AVERAGE;
|
||||
import static com.facebook.presto.operator.aggregation.CountAggregation.COUNT;
|
||||
import static com.facebook.presto.operator.aggregation.DoubleSumAggregation.DOUBLE_SUM;
|
||||
import static com.facebook.presto.operator.aggregation.LongSumAggregation.LONG_SUM;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.DateType.DATE;
|
||||
import static com.facebook.presto.spi.type.DoubleType.DOUBLE;
|
||||
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static io.airlift.units.DataSize.Unit.MEGABYTE;
|
||||
|
||||
public class HandTpchQuery1
|
||||
extends AbstractSimpleOperatorBenchmark
|
||||
{
|
||||
public HandTpchQuery1(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "hand_tpch_query_1", 1, 5);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<? extends OperatorFactory> createOperatorFactories()
|
||||
{
|
||||
// select
|
||||
// returnflag,
|
||||
// linestatus,
|
||||
// sum(quantity) as sum_qty,
|
||||
// sum(extendedprice) as sum_base_price,
|
||||
// sum(extendedprice * (1 - discount)) as sum_disc_price,
|
||||
// sum(extendedprice * (1 - discount) * (1 + tax)) as sum_charge,
|
||||
// avg(quantity) as avg_qty,
|
||||
// avg(extendedprice) as avg_price,
|
||||
// avg(discount) as avg_disc,
|
||||
// count(*) as count_order
|
||||
// from
|
||||
// lineitem
|
||||
// where
|
||||
// shipdate <= '1998-09-02'
|
||||
// group by
|
||||
// returnflag,
|
||||
// linestatus
|
||||
// order by
|
||||
// returnflag,
|
||||
// linestatus
|
||||
|
||||
OperatorFactory tableScanOperator = createTableScanOperator(
|
||||
0,
|
||||
"lineitem",
|
||||
"returnflag",
|
||||
"linestatus",
|
||||
"quantity",
|
||||
"extendedprice",
|
||||
"discount",
|
||||
"tax",
|
||||
"shipdate");
|
||||
|
||||
TpchQuery1OperatorFactory tpchQuery1Operator = new TpchQuery1OperatorFactory(1);
|
||||
HashAggregationOperatorFactory aggregationOperator = new HashAggregationOperatorFactory(
|
||||
2,
|
||||
ImmutableList.of(tpchQuery1Operator.getTypes().get(0), tpchQuery1Operator.getTypes().get(1)),
|
||||
Ints.asList(0, 1),
|
||||
Step.SINGLE,
|
||||
ImmutableList.of(
|
||||
LONG_SUM.bind(ImmutableList.of(2), Optional.empty(), Optional.empty(), 1.0),
|
||||
DOUBLE_SUM.bind(ImmutableList.of(3), Optional.empty(), Optional.empty(), 1.0),
|
||||
DOUBLE_SUM.bind(ImmutableList.of(4), Optional.empty(), Optional.empty(), 1.0),
|
||||
LONG_AVERAGE.bind(ImmutableList.of(2), Optional.empty(), Optional.empty(), 1.0),
|
||||
DOUBLE_AVERAGE.bind(ImmutableList.of(5), Optional.empty(), Optional.empty(), 1.0),
|
||||
DOUBLE_AVERAGE.bind(ImmutableList.of(6), Optional.empty(), Optional.empty(), 1.0),
|
||||
COUNT.bind(ImmutableList.of(2), Optional.empty(), Optional.empty(), 1.0)
|
||||
),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
10_000,
|
||||
new DataSize(16, MEGABYTE));
|
||||
|
||||
return ImmutableList.of(tableScanOperator, tpchQuery1Operator, aggregationOperator);
|
||||
}
|
||||
|
||||
public static class TpchQuery1Operator
|
||||
implements com.facebook.presto.operator.Operator // TODO: use import when Java 7 compiler bug is fixed
|
||||
{
|
||||
private static final ImmutableList<Type> TYPES = ImmutableList.<Type>of(
|
||||
VARCHAR,
|
||||
VARCHAR,
|
||||
BIGINT,
|
||||
DOUBLE,
|
||||
DOUBLE,
|
||||
DOUBLE,
|
||||
DOUBLE);
|
||||
|
||||
public static class TpchQuery1OperatorFactory
|
||||
implements OperatorFactory
|
||||
{
|
||||
private final int operatorId;
|
||||
|
||||
public TpchQuery1OperatorFactory(int operatorId)
|
||||
{
|
||||
this.operatorId = operatorId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Type> getTypes()
|
||||
{
|
||||
return TYPES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Operator createOperator(DriverContext driverContext)
|
||||
{
|
||||
OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, TpchQuery1Operator.class.getSimpleName());
|
||||
return new TpchQuery1Operator(operatorContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private final OperatorContext operatorContext;
|
||||
private final PageBuilder pageBuilder;
|
||||
private boolean finishing;
|
||||
|
||||
public TpchQuery1Operator(OperatorContext operatorContext)
|
||||
{
|
||||
this.operatorContext = checkNotNull(operatorContext, "operatorContext is null");
|
||||
this.pageBuilder = new PageBuilder(TYPES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OperatorContext getOperatorContext()
|
||||
{
|
||||
return operatorContext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Type> getTypes()
|
||||
{
|
||||
return TYPES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish()
|
||||
{
|
||||
finishing = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isFinished()
|
||||
{
|
||||
return finishing && pageBuilder.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needsInput()
|
||||
{
|
||||
return !pageBuilder.isFull();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addInput(Page page)
|
||||
{
|
||||
checkNotNull(page, "page is null");
|
||||
checkState(!pageBuilder.isFull(), "Output buffer is full");
|
||||
checkState(!finishing, "Operator is finished");
|
||||
|
||||
filterAndProjectRowOriented(pageBuilder,
|
||||
page.getBlock(0),
|
||||
page.getBlock(1),
|
||||
page.getBlock(2),
|
||||
page.getBlock(3),
|
||||
page.getBlock(4),
|
||||
page.getBlock(5),
|
||||
page.getBlock(6));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page getOutput()
|
||||
{
|
||||
// only return a page if the page buffer isFull or we are finishing and the page buffer has data
|
||||
if (pageBuilder.isFull() || (finishing && !pageBuilder.isEmpty())) {
|
||||
Page page = pageBuilder.build();
|
||||
pageBuilder.reset();
|
||||
return page;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static final int MAX_SHIP_DATE = DateTimeUtils.parseDate("1998-09-02");
|
||||
|
||||
private static void filterAndProjectRowOriented(PageBuilder pageBuilder,
|
||||
Block returnFlagBlock,
|
||||
Block lineStatusBlock,
|
||||
Block quantityBlock,
|
||||
Block extendedPriceBlock,
|
||||
Block discountBlock,
|
||||
Block taxBlock,
|
||||
Block shipDateBlock)
|
||||
{
|
||||
int rows = returnFlagBlock.getPositionCount();
|
||||
for (int position = 0; position < rows; position++) {
|
||||
if (shipDateBlock.isNull(position)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int shipDate = (int) DATE.getLong(shipDateBlock, position);
|
||||
|
||||
// where
|
||||
// shipdate <= '1998-09-02'
|
||||
if (shipDate <= MAX_SHIP_DATE) {
|
||||
// returnflag,
|
||||
// linestatus
|
||||
// quantity
|
||||
// extendedprice
|
||||
// extendedprice * (1 - discount)
|
||||
// extendedprice * (1 - discount) * (1 + tax)
|
||||
// discount
|
||||
|
||||
pageBuilder.declarePosition();
|
||||
if (returnFlagBlock.isNull(position)) {
|
||||
pageBuilder.getBlockBuilder(0).appendNull();
|
||||
}
|
||||
else {
|
||||
VARCHAR.appendTo(returnFlagBlock, position, pageBuilder.getBlockBuilder(0));
|
||||
}
|
||||
if (lineStatusBlock.isNull(position)) {
|
||||
pageBuilder.getBlockBuilder(1).appendNull();
|
||||
}
|
||||
else {
|
||||
VARCHAR.appendTo(lineStatusBlock, position, pageBuilder.getBlockBuilder(1));
|
||||
}
|
||||
|
||||
long quantity = BIGINT.getLong(quantityBlock, position);
|
||||
double extendedPrice = DOUBLE.getDouble(extendedPriceBlock, position);
|
||||
double discount = DOUBLE.getDouble(discountBlock, position);
|
||||
double tax = DOUBLE.getDouble(taxBlock, position);
|
||||
|
||||
boolean quantityIsNull = quantityBlock.isNull(position);
|
||||
boolean extendedPriceIsNull = extendedPriceBlock.isNull(position);
|
||||
boolean discountIsNull = discountBlock.isNull(position);
|
||||
boolean taxIsNull = taxBlock.isNull(position);
|
||||
|
||||
if (quantityIsNull) {
|
||||
pageBuilder.getBlockBuilder(2).appendNull();
|
||||
}
|
||||
else {
|
||||
BIGINT.writeLong(pageBuilder.getBlockBuilder(2), quantity);
|
||||
}
|
||||
|
||||
if (extendedPriceIsNull) {
|
||||
pageBuilder.getBlockBuilder(3).appendNull();
|
||||
}
|
||||
else {
|
||||
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(3), extendedPrice);
|
||||
}
|
||||
|
||||
if (extendedPriceIsNull || discountIsNull) {
|
||||
pageBuilder.getBlockBuilder(4).appendNull();
|
||||
}
|
||||
else {
|
||||
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(4), extendedPrice * (1 - discount));
|
||||
}
|
||||
|
||||
if (extendedPriceIsNull || discountIsNull || taxIsNull) {
|
||||
pageBuilder.getBlockBuilder(5).appendNull();
|
||||
}
|
||||
else {
|
||||
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(5), extendedPrice * (1 - discount) * (1 + tax));
|
||||
}
|
||||
|
||||
if (discountIsNull) {
|
||||
pageBuilder.getBlockBuilder(6).appendNull();
|
||||
}
|
||||
else {
|
||||
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(6), discount);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new HandTpchQuery1(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory;
|
||||
import com.facebook.presto.operator.FilterAndProjectOperator;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.operator.PageProcessor;
|
||||
import com.facebook.presto.spi.ConnectorSession;
|
||||
import com.facebook.presto.spi.Page;
|
||||
import com.facebook.presto.spi.PageBuilder;
|
||||
import com.facebook.presto.spi.block.Block;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.facebook.presto.util.DateTimeUtils;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
import static com.facebook.presto.operator.aggregation.DoubleSumAggregation.DOUBLE_SUM;
|
||||
import static com.facebook.presto.spi.type.BigintType.BIGINT;
|
||||
import static com.facebook.presto.spi.type.DateType.DATE;
|
||||
import static com.facebook.presto.spi.type.DoubleType.DOUBLE;
|
||||
|
||||
public class HandTpchQuery6
|
||||
extends AbstractSimpleOperatorBenchmark
|
||||
{
|
||||
public HandTpchQuery6(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "hand_tpch_query_6", 10, 100);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<? extends OperatorFactory> createOperatorFactories()
|
||||
{
|
||||
// select sum(extendedprice * discount) as revenue
|
||||
// from lineitem
|
||||
// where shipdate >= '1994-01-01'
|
||||
// and shipdate < '1995-01-01'
|
||||
// and discount >= 0.05
|
||||
// and discount <= 0.07
|
||||
// and quantity < 24;
|
||||
OperatorFactory tableScanOperator = createTableScanOperator(0, "lineitem", "extendedprice", "discount", "shipdate", "quantity");
|
||||
|
||||
FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new TpchQuery6Processor(), ImmutableList.<Type>of(DOUBLE));
|
||||
|
||||
AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(
|
||||
2,
|
||||
Step.SINGLE,
|
||||
ImmutableList.of(
|
||||
DOUBLE_SUM.bind(ImmutableList.of(0), Optional.empty(), Optional.empty(), 1.0)
|
||||
));
|
||||
|
||||
return ImmutableList.of(tableScanOperator, tpchQuery6Operator, aggregationOperator);
|
||||
}
|
||||
|
||||
public static class TpchQuery6Processor
|
||||
implements PageProcessor
|
||||
{
|
||||
private static final int MIN_SHIP_DATE = DateTimeUtils.parseDate("1994-01-01");
|
||||
private static final int MAX_SHIP_DATE = DateTimeUtils.parseDate("1995-01-01");
|
||||
|
||||
@Override
|
||||
public int process(ConnectorSession session, Page page, int start, int end, PageBuilder pageBuilder)
|
||||
{
|
||||
Block discountBlock = page.getBlock(1);
|
||||
int position = start;
|
||||
for (; position < end; position++) {
|
||||
// where shipdate >= '1994-01-01'
|
||||
// and shipdate < '1995-01-01'
|
||||
// and discount >= 0.05
|
||||
// and discount <= 0.07
|
||||
// and quantity < 24;
|
||||
if (filter(position, discountBlock, page.getBlock(2), page.getBlock(3))) {
|
||||
project(position, pageBuilder, page.getBlock(0), discountBlock);
|
||||
}
|
||||
}
|
||||
return position;
|
||||
}
|
||||
|
||||
private static void project(int position, PageBuilder pageBuilder, Block extendedPriceBlock, Block discountBlock)
|
||||
{
|
||||
if (discountBlock.isNull(position) || extendedPriceBlock.isNull(position)) {
|
||||
pageBuilder.getBlockBuilder(0).appendNull();
|
||||
}
|
||||
else {
|
||||
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), DOUBLE.getDouble(extendedPriceBlock, position) * DOUBLE.getDouble(discountBlock, position));
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean filter(int position, Block discountBlock, Block shipDateBlock, Block quantityBlock)
|
||||
{
|
||||
return !shipDateBlock.isNull(position) && DATE.getLong(shipDateBlock, position) >= MIN_SHIP_DATE &&
|
||||
!shipDateBlock.isNull(position) && DATE.getLong(shipDateBlock, position) < MAX_SHIP_DATE &&
|
||||
!discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) >= 0.05 &&
|
||||
!discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) <= 0.07 &&
|
||||
!quantityBlock.isNull(position) && BIGINT.getLong(quantityBlock, position) < 24;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new HandTpchQuery6(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.facebook.presto.benchmark;
|
||||
|
||||
import com.facebook.presto.operator.HashAggregationOperator.HashAggregationOperatorFactory;
|
||||
import com.facebook.presto.operator.OperatorFactory;
|
||||
import com.facebook.presto.spi.type.Type;
|
||||
import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
|
||||
import com.facebook.presto.testing.LocalQueryRunner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.primitives.Ints;
|
||||
import io.airlift.units.DataSize;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner;
|
||||
import static com.facebook.presto.operator.aggregation.DoubleSumAggregation.DOUBLE_SUM;
|
||||
import static io.airlift.units.DataSize.Unit.MEGABYTE;
|
||||
|
||||
public class HashAggregationBenchmark
|
||||
extends AbstractSimpleOperatorBenchmark
|
||||
{
|
||||
public HashAggregationBenchmark(LocalQueryRunner localQueryRunner)
|
||||
{
|
||||
super(localQueryRunner, "hash_agg", 5, 25);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<? extends OperatorFactory> createOperatorFactories()
|
||||
{
|
||||
OperatorFactory tableScanOperator = createTableScanOperator(0, "orders", "orderstatus", "totalprice");
|
||||
List<Type> types = ImmutableList.of(tableScanOperator.getTypes().get(0));
|
||||
HashAggregationOperatorFactory aggregationOperator = new HashAggregationOperatorFactory(1,
|
||||
types,
|
||||
Ints.asList(0),
|
||||
Step.SINGLE,
|
||||
ImmutableList.of(DOUBLE_SUM.bind(ImmutableList.of(1), Optional.empty(), Optional.empty(), 1.0)),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
100_000,
|
||||
new DataSize(16, MEGABYTE));
|
||||
return ImmutableList.of(tableScanOperator, aggregationOperator);
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
new HashAggregationBenchmark(createLocalQueryRunner()).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue