parent
7deae2d660
commit
9261136829
@ -0,0 +1,3 @@
|
||||
*.js linguist-language=java
|
||||
*.css linguist-language=java
|
||||
*.html linguist-language=java
|
||||
@ -0,0 +1,9 @@
|
||||
target
|
||||
*.iml
|
||||
out/
|
||||
.idea
|
||||
.classpath
|
||||
.project
|
||||
.settings
|
||||
bin/
|
||||
.myeclipse
|
||||
@ -0,0 +1,13 @@
|
||||
FROM java:8
|
||||
|
||||
MAINTAINER octopus
|
||||
|
||||
RUN mkdir -p /spider-flow
|
||||
|
||||
WORKDIR /spider-flow
|
||||
|
||||
EXPOSE 8088
|
||||
|
||||
ADD ./spider-flow-web/target/spider-flow.jar ./
|
||||
|
||||
CMD sleep 30;java -Djava.security.egd=file:/dev/./urandom -jar spider-flow.jar
|
||||
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 小东
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,174 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow</artifactId>
|
||||
<version>0.5.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>spider-flow</name>
|
||||
<url>https://gitee.com/jmxd/spider-flow</url>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.0.7.RELEASE</version>
|
||||
</parent>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<spider-flow.version>${project.version}</spider-flow.version>
|
||||
<alibaba.fastjson.version>1.2.58</alibaba.fastjson.version>
|
||||
<alibaba.druid.version>1.1.16</alibaba.druid.version>
|
||||
<alibaba.transmittable.version>2.11.2</alibaba.transmittable.version>
|
||||
<mybatis.plus.version>3.1.0</mybatis.plus.version>
|
||||
<apache.commons.text.verion>1.6</apache.commons.text.verion>
|
||||
<apache.commons.csv.verion>1.8</apache.commons.csv.verion>
|
||||
<commons.io.version>2.6</commons.io.version>
|
||||
<guava.version>28.2-jre</guava.version>
|
||||
<jsoup.version>1.11.3</jsoup.version>
|
||||
<xsoup.version>0.3.1</xsoup.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- spring-boot相关配置 -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-quartz</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-mail</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-websocket</artifactId>
|
||||
</dependency>
|
||||
<!-- 数据库相关 -->
|
||||
<dependency>
|
||||
<groupId>com.baomidou</groupId>
|
||||
<artifactId>mybatis-plus-boot-starter</artifactId>
|
||||
<version>${mybatis.plus.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
</dependency>
|
||||
<!-- alibaba相关包 -->
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
<version>${alibaba.fastjson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>druid-spring-boot-starter</artifactId>
|
||||
<version>${alibaba.druid.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>transmittable-thread-local</artifactId>
|
||||
<version>${alibaba.transmittable.version}</version>
|
||||
</dependency>
|
||||
<!-- apache commons相关 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<version>${apache.commons.text.verion}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
<version>${apache.commons.csv.verion}</version>
|
||||
</dependency>
|
||||
<!-- commons包 -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>${commons.io.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</dependency>
|
||||
<!-- 其它包 -->
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>${guava.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.jsoup</groupId>
|
||||
<artifactId>jsoup</artifactId>
|
||||
<version>${jsoup.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>us.codecraft</groupId>
|
||||
<artifactId>xsoup</artifactId>
|
||||
<version>${xsoup.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-api</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-core</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-selenium</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-proxypool</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-mongodb</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-redis</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-ocr</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-oss</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-mailbox</artifactId>
|
||||
<version>${spider-flow.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<modules>
|
||||
<module>spider-flow-api</module>
|
||||
<module>spider-flow-core</module>
|
||||
<module>spider-flow-web</module>
|
||||
</modules>
|
||||
</project>
|
||||
@ -0,0 +1,15 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow</artifactId>
|
||||
<version>0.5.0</version>
|
||||
</parent>
|
||||
<artifactId>spider-flow-api</artifactId>
|
||||
<name>spider-flow-api</name>
|
||||
<url>https://gitee.com/jmxd/spider-flow/tree/master/spider-flow-api</url>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
</project>
|
||||
@ -0,0 +1,18 @@
|
||||
package org.spiderflow;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 表达式引擎
|
||||
*/
|
||||
public interface ExpressionEngine {
|
||||
|
||||
/**
|
||||
* 执行表达式
|
||||
* @param expression 表达式
|
||||
* @param variables 变量
|
||||
* @return
|
||||
*/
|
||||
Object execute(String expression, Map<String, Object> variables);
|
||||
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package org.spiderflow;
|
||||
|
||||
import java.util.List;
|
||||
import org.spiderflow.model.Grammer;
|
||||
|
||||
public interface Grammerable {
|
||||
|
||||
List<Grammer> grammers();
|
||||
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
package org.spiderflow.common;
|
||||
|
||||
import org.spiderflow.model.JsonBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
|
||||
public abstract class CURDController<S extends ServiceImpl<M, T>,M extends BaseMapper<T>, T> {
|
||||
|
||||
@Autowired
|
||||
private S service;
|
||||
|
||||
@RequestMapping("/list")
|
||||
public IPage<T> list(@RequestParam(name = "page",defaultValue = "1")Integer page, @RequestParam(name = "limit",defaultValue = "1")Integer size){
|
||||
return service.page(new Page<T>(page, size), new QueryWrapper<T>().orderByDesc("create_date"));
|
||||
}
|
||||
|
||||
@RequestMapping("get")
|
||||
public JsonBean<T> get(String id) {
|
||||
return new JsonBean<T>(service.getById(id));
|
||||
}
|
||||
|
||||
@RequestMapping("delete")
|
||||
public JsonBean<Boolean> delete(String id){
|
||||
return new JsonBean<Boolean>(service.removeById(id));
|
||||
}
|
||||
|
||||
@RequestMapping("save")
|
||||
public JsonBean<Boolean> save(T t){
|
||||
return new JsonBean<Boolean>(service.saveOrUpdate(t));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
public class ChildPriorThreadSubmitStrategy implements ThreadSubmitStrategy{
|
||||
|
||||
private Object mutex = this;
|
||||
|
||||
private Comparator<SpiderNode> comparator = (o1, o2) -> {
|
||||
if(o1.hasLeftNode(o2.getNodeId())){
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
};
|
||||
|
||||
private PriorityQueue<SpiderFutureTask<?>> priorityQueue = new PriorityQueue<>((o1, o2) -> comparator.compare(o1.getNode(),o2.getNode()));
|
||||
|
||||
@Override
|
||||
public Comparator<SpiderNode> comparator() {
|
||||
return comparator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(SpiderFutureTask<?> task) {
|
||||
synchronized (mutex){
|
||||
priorityQueue.add(task);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
synchronized (mutex){
|
||||
return priorityQueue.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SpiderFutureTask<?> get() {
|
||||
synchronized (mutex){
|
||||
return priorityQueue.poll();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
public class LinkedThreadSubmitStrategy implements ThreadSubmitStrategy{
|
||||
|
||||
private List<SpiderFutureTask<?>> taskList = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public Comparator<SpiderNode> comparator() {
|
||||
return (o1, o2) -> -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(SpiderFutureTask<?> task) {
|
||||
taskList.add(task);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return taskList.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public SpiderFutureTask<?> get() {
|
||||
return taskList.remove(0);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
public class ParentPriorThreadSubmitStrategy implements ThreadSubmitStrategy {
|
||||
|
||||
private Object mutex = this;
|
||||
|
||||
private Comparator<SpiderNode> comparator = (o1, o2) -> {
|
||||
if (o1.hasLeftNode(o2.getNodeId())) {
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
};
|
||||
|
||||
private PriorityQueue<SpiderFutureTask<?>> priorityQueue = new PriorityQueue<>((o1, o2) -> comparator.compare(o1.getNode(), o2.getNode()));
|
||||
|
||||
@Override
|
||||
public Comparator<SpiderNode> comparator() {
|
||||
return comparator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(SpiderFutureTask<?> task) {
|
||||
synchronized (mutex) {
|
||||
priorityQueue.add(task);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
synchronized (mutex) {
|
||||
return priorityQueue.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SpiderFutureTask<?> get() {
|
||||
synchronized (mutex) {
|
||||
return priorityQueue.poll();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
public class RandomThreadSubmitStrategy implements ThreadSubmitStrategy{
|
||||
|
||||
private List<SpiderFutureTask<?>> taskList = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public Comparator<SpiderNode> comparator() {
|
||||
return (o1, o2) -> RandomUtils.nextInt(0,3) - 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(SpiderFutureTask<?> task) {
|
||||
taskList.add(task);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return taskList.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public SpiderFutureTask<?> get() {
|
||||
return taskList.remove(RandomUtils.nextInt(0, taskList.size()));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import java.util.concurrent.FutureTask;
|
||||
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
public class SpiderFutureTask<V> extends FutureTask {
|
||||
|
||||
private SubThreadPoolExecutor executor;
|
||||
|
||||
private SpiderNode node;
|
||||
|
||||
public SpiderFutureTask(Runnable runnable, V result, SpiderNode node,SubThreadPoolExecutor executor) {
|
||||
super(runnable,result);
|
||||
this.executor = executor;
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public SubThreadPoolExecutor getExecutor() {
|
||||
return executor;
|
||||
}
|
||||
|
||||
public SpiderNode getNode() {
|
||||
return node;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.spiderflow.concurrent;
|
||||
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
|
||||
import java.util.Comparator;
|
||||
|
||||
public interface ThreadSubmitStrategy {
|
||||
|
||||
Comparator<SpiderNode> comparator();
|
||||
|
||||
void add(SpiderFutureTask<?> task);
|
||||
|
||||
boolean isEmpty();
|
||||
|
||||
SpiderFutureTask<?> get();
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package org.spiderflow.context;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
/**
|
||||
* Cookie上下文
|
||||
*/
|
||||
public class CookieContext extends HashMap<String, String> {
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
package org.spiderflow.context;
|
||||
|
||||
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.spiderflow.model.SpiderOutput;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* 爬虫上下文
|
||||
* @author jmxd
|
||||
*
|
||||
*/
|
||||
public class SpiderContext extends HashMap<String, Object>{
|
||||
|
||||
private String id = UUID.randomUUID().toString().replace("-", "");
|
||||
|
||||
/**
|
||||
* 流程ID
|
||||
*/
|
||||
private String flowId;
|
||||
|
||||
private static final long serialVersionUID = 8379177178417619790L;
|
||||
|
||||
/**
|
||||
* 流程执行线程
|
||||
*/
|
||||
private SubThreadPoolExecutor threadPool;
|
||||
|
||||
/**
|
||||
* 根节点
|
||||
*/
|
||||
private SpiderNode rootNode;
|
||||
|
||||
/**
|
||||
* 爬虫是否运行中
|
||||
*/
|
||||
private volatile boolean running = true;
|
||||
|
||||
/**
|
||||
* Future队列
|
||||
*/
|
||||
private LinkedBlockingQueue<Future<?>> futureQueue = new LinkedBlockingQueue<>();
|
||||
|
||||
/**
|
||||
* Cookie上下文
|
||||
*/
|
||||
private CookieContext cookieContext = new CookieContext();
|
||||
|
||||
public List<SpiderOutput> getOutputs() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
public <T> T get(String key){
|
||||
return (T) super.get(key);
|
||||
}
|
||||
|
||||
public <T> T get(String key,T defaultValue){
|
||||
T value = this.get(key);
|
||||
return value == null ? defaultValue : value;
|
||||
}
|
||||
|
||||
public String getFlowId() {
|
||||
return flowId;
|
||||
}
|
||||
|
||||
public void setFlowId(String flowId) {
|
||||
this.flowId = flowId;
|
||||
}
|
||||
|
||||
public LinkedBlockingQueue<Future<?>> getFutureQueue() {
|
||||
return futureQueue;
|
||||
}
|
||||
|
||||
public boolean isRunning() {
|
||||
return running;
|
||||
}
|
||||
|
||||
public void setRunning(boolean running) {
|
||||
this.running = running;
|
||||
}
|
||||
|
||||
public void addOutput(SpiderOutput output){
|
||||
|
||||
}
|
||||
|
||||
public SubThreadPoolExecutor getThreadPool() {
|
||||
return threadPool;
|
||||
}
|
||||
|
||||
public void setThreadPool(SubThreadPoolExecutor threadPool) {
|
||||
this.threadPool = threadPool;
|
||||
}
|
||||
|
||||
public SpiderNode getRootNode() {
|
||||
return rootNode;
|
||||
}
|
||||
|
||||
public void setRootNode(SpiderNode rootNode) {
|
||||
this.rootNode = rootNode;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public CookieContext getCookieContext() {
|
||||
return cookieContext;
|
||||
}
|
||||
|
||||
public void pause(String nodeId,String event,String key,Object value){}
|
||||
|
||||
public void resume(){}
|
||||
|
||||
public void stop(){}
|
||||
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
package org.spiderflow.context;
|
||||
|
||||
import com.alibaba.ttl.TransmittableThreadLocal;
|
||||
|
||||
public class SpiderContextHolder {
|
||||
|
||||
private static final ThreadLocal<SpiderContext> THREAD_LOCAL = new TransmittableThreadLocal<>();
|
||||
|
||||
public static SpiderContext get() {
|
||||
return THREAD_LOCAL.get();
|
||||
}
|
||||
|
||||
public static void set(SpiderContext context) {
|
||||
THREAD_LOCAL.set(context);
|
||||
}
|
||||
|
||||
public static void remove() {
|
||||
THREAD_LOCAL.remove();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package org.spiderflow.enums;
|
||||
|
||||
/**
|
||||
* 流程通知类型
|
||||
*
|
||||
* @author BillDowney
|
||||
* @date 2020年4月4日 上午1:32:53
|
||||
*/
|
||||
public enum FlowNoticeType {
|
||||
/**
|
||||
* 流程开始通知
|
||||
*/
|
||||
startNotice,
|
||||
/**
|
||||
* 流程异常通知
|
||||
*/
|
||||
exceptionNotice,
|
||||
/**
|
||||
* 流程结束通知
|
||||
*/
|
||||
endNotice
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package org.spiderflow.enums;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 流程通知方式
|
||||
*
|
||||
* @author BillDowney
|
||||
* @date 2020年4月3日 下午3:26:18
|
||||
*/
|
||||
public enum FlowNoticeWay {
|
||||
|
||||
email("邮件通知");
|
||||
|
||||
private FlowNoticeWay(String title) {
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
private String title;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.name() + ":" + this.title;
|
||||
}
|
||||
|
||||
public static Map<String, String> getMap() {
|
||||
Map<String, String> map = new LinkedHashMap<String, String>();
|
||||
for (FlowNoticeWay type : FlowNoticeWay.values()) {
|
||||
map.put(type.name(), type.toString());
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package org.spiderflow.executor;
|
||||
|
||||
public interface FunctionExecutor {
|
||||
|
||||
String getFunctionPrefix();
|
||||
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
package org.spiderflow.executor;
|
||||
|
||||
public interface FunctionExtension {
|
||||
|
||||
Class<?> support();
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package org.spiderflow.executor;
|
||||
|
||||
import org.spiderflow.model.Plugin;
|
||||
|
||||
public interface PluginConfig {
|
||||
|
||||
Plugin plugin();
|
||||
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
package org.spiderflow.executor;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.model.Shape;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
/**
|
||||
* 执行器接口
|
||||
* @author jmxd
|
||||
*
|
||||
*/
|
||||
public interface ShapeExecutor {
|
||||
|
||||
String LOOP_VARIABLE_NAME = "loopVariableName";
|
||||
|
||||
String LOOP_COUNT = "loopCount";
|
||||
|
||||
String THREAD_COUNT = "threadCount";
|
||||
|
||||
default Shape shape(){
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 节点形状
|
||||
* @return 节点形状名称
|
||||
*/
|
||||
String supportShape();
|
||||
|
||||
/**
|
||||
* 执行器具体的功能实现
|
||||
* @param node 当前要执行的爬虫节点
|
||||
* @param context 爬虫上下文
|
||||
* @param variables 节点流程的全部变量的集合
|
||||
*/
|
||||
void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables);
|
||||
|
||||
default boolean allowExecuteNext(SpiderNode node, SpiderContext context, Map<String, Object> variables){
|
||||
return true;
|
||||
}
|
||||
|
||||
default boolean isThread(){
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package org.spiderflow.expression;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface DynamicMethod {
|
||||
|
||||
Object execute(String methodName, List<Object> parameters);
|
||||
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package org.spiderflow.io;
|
||||
|
||||
public class Line {
|
||||
|
||||
private long from;
|
||||
|
||||
private String text;
|
||||
|
||||
private long to;
|
||||
|
||||
public Line(long from, String text, long to) {
|
||||
this.from = from;
|
||||
this.text = text;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
public long getFrom() {
|
||||
return from;
|
||||
}
|
||||
|
||||
public void setFrom(long from) {
|
||||
this.from = from;
|
||||
}
|
||||
|
||||
public String getText() {
|
||||
return text;
|
||||
}
|
||||
|
||||
public void setText(String text) {
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
public long getTo() {
|
||||
return to;
|
||||
}
|
||||
|
||||
public void setTo(long to) {
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Line{" +
|
||||
"from=" + from +
|
||||
", text='" + text + '\'' +
|
||||
", to=" + to +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
package org.spiderflow.io;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
|
||||
public interface SpiderResponse {
|
||||
|
||||
@Comment("获取返回状态码")
|
||||
@Example("${resp.statusCode}")
|
||||
int getStatusCode();
|
||||
|
||||
@Comment("获取网页标题")
|
||||
@Example("${resp.title}")
|
||||
String getTitle();
|
||||
|
||||
@Comment("获取网页html")
|
||||
@Example("${resp.html}")
|
||||
String getHtml();
|
||||
|
||||
@Comment("获取json")
|
||||
@Example("${resp.json}")
|
||||
default Object getJson(){
|
||||
return JSON.parse(getHtml());
|
||||
}
|
||||
@Comment("获取cookies")
|
||||
@Example("${resp.cookies}")
|
||||
Map<String,String> getCookies();
|
||||
|
||||
@Comment("获取headers")
|
||||
@Example("${resp.headers}")
|
||||
Map<String,String> getHeaders();
|
||||
|
||||
@Comment("获取byte[]")
|
||||
@Example("${resp.bytes}")
|
||||
byte[] getBytes();
|
||||
|
||||
@Comment("获取ContentType")
|
||||
@Example("${resp.contentType}")
|
||||
String getContentType();
|
||||
|
||||
@Comment("获取当前url")
|
||||
@Example("${resp.url}")
|
||||
String getUrl();
|
||||
|
||||
@Example("${resp.setCharset('UTF-8')}")
|
||||
default void setCharset(String charset){
|
||||
|
||||
}
|
||||
@Example("${resp.stream}")
|
||||
default InputStream getStream(){
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package org.spiderflow.listener;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
|
||||
public interface SpiderListener {
|
||||
|
||||
/**
|
||||
* 开始执行之前
|
||||
*/
|
||||
void beforeStart(SpiderContext context);
|
||||
|
||||
/**
|
||||
* 执行完毕之后
|
||||
*/
|
||||
void afterEnd(SpiderContext context);
|
||||
|
||||
}
|
||||
@ -0,0 +1,108 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.annotation.Return;
|
||||
|
||||
public class Grammer {
|
||||
|
||||
private String owner;
|
||||
|
||||
private String method;
|
||||
|
||||
private String comment;
|
||||
|
||||
private String example;
|
||||
|
||||
private String function;
|
||||
|
||||
private List<String> returns;
|
||||
|
||||
public String getOwner() {
|
||||
return owner;
|
||||
}
|
||||
|
||||
public void setOwner(String owner) {
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
public String getMethod() {
|
||||
return method;
|
||||
}
|
||||
|
||||
public void setMethod(String method) {
|
||||
this.method = method;
|
||||
}
|
||||
|
||||
public String getFunction() {
|
||||
return function;
|
||||
}
|
||||
|
||||
public void setFunction(String function) {
|
||||
this.function = function;
|
||||
}
|
||||
|
||||
public String getComment() {
|
||||
return comment;
|
||||
}
|
||||
|
||||
public void setComment(String comment) {
|
||||
this.comment = comment;
|
||||
}
|
||||
|
||||
public String getExample() {
|
||||
return example;
|
||||
}
|
||||
|
||||
public void setExample(String example) {
|
||||
this.example = example;
|
||||
}
|
||||
|
||||
public List<String> getReturns() {
|
||||
return returns;
|
||||
}
|
||||
|
||||
public void setReturns(List<String> returns) {
|
||||
this.returns = returns;
|
||||
}
|
||||
|
||||
public static List<Grammer> findGrammers(Class<?> clazz,String function,String owner,boolean mustStatic){
|
||||
Method[] methods = clazz.getDeclaredMethods();
|
||||
List<Grammer> grammers = new ArrayList<>();
|
||||
for (Method method : methods) {
|
||||
if(Modifier.isPublic(method.getModifiers()) && (Modifier.isStatic(method.getModifiers())||!mustStatic)){
|
||||
Grammer grammer = new Grammer();
|
||||
grammer.setMethod(method.getName());
|
||||
Comment comment = method.getAnnotation(Comment.class);
|
||||
if(comment != null){
|
||||
grammer.setComment(comment.value());
|
||||
}
|
||||
Example example = method.getAnnotation(Example.class);
|
||||
if(example != null){
|
||||
grammer.setExample(example.value());
|
||||
}
|
||||
Return returns = method.getAnnotation(Return.class);
|
||||
if(returns != null){
|
||||
Class<?>[] clazzs = returns.value();
|
||||
List<String> returnTypes = new ArrayList<>();
|
||||
for (int i = 0; i < clazzs.length; i++) {
|
||||
returnTypes.add(clazzs[i].getSimpleName());
|
||||
}
|
||||
grammer.setReturns(returnTypes);
|
||||
}else{
|
||||
grammer.setReturns(Collections.singletonList(method.getReturnType().getSimpleName()));
|
||||
}
|
||||
grammer.setFunction(function);
|
||||
grammer.setOwner(owner);
|
||||
grammers.add(grammer);
|
||||
}
|
||||
}
|
||||
return grammers;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
public class JsonBean<T> {
|
||||
|
||||
private Integer code = 1;
|
||||
|
||||
private String message = "执行成功";
|
||||
|
||||
private T data;
|
||||
|
||||
public JsonBean(Integer code, String message, T data) {
|
||||
this.code = code;
|
||||
this.message = message;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public JsonBean(Integer code, String message) {
|
||||
this.code = code;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public JsonBean(T data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public Integer getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public void setCode(Integer code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public T getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(T data) {
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
public class Plugin {
|
||||
|
||||
private String name;
|
||||
|
||||
private String url;
|
||||
|
||||
public Plugin(String name, String url) {
|
||||
this.name = name;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
public class Shape {
|
||||
|
||||
private String name;
|
||||
|
||||
private String label;
|
||||
|
||||
private String title;
|
||||
|
||||
private String image;
|
||||
|
||||
private String desc;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getLabel() {
|
||||
return label;
|
||||
}
|
||||
|
||||
public void setLabel(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public void setTitle(String title) {
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
public String getImage() {
|
||||
return image;
|
||||
}
|
||||
|
||||
public void setImage(String image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
public String getDesc() {
|
||||
return desc;
|
||||
}
|
||||
|
||||
public void setDesc(String desc) {
|
||||
this.desc = desc;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
|
||||
public class SpiderLog {
|
||||
|
||||
private String level;
|
||||
|
||||
private String message;
|
||||
|
||||
private List<Object> variables;
|
||||
|
||||
public SpiderLog(String level,String message, List<Object> variables) {
|
||||
if(variables != null && variables.size() > 0){
|
||||
List<Object> nVariables = new ArrayList<>(variables.size());
|
||||
for (Object object : variables) {
|
||||
if(object instanceof Throwable){
|
||||
nVariables.add(ExceptionUtils.getStackTrace((Throwable) object));
|
||||
}else{
|
||||
nVariables.add(object);
|
||||
}
|
||||
}
|
||||
this.variables = nVariables;
|
||||
}
|
||||
this.level = level;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public String getLevel() {
|
||||
return level;
|
||||
}
|
||||
|
||||
public void setLevel(String level) {
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public List<Object> getVariables() {
|
||||
return variables;
|
||||
}
|
||||
|
||||
public void setVariables(List<Object> variables) {
|
||||
this.variables = variables;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,205 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.text.StringEscapeUtils;
|
||||
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
|
||||
|
||||
/**
|
||||
* 爬虫节点
|
||||
* @author jmxd
|
||||
*
|
||||
*/
|
||||
public class SpiderNode {
|
||||
/**
|
||||
* 节点的Json属性
|
||||
*/
|
||||
private Map<String,Object> jsonProperty = new HashMap<>();
|
||||
/**
|
||||
* 节点列表中的下一个节点
|
||||
*/
|
||||
private List<SpiderNode> nextNodes = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 节点列表中的上一个节点
|
||||
*/
|
||||
private List<SpiderNode> prevNodes = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 父级节点ID
|
||||
*/
|
||||
private Set<String> parentNodes;
|
||||
|
||||
/**
|
||||
* 节点流转条件
|
||||
*/
|
||||
private Map<String,String> condition = new HashMap<>();
|
||||
|
||||
/**
|
||||
* 异常流转
|
||||
*/
|
||||
private Map<String,String> exception = new HashMap<>();
|
||||
|
||||
/**
|
||||
* 传递变量
|
||||
*/
|
||||
private Map<String,String> transmitVariable = new HashMap<>();
|
||||
/**
|
||||
* 节点名称
|
||||
*/
|
||||
private String nodeName;
|
||||
/**
|
||||
* 节点ID
|
||||
*/
|
||||
private String nodeId;
|
||||
|
||||
/**
|
||||
* 计数器,用来计算当前节点执行中的个数
|
||||
*/
|
||||
private AtomicInteger counter = new AtomicInteger();
|
||||
|
||||
public String getNodeId() {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
public void setNodeId(String nodeId) {
|
||||
this.nodeId = nodeId;
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return nodeName;
|
||||
}
|
||||
|
||||
public void setNodeName(String nodeName) {
|
||||
this.nodeName = nodeName;
|
||||
}
|
||||
|
||||
public String getStringJsonValue(String key){
|
||||
String value = (String) this.jsonProperty.get(key);
|
||||
if(value != null){
|
||||
value = StringEscapeUtils.unescapeHtml4(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public String getStringJsonValue(String key,String defaultValue){
|
||||
String value = getStringJsonValue(key);
|
||||
return StringUtils.isNotBlank(value) ? value : defaultValue;
|
||||
}
|
||||
|
||||
public List<Map<String,String>> getListJsonValue(String ... keys){
|
||||
List<JSONArray> arrays = new ArrayList<>();
|
||||
int size = -1;
|
||||
List<Map<String,String>> result = new ArrayList<>();
|
||||
for (int i = 0; i < keys.length; i++) {
|
||||
JSONArray jsonArray = (JSONArray) this.jsonProperty.get(keys[i]);
|
||||
if(jsonArray != null){
|
||||
if(size == -1){
|
||||
size = jsonArray.size();
|
||||
}else if(size != jsonArray.size()){
|
||||
throw new ArrayIndexOutOfBoundsException();
|
||||
}
|
||||
arrays.add(jsonArray);
|
||||
}
|
||||
}
|
||||
for (int i = 0;i < size;i++) {
|
||||
Map<String,String> item = new HashMap<>();
|
||||
for (int j = 0; j < keys.length; j++) {
|
||||
String val = arrays.get(j).getString(i);
|
||||
if(val != null){
|
||||
val = StringEscapeUtils.unescapeHtml4(val);
|
||||
}
|
||||
item.put(keys[j],val);
|
||||
}
|
||||
result.add(item);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
public void setJsonProperty(Map<String, Object> jsonProperty) {
|
||||
this.jsonProperty = jsonProperty;
|
||||
}
|
||||
|
||||
public void addNextNode(SpiderNode nextNode){
|
||||
nextNode.prevNodes.add(this);
|
||||
this.nextNodes.add(nextNode);
|
||||
}
|
||||
|
||||
public String getExceptionFlow(String fromNodeId) {
|
||||
return exception.get(fromNodeId);
|
||||
}
|
||||
|
||||
public boolean isTransmitVariable(String fromNodeId) {
|
||||
String value = transmitVariable.get(fromNodeId);
|
||||
return value == null || "1".equalsIgnoreCase(value);
|
||||
}
|
||||
|
||||
public void setTransmitVariable(String fromNodeId,String value){
|
||||
this.transmitVariable.put(fromNodeId,value);
|
||||
}
|
||||
|
||||
public void setExceptionFlow(String fromNodeId,String value){
|
||||
this.exception.put(fromNodeId,value);
|
||||
}
|
||||
|
||||
public List<SpiderNode> getNextNodes() {
|
||||
return nextNodes;
|
||||
}
|
||||
|
||||
public String getCondition(String fromNodeId) {
|
||||
return condition.get(fromNodeId);
|
||||
}
|
||||
|
||||
public void setCondition(String fromNodeId,String condition) {
|
||||
this.condition.put(fromNodeId, condition);
|
||||
}
|
||||
|
||||
public void increment(){
|
||||
counter.incrementAndGet();
|
||||
}
|
||||
|
||||
public void decrement(){
|
||||
counter.decrementAndGet();
|
||||
}
|
||||
|
||||
public boolean hasLeftNode(String nodeId){
|
||||
if(parentNodes == null){
|
||||
Set<String> parents = new HashSet<>();
|
||||
generateParents(parents);
|
||||
this.parentNodes = parents;
|
||||
}
|
||||
return this.parentNodes.contains(nodeId);
|
||||
}
|
||||
|
||||
private void generateParents(Set<String> parents){
|
||||
for (SpiderNode prevNode : prevNodes) {
|
||||
if(parents.add(prevNode.nodeId)){
|
||||
prevNode.generateParents(parents);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isDone(){
|
||||
return isDone(new HashSet<>());
|
||||
}
|
||||
public boolean isDone(Set<String> visited){
|
||||
if(this.counter.get() == 0){
|
||||
for (SpiderNode prevNode : prevNodes) {
|
||||
if(visited.add(nodeId)&&!prevNode.isDone(visited)){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SpiderNode [jsonProperty=" + jsonProperty + ", nextNodes=" + nextNodes + ", condition=" + condition
|
||||
+ ", nodeName=" + nodeName + ", nodeId=" + nodeId + "]";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
package org.spiderflow.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SpiderOutput {
|
||||
|
||||
/**
|
||||
* 节点名称
|
||||
*/
|
||||
private String nodeName;
|
||||
|
||||
/**
|
||||
* 节点Id
|
||||
*/
|
||||
private String nodeId;
|
||||
|
||||
/**
|
||||
* 输出项的名
|
||||
*/
|
||||
private List<String> outputNames = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 输出项的值
|
||||
*/
|
||||
private List<Object> values = new ArrayList<>();
|
||||
|
||||
public String getNodeName() {
|
||||
return nodeName;
|
||||
}
|
||||
|
||||
public void setNodeName(String nodeName) {
|
||||
this.nodeName = nodeName;
|
||||
}
|
||||
|
||||
public List<String> getOutputNames() {
|
||||
return outputNames;
|
||||
}
|
||||
|
||||
public void setOutputNames(List<String> outputNames) {
|
||||
this.outputNames = outputNames;
|
||||
}
|
||||
|
||||
public List<Object> getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
public void setValues(List<Object> values) {
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
public void addOutput(String name,Object value){
|
||||
this.outputNames.add(name);
|
||||
this.values.add(value);
|
||||
}
|
||||
|
||||
public String getNodeId() {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
public void setNodeId(String nodeId) {
|
||||
this.nodeId = nodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SpiderOutput [nodeName=" + nodeName + ", nodeId=" + nodeId + ", outputNames=" + outputNames
|
||||
+ ", values=" + values + "]";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package org.spiderflow.utils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class Maps {
|
||||
|
||||
public static <K,V> Map<K,V> add(Map<K,V> srcMap,K k,V v){
|
||||
HashMap<K, V> destMap = new HashMap<>(srcMap);
|
||||
destMap.put(k, v);
|
||||
return destMap;
|
||||
}
|
||||
|
||||
public static <K,V> Map<K,V> newMap(K key,V value){
|
||||
HashMap<K, V> map = new HashMap<>();
|
||||
map.put(key, value);
|
||||
return map;
|
||||
}
|
||||
|
||||
public static <K,V> Map<K,V> add(Map<K,V> srcMap,List<K> ks,List<V> vs){
|
||||
HashMap<K, V> destMap = new HashMap<>(srcMap);
|
||||
if(ks != null && vs != null && ks.size() == vs.size()){
|
||||
int size = ks.size();
|
||||
for (int i = 0; i < size; i++) {
|
||||
destMap.put(ks.get(0), vs.get(0));
|
||||
}
|
||||
}
|
||||
return destMap;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow</artifactId>
|
||||
<version>0.5.0</version>
|
||||
</parent>
|
||||
<artifactId>spider-flow-core</artifactId>
|
||||
<name>spider-flow-core</name>
|
||||
<url>https://gitee.com/jmxd/spider-flow/tree/master/spider-flow-core</url>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.spiderflow</groupId>
|
||||
<artifactId>spider-flow-api</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@ -0,0 +1,88 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 字符串内容和Base64互相转换 工具类 防止NPE
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("base64常用方法")
|
||||
public class Base64FunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "base64";
|
||||
}
|
||||
|
||||
@Comment("根据byte[]进行base64加密")
|
||||
@Example("${base64.encode(resp.bytes)}")
|
||||
public static String encode(byte[] bytes){
|
||||
return bytes != null ? Base64.encodeBase64String(bytes) : null;
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64加密")
|
||||
@Example("${base64.encode(resp.bytes,'UTF-8')}")
|
||||
public static String encode(String content,String charset){
|
||||
return encode(StringFunctionExecutor.bytes(content,charset));
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64加密")
|
||||
@Example("${base64.encode(resp.html)}")
|
||||
public static String encode(String content){
|
||||
return encode(StringFunctionExecutor.bytes(content));
|
||||
}
|
||||
|
||||
@Comment("根据byte[]进行base64加密")
|
||||
@Example("${base64.encodeBytes(resp.bytes)}")
|
||||
public static byte[] encodeBytes(byte[] bytes){
|
||||
return bytes != null ? Base64.encodeBase64(bytes) : null;
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64加密")
|
||||
@Example("${base64.encodeBytes(resp.html,'UTF-8')}")
|
||||
public static byte[] encodeBytes(String content,String charset){
|
||||
return encodeBytes(StringFunctionExecutor.bytes(content,charset));
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64加密")
|
||||
@Example("${base64.encodeBytes(resp.html)}")
|
||||
public static byte[] encodeBytes(String content){
|
||||
return encodeBytes(StringFunctionExecutor.bytes(content));
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64解密")
|
||||
@Example("${base64.decode(resp.html)}")
|
||||
public static byte[] decode(String base64){
|
||||
return base64 != null ? Base64.decodeBase64(base64) :null;
|
||||
}
|
||||
|
||||
@Comment("根据byte[]进行base64解密")
|
||||
@Example("${base64.decode(resp.bytes)}")
|
||||
public static byte[] decode(byte[] base64){
|
||||
return base64 != null ? Base64.decodeBase64(base64) :null;
|
||||
}
|
||||
|
||||
@Comment("根据String进行base64解密")
|
||||
@Example("${base64.decodeString(resp.html)}")
|
||||
public static String decodeString(String base64){
|
||||
return base64 != null ? new String(Base64.decodeBase64(base64)) :null;
|
||||
}
|
||||
|
||||
@Comment("根据byte[]进行base64解密")
|
||||
@Example("${base64.decodeString(resp.bytes)}")
|
||||
public static String decodeString(byte[] base64){
|
||||
return base64 != null ? new String(Base64.decodeBase64(base64)) :null;
|
||||
}
|
||||
|
||||
@Comment("根据byte[]进行base64解密")
|
||||
@Example("${base64.decodeString(resp.bytes,'UTF-8')}")
|
||||
public static String decodeString(byte[] base64,String charset){
|
||||
return base64 != null ? StringFunctionExecutor.newString(Base64.decodeBase64(base64),charset) :null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 时间获取/格式化 工具类 防止NPE 默认格式(yyyy-MM-dd HH:mm:ss)
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("日期常用方法")
|
||||
public class DateFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "date";
|
||||
}
|
||||
|
||||
private static final String DEFAULT_PATTERN = "yyyy-MM-dd HH:mm:ss";
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${date.format(date.now())}")
|
||||
public static String format(Date date) {
|
||||
return format(date, DEFAULT_PATTERN);
|
||||
}
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${date.format(1569059534000l)}")
|
||||
public static String format(Long millis) {
|
||||
return format(millis, DEFAULT_PATTERN);
|
||||
}
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${date.format(date.now(),'yyyy-MM-dd')}")
|
||||
public static String format(Date date, String pattern) {
|
||||
return date != null ? DateFormatUtils.format(date, pattern) : null;
|
||||
}
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${date.format(1569059534000l,'yyyy-MM-dd')}")
|
||||
public static String format(Long millis, String pattern) {
|
||||
return millis != null ? DateFormatUtils.format(millis, pattern) : null;
|
||||
}
|
||||
|
||||
@Comment("字符串转为日期类型")
|
||||
@Example("${date.parse('2019-01-01 00:00:00')}")
|
||||
public static Date parse(String date) throws ParseException{
|
||||
return date != null ? DateUtils.parseDate(date, DEFAULT_PATTERN) : null;
|
||||
}
|
||||
|
||||
@Comment("字符串转为日期类型")
|
||||
@Example("${date.parse('2019-01-01','yyyy-MM-dd')}")
|
||||
public static Date parse(String date,String pattern) throws ParseException{
|
||||
return date != null ? DateUtils.parseDate(date, pattern) : null;
|
||||
}
|
||||
|
||||
@Comment("数字为日期类型")
|
||||
@Example("${date.parse(1569059534000l)}")
|
||||
public static Date parse(Long millis){
|
||||
return new Date(millis);
|
||||
}
|
||||
|
||||
@Comment("获取当前时间")
|
||||
@Example("${date.now()}")
|
||||
public static Date now(){
|
||||
return new Date();
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n年后的日期")
|
||||
@Example("${date.addYears(date.now(),2)}")
|
||||
public static Date addYears(Date date,int amount){
|
||||
return DateUtils.addYears(date, amount);
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n月后的日期")
|
||||
@Example("${date.addMonths(date.now(),2)}")
|
||||
public static Date addMonths(Date date,int amount){
|
||||
return DateUtils.addMonths(date, amount);
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n天后的日期")
|
||||
@Example("${date.addDays(date.now(),2)}")
|
||||
public static Date addDays(Date date,int amount){
|
||||
return DateUtils.addDays(date, amount);
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n小时后的日期")
|
||||
@Example("${date.addHours(date.now(),2)}")
|
||||
public static Date addHours(Date date,int amount){
|
||||
return DateUtils.addHours(date, amount);
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n分钟后的日期")
|
||||
@Example("${date.addMinutes(date.now(),2)}")
|
||||
public static Date addMinutes(Date date,int amount){
|
||||
return DateUtils.addMinutes(date, amount);
|
||||
}
|
||||
|
||||
@Comment("获取指定日期n秒后的日期")
|
||||
@Example("${date.addSeconds(date.now(),2)}")
|
||||
public static Date addSeconds(Date date,int amount){
|
||||
return DateUtils.addSeconds(date, amount);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,150 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Comment("数据抽取常用方法")
|
||||
public class ExtractFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "extract";
|
||||
}
|
||||
|
||||
@Comment("根据jsonpath提取内容")
|
||||
@Example("${extract.jsonpath(resp.json,'$.code')}")
|
||||
public static Object jsonpath(Object root,String jsonpath){
|
||||
return ExtractUtils.getValueByJsonPath(root, jsonpath);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regx(resp.html,'<title>(.*?)</title>')}")
|
||||
public static String regx(String content,String pattern){
|
||||
return ExtractUtils.getFirstMatcher(content, pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regx(resp.html,'<title>(.*?)</title>',1)}")
|
||||
public static String regx(String content,String pattern,int groupIndex){
|
||||
return ExtractUtils.getFirstMatcher(content, pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regx(resp.html,'<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<String> regx(String content,String pattern,List<Integer> groups){
|
||||
return ExtractUtils.getFirstMatcher(content, pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regxs(resp.html,'<h2>(.*?)</h2>')}")
|
||||
public static List<String> regxs(String content,String pattern){
|
||||
return ExtractUtils.getMatchers(content, pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regxs(resp.html,'<h2>(.*?)</h2>',1)}")
|
||||
public static List<String> regxs(String content,String pattern,int groupIndex){
|
||||
return ExtractUtils.getMatchers(content, pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${extract.regxs(resp.html,'<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<List<String>> regxs(String content,String pattern,List<Integer> groups){
|
||||
return ExtractUtils.getMatchers(content, pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${extract.xpath(resp.element(),'//title/text()')}")
|
||||
public static String xpath(Element element,String xpath){
|
||||
return ExtractUtils.getValueByXPath(element, xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${extract.xpath(resp.html,'//title/text()')}")
|
||||
public static String xpath(String content,String xpath){
|
||||
return xpath(Jsoup.parse(content),xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpaths提取内容")
|
||||
@Example("${extract.xpaths(resp.element(),'//h2/text()')}")
|
||||
public static List<String> xpaths(Element element,String xpath){
|
||||
return ExtractUtils.getValuesByXPath(element, xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpaths提取内容")
|
||||
@Example("${extract.xpaths(resp.html,'//h2/text()')}")
|
||||
public static List<String> xpaths(String content,String xpath){
|
||||
return xpaths(Jsoup.parse(content),xpath);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selectors(resp.html,'div > a')}")
|
||||
public static List<String> selectors(Object object,String selector){
|
||||
return ExtractUtils.getHTMLBySelector(getElement(object), selector);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selector(resp.html,'div > a','text')}")
|
||||
public static Object selector(Object object,String selector,String type){
|
||||
if("element".equals(type)){
|
||||
return ExtractUtils.getFirstElement(getElement(object), selector);
|
||||
}else if("text".equals(type)){
|
||||
return ExtractUtils.getFirstTextBySelector(getElement(object), selector);
|
||||
}else if("outerhtml".equals(type)){
|
||||
return ExtractUtils.getFirstOuterHTMLBySelector(getElement(object), selector);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selector(resp.html,'div > a','attr','href')}")
|
||||
public static String selector(Object object,String selector,String type,String attrValue){
|
||||
if("attr".equals(type)){
|
||||
return ExtractUtils.getFirstAttrBySelector(getElement(object), selector,attrValue);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selector(resp.html,'div > a')}")
|
||||
public static String selector(Object object,String selector){
|
||||
return ExtractUtils.getFirstHTMLBySelector(getElement(object), selector);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selectors(resp.html,'div > a','element')}")
|
||||
public static Object selectors(Object object,String selector,String type){
|
||||
if("element".equals(type)){
|
||||
return ExtractUtils.getElements(getElement(object), selector);
|
||||
}else if("text".equals(type)){
|
||||
return ExtractUtils.getTextBySelector(getElement(object), selector);
|
||||
}else if("outerhtml".equals(type)){
|
||||
return ExtractUtils.getOuterHTMLBySelector(getElement(object), selector);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${extract.selectors(resp.html,'div > a','attr','href')}")
|
||||
public static Object selectors(Object object,String selector,String type,String attrValue){
|
||||
if("attr".equals(type)){
|
||||
return ExtractUtils.getAttrBySelector(getElement(object), selector,attrValue);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Element getElement(Object object){
|
||||
if(object != null){
|
||||
return object instanceof Element ? (Element)object:Jsoup.parse((String) object);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,131 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.core.utils.FileUtils;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
/**
|
||||
* 文件读写 工具类 防止NPE
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("file常用方法")
|
||||
public class FileFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "file";
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param path 文件路径/名
|
||||
* @param createDirectory 是否需要创建
|
||||
* @return File 文件
|
||||
*/
|
||||
private static File getFile(String path,boolean createDirectory){
|
||||
File f = new File(path);
|
||||
if(createDirectory&&!f.getParentFile().exists()){
|
||||
f.getParentFile().mkdirs();
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.html,false)}")
|
||||
public static void write(String path,String content,boolean append) throws IOException{
|
||||
write(path,content,Charset.defaultCharset().name(),append);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.html,'UTF-8',false)}")
|
||||
public static void write(String path,String content,String charset,boolean append) throws IOException{
|
||||
write(path,StringFunctionExecutor.bytes(content, charset),append);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.bytes,false)}")
|
||||
public static void write(String path,byte[] bytes,boolean append) throws IOException{
|
||||
write(path, new ByteArrayInputStream(bytes),append);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.stream,false)}")
|
||||
public static void write(String path, InputStream stream, boolean append) throws IOException {
|
||||
try(FileOutputStream fos = new FileOutputStream(getFile(path,true),append)){
|
||||
IOUtils.copyLarge(stream, fos);
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.bytes,false)}")
|
||||
public static void write(String path, InputStream stream) throws IOException {
|
||||
write(path, stream,false);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.html)}")
|
||||
public static void write(String path,String content) throws IOException{
|
||||
write(path,content,false);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.html,'UTF-8')}")
|
||||
public static void write(String path,String content,String charset) throws IOException{
|
||||
write(path,content,charset,false);
|
||||
}
|
||||
|
||||
@Comment("写出文件")
|
||||
@Example("${file.write('e:/result.html',resp.bytes)}")
|
||||
public static void write(String path,byte[] bytes) throws IOException{
|
||||
write(path,bytes,false);
|
||||
}
|
||||
|
||||
@Comment("下载Url资源")
|
||||
@Example("${file.download('e:/downloadPath',urls)}")
|
||||
public static void download(String path, List<String> urls) throws IOException{
|
||||
if(!CollectionUtils.isEmpty(urls)) {
|
||||
for (String url : urls) {
|
||||
FileUtils.downloadFile(path, url, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("下载Url资源")
|
||||
@Example("${file.download('e:/downloadPath',urls)}")
|
||||
public static void download(String path, String url) throws IOException {
|
||||
if (url != null) {
|
||||
FileUtils.downloadFile(path, url, true);
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("读取文件")
|
||||
@Example("${file.bytes('e:/result.html')}")
|
||||
public static byte[] bytes(String path) throws IOException{
|
||||
try(FileInputStream fis = new FileInputStream(getFile(path, false))){
|
||||
return IOUtils.toByteArray(fis);
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("读取文件")
|
||||
@Example("${file.string('e:/result.html','UTF-8')}")
|
||||
public static String string(String path,String charset) throws IOException{
|
||||
return StringFunctionExecutor.newString(bytes(path), charset);
|
||||
}
|
||||
|
||||
@Comment("读取文件")
|
||||
@Example("${file.string('e:/result.html')}")
|
||||
public static String string(String path) throws IOException{
|
||||
return StringFunctionExecutor.newString(bytes(path), Charset.defaultCharset().name());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
|
||||
/**
|
||||
* Json和String互相转换 工具类 防止NPE
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("json常用方法")
|
||||
public class JsonFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "json";
|
||||
}
|
||||
|
||||
@Comment("将字符串转为json对象")
|
||||
@Example("${json.parse('{code : 1}')}")
|
||||
public static Object parse(String jsonString){
|
||||
return jsonString != null ? JSON.parse(jsonString) : null;
|
||||
}
|
||||
|
||||
@Comment("将对象转为json字符串")
|
||||
@Example("${json.stringify(objVar)}")
|
||||
public static String stringify(Object object){
|
||||
return object != null ? JSON.toJSONString(object) : null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* List 工具类 防止NPE 添加了类似python的split()方法
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("list常用方法")
|
||||
public class ListFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "list";
|
||||
}
|
||||
|
||||
@Comment("获取list的长度")
|
||||
@Example("${list.length(listVar)}")
|
||||
public static int length(List<?> list){
|
||||
return list != null ? list.size() : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param list 原List
|
||||
* @param len 按多长进行分割
|
||||
* @return List<List<?>> 分割后的数组
|
||||
*/
|
||||
@Comment("分割List")
|
||||
@Example("${list.split(listVar,10)}")
|
||||
public static List<List<?>> split(List<?> list,int len){
|
||||
List<List<?>> result = new ArrayList<>();
|
||||
if (list == null || list.size() == 0 || len < 1) {
|
||||
return result;
|
||||
}
|
||||
int size = list.size();
|
||||
int count = (size + len - 1) / len;
|
||||
for (int i = 0; i < count; i++) {
|
||||
List<?> subList = list.subList(i * len, ((i + 1) * len > size ? size : len * (i + 1)));
|
||||
result.add(subList);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Comment("截取List")
|
||||
@Example("${list.sublist(listVar,fromIndex,toIndex)}")
|
||||
public static List<?> sublist(List<?> list,int fromIndex,int toIndex){
|
||||
return list!= null ? list.subList(fromIndex, toIndex) : new ArrayList<>();
|
||||
}
|
||||
|
||||
@Comment("过滤字符串list元素")
|
||||
@Example("${listVar.filterStr(pattern)}")
|
||||
public static List<String> filterStr(List<String> list, String pattern) {
|
||||
if (list == null || list.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
List<String> result = new ArrayList<>(list.size());
|
||||
for (String item : list) {
|
||||
if (Pattern.matches(pattern, item)) {
|
||||
result.add(item);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@Component
|
||||
@Comment("MD5常用方法")
|
||||
public class MD5FunctionExecutor implements FunctionExecutor {
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "md5";
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.string(resp.html)}")
|
||||
public static String string(String str){
|
||||
return DigestUtils.md5Hex(str);
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.string(resp.bytes)}")
|
||||
public static String string(byte[] bytes){
|
||||
return DigestUtils.md5Hex(bytes);
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.string(resp.stream)}")
|
||||
public static String string(InputStream stream) throws IOException {
|
||||
return DigestUtils.md5Hex(stream);
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.bytes(resp.html)}")
|
||||
public static byte[] bytes(String str){
|
||||
return DigestUtils.md5(str);
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.bytes(resp.bytes)}")
|
||||
public static byte[] bytes(byte[] bytes){
|
||||
return DigestUtils.md5(bytes);
|
||||
}
|
||||
|
||||
@Comment("md5加密")
|
||||
@Example("${md5.bytes(resp.stream)}")
|
||||
public static byte[] bytes(InputStream stream) throws IOException {
|
||||
return DigestUtils.md5(stream);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 随机数/字符串 生成方法
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class RandomFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "random";
|
||||
}
|
||||
|
||||
@Comment("随机获取int")
|
||||
@Example("${random.randomInt(1,10)}")
|
||||
public static int randomInt(int min,int max){
|
||||
return RandomUtils.nextInt(min, max);
|
||||
}
|
||||
|
||||
@Comment("随机获取double")
|
||||
@Example("${random.randomDouble(1,10)}")
|
||||
public static double randomDouble(double min,double max){
|
||||
return RandomUtils.nextDouble(min, max);
|
||||
}
|
||||
|
||||
@Comment("随机获取long")
|
||||
@Example("${random.randomLong(1,10)}")
|
||||
public static long randomLong(long min,long max){
|
||||
return RandomUtils.nextLong(min, max);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param chars 字符个数
|
||||
* @param length 字符范围
|
||||
* @return String 随机字符串
|
||||
*/
|
||||
@Comment("随机获取字符串")
|
||||
@Example("${random.string('abcde',10)}")
|
||||
public static String string(String chars,int length){
|
||||
if (chars != null) {
|
||||
char[] newChars = new char[length];
|
||||
int len = chars.length();
|
||||
for (int i = 0; i < length; i++) {
|
||||
newChars[i] = chars.charAt(randomInt(0,len));
|
||||
}
|
||||
return new String(newChars);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* String 工具类 防止NPE
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Comment("string常用方法")
|
||||
public class StringFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "string";
|
||||
}
|
||||
|
||||
@Comment("截取字符串方法")
|
||||
@Example("${string.substring(str,5)}")
|
||||
public static String substring(String content, int beginIndex) {
|
||||
return content != null ? content.substring(beginIndex) : null;
|
||||
}
|
||||
|
||||
@Comment("截取字符串方法")
|
||||
@Example("${string.substring(str,0,str.length() - 1)}")
|
||||
public static String substring(String content, int beginIndex, int endIndex) {
|
||||
return content != null ? content.substring(beginIndex, endIndex) : null;
|
||||
}
|
||||
|
||||
@Comment("将字符串转为小写")
|
||||
@Example("${string.lower(str)}")
|
||||
public static String lower(String content) {
|
||||
return content != null ? content.toLowerCase() : null;
|
||||
}
|
||||
|
||||
@Comment("将字符串转为大写")
|
||||
@Example("${string.upper(str)}")
|
||||
public static String upper(String content) {
|
||||
return content != null ? content.toUpperCase() : null;
|
||||
}
|
||||
|
||||
@Comment("查找指定字符在字符串在中的位置")
|
||||
@Example("${string.indexOf(content,str)}")
|
||||
public static int indexOf(String content, String str) {
|
||||
return content != null ? content.indexOf(str) : -1;
|
||||
}
|
||||
|
||||
@Comment("查找指定字符在字符串中最后出现的位置")
|
||||
@Example("${string.lastIndexOf(content,str)}")
|
||||
public static int lastIndexOf(String content, String str) {
|
||||
return content != null ? content.lastIndexOf(str) : -1;
|
||||
}
|
||||
|
||||
@Comment("查找指定字符在字符串在中的位置")
|
||||
@Example("${string.indexOf(content,str,fromIndex)}")
|
||||
public static int indexOf(String content, String str, int fromIndex) {
|
||||
return content != null ? content.indexOf(str, fromIndex) : -1;
|
||||
}
|
||||
|
||||
@Comment("将字符串转为int")
|
||||
@Example("${string.toInt(value)}")
|
||||
public static int toInt(String value){
|
||||
return Integer.parseInt(value);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为Integer")
|
||||
@Example("${string.toInt(value,defaultValue)}")
|
||||
public static Integer toInt(String value,Integer defaultValue){
|
||||
try {
|
||||
return Integer.parseInt(value);
|
||||
} catch (Exception e) {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("字符串替换")
|
||||
@Example("${string.replace(content,source,target)}")
|
||||
public static String replace(String content,String source,String target){
|
||||
return content != null ? content.replace(source, target): null;
|
||||
}
|
||||
|
||||
@Comment("正则替换字符串")
|
||||
@Example("${string.replaceAll(content,regx,target)}")
|
||||
public static String replaceAll(String content,String regx,String target){
|
||||
return content != null ? content.replaceAll(regx, target): null;
|
||||
}
|
||||
|
||||
@Comment("正则替换字符串")
|
||||
@Example("${string.replaceFirst(content,regx,target)}")
|
||||
public static String replaceFirst(String content,String regx,String target){
|
||||
return content != null ? content.replaceFirst(regx, target): null;
|
||||
}
|
||||
|
||||
@Comment("正则替换字符串")
|
||||
@Example("${string.length(content)}")
|
||||
public static int length(String content){
|
||||
return content != null ? content.length() : -1;
|
||||
}
|
||||
|
||||
@Comment("去除字符串两边的空格")
|
||||
@Example("${string.trim(content)}")
|
||||
public static String trim(String content){
|
||||
return content != null ? content.trim() : null;
|
||||
}
|
||||
|
||||
@Comment("分割字符串")
|
||||
@Example("${string.split(content,regx)}")
|
||||
public static List<String> split(String content,String regx){
|
||||
return content != null ? Arrays.asList(content.split(regx)) : new ArrayList<>(0);
|
||||
}
|
||||
|
||||
@Comment("获取字符串的byte[]")
|
||||
@Example("${string.bytes(content)}")
|
||||
public static byte[] bytes(String content){
|
||||
return content != null ? content.getBytes() : null;
|
||||
}
|
||||
|
||||
@Comment("获取字符串的byte[]")
|
||||
@Example("${string.bytes(content,charset)}")
|
||||
public static byte[] bytes(String content,String charset){
|
||||
try {
|
||||
return content != null ? content.getBytes(charset) : null;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("byte[]转String")
|
||||
@Example("${string.newString(bytes)}")
|
||||
public static String newString(byte[] bytes){
|
||||
return bytes != null ? new String(bytes) : null;
|
||||
}
|
||||
|
||||
@Comment("byte[]转String")
|
||||
@Example("${string.newString(bytes,charset)}")
|
||||
public static String newString(byte[] bytes,String charset){
|
||||
try {
|
||||
return bytes != null ? new String(bytes,charset) : null;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("判断两个字符串是否相同")
|
||||
@Example("${string.newString(bytes,charset)}")
|
||||
public static boolean equals(String str1,String str2){
|
||||
return str1 == null ? str2 == null : str1.equals(str2);
|
||||
}
|
||||
|
||||
@Comment("生成UUID")
|
||||
@Example("${string.uuid()}")
|
||||
public static String uuid() {
|
||||
return UUID.randomUUID().toString().replace("-", "");
|
||||
}
|
||||
|
||||
@Comment("生成多个UUID")
|
||||
@Example("${string.uuid(size)}")
|
||||
public static List<String> uuids(Integer size) {
|
||||
List<String> ids = new ArrayList<String>();
|
||||
for (int i = 0; i < size; i++) {
|
||||
ids.add(UUID.randomUUID().toString().replace("-", ""));
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Created on 2019-12-06
|
||||
*
|
||||
* @author Octopus
|
||||
*/
|
||||
@Component
|
||||
@Comment("thread常用方法")
|
||||
public class ThreadFunctionExecutor implements FunctionExecutor {
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "thread";
|
||||
}
|
||||
|
||||
@Comment("线程休眠")
|
||||
@Example("${thread.sleep(1000L)}")
|
||||
public static void sleep(Long sleepTime){
|
||||
try {
|
||||
Thread.sleep(sleepTime);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* url 按指定字符集进行编码/解码 默认字符集(UTF-8) 工具类 防止NPE
|
||||
*/
|
||||
@Component
|
||||
public class UrlFunctionExecutor implements FunctionExecutor{
|
||||
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "url";
|
||||
}
|
||||
|
||||
@Comment("获取url参数")
|
||||
@Example("${url.parameter('http://www.baidu.com/s?wd=spider-flow','wd')}")
|
||||
public static String parameter(String url,String key){
|
||||
return parameterMap(url).get(key);
|
||||
}
|
||||
|
||||
@Comment("获取url全部参数")
|
||||
@Example("${url.parameterMap('http://www.baidu.com/s?wd=spider-flow&abbr=sf')}")
|
||||
public static Map<String,String> parameterMap(String url){
|
||||
Map<String,String> map = new HashMap<String,String>();
|
||||
int index = url.indexOf("?");
|
||||
if(index != -1) {
|
||||
String param = url.substring(index+1);
|
||||
if(StringUtils.isNotBlank(param)) {
|
||||
String[] params = param.split("&");
|
||||
for (String item : params) {
|
||||
String[] kv = item.split("=");
|
||||
if(kv.length > 0) {
|
||||
if(StringUtils.isNotBlank(kv[0])) {
|
||||
String value = "";
|
||||
if(StringUtils.isNotBlank(kv[1])) {
|
||||
int kv1Index = kv[1].indexOf("#");
|
||||
if(kv1Index != -1) {
|
||||
value = kv[1].substring(0,kv1Index);
|
||||
}else {
|
||||
value = kv[1];
|
||||
}
|
||||
}
|
||||
map.put(kv[0],value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@Comment("url编码")
|
||||
@Example("${url.encode('http://www.baidu.com/s?wd=spider-flow')}")
|
||||
public static String encode(String url){
|
||||
return encode(url,Charset.defaultCharset().name());
|
||||
}
|
||||
|
||||
@Comment("url编码")
|
||||
@Example("${url.encode('http://www.baidu.com/s?wd=spider-flow','UTF-8')}")
|
||||
public static String encode(String url,String charset){
|
||||
try {
|
||||
return url != null ? URLEncoder.encode(url,charset) : null;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("url解码")
|
||||
@Example("${url.decode(strVar)}")
|
||||
public static String decode(String url){
|
||||
return decode(url,Charset.defaultCharset().name());
|
||||
}
|
||||
|
||||
@Comment("url解码")
|
||||
@Example("${url.decode(strVar,'UTF-8')}")
|
||||
public static String decode(String url,String charset){
|
||||
try {
|
||||
return url != null ? URLDecoder.decode(url, charset) : null;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ArrayFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Object[].class;
|
||||
}
|
||||
|
||||
@Comment("获取数组的长度")
|
||||
@Example("${arrayVar.size()}")
|
||||
public static int size(Object[] objs){
|
||||
return objs.length;
|
||||
}
|
||||
|
||||
@Comment("将数组拼接起来")
|
||||
@Example("${arrayVar.join()}")
|
||||
public static String join(Object[] objs,String separator){
|
||||
return StringUtils.join(objs,separator);
|
||||
}
|
||||
|
||||
@Comment("将数组用separator拼接起来")
|
||||
@Example("${arrayVar.join('-')}")
|
||||
public static String join(Object[] objs){
|
||||
return StringUtils.join(objs);
|
||||
}
|
||||
|
||||
@Comment("将数组转为List")
|
||||
@Example("${arrayVar.toList()}")
|
||||
public static List<?> toList(Object[] objs){
|
||||
return Arrays.asList(objs);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class DateFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Date.class;
|
||||
}
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${dateVar.format()}")
|
||||
public static String format(Date date){
|
||||
return format(date, "yyyy-MM-dd HH:mm:ss");
|
||||
}
|
||||
|
||||
@Comment("格式化日期")
|
||||
@Example("${dateVar.format('yyyy-MM-dd HH:mm:ss')}")
|
||||
public static String format(Date date,String pattern){
|
||||
return DateFormatUtils.format(date,pattern);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.annotation.Return;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ElementFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Element.class;
|
||||
}
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${elementVar.xpath('//title/text()')}")
|
||||
@Return({Element.class,String.class})
|
||||
public static String xpath(Element element,String xpath){
|
||||
return ExtractUtils.getValueByXPath(element, xpath);
|
||||
}
|
||||
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${elementVar.xpaths('//h2/text()')}")
|
||||
@Return({Element.class,String.class})
|
||||
public static List<String> xpaths(Element element,String xpath){
|
||||
return ExtractUtils.getValuesByXPath(element, xpath);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regx('<title>(.*?)</title>')}")
|
||||
public static String regx(Element element,String regx){
|
||||
return ExtractUtils.getFirstMatcher(element.html(), regx, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regx('<title>(.*?)</title>',1)}")
|
||||
public static String regx(Element element,String regx,int groupIndex){
|
||||
return ExtractUtils.getFirstMatcher(element.html(), regx, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<String> regx(Element element,String regx,List<Integer> groups){
|
||||
return ExtractUtils.getFirstMatcher(element.html(), regx, groups);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regxs('<h2>(.*?)</h2>')}")
|
||||
public static List<String> regxs(Element element,String regx){
|
||||
return ExtractUtils.getMatchers(element.html(), regx, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regxs('<h2>(.*?)</h2>',1)}")
|
||||
public static List<String> regxs(Element element,String regx,int groupIndex){
|
||||
return ExtractUtils.getMatchers(element.html(), regx, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<List<String>> regxs(Element element,String regx,List<Integer> groups){
|
||||
return ExtractUtils.getMatchers(element.html(), regx, groups);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${elementVar.selector('div > a')}")
|
||||
public static Element selector(Element element,String cssQuery){
|
||||
return element.selectFirst(cssQuery);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${elementVar.selectors('div > a')}")
|
||||
public static Elements selectors(Element element,String cssQuery){
|
||||
return element.select(cssQuery);
|
||||
}
|
||||
|
||||
@Comment("获取同级节点")
|
||||
@Example("${elementVar.subling()}")
|
||||
public static Elements subling(Element element){
|
||||
return element.siblingElements();
|
||||
}
|
||||
|
||||
@Comment("获取上级节点")
|
||||
@Example("${elementVar.parent()}")
|
||||
public static Element parent(Element element){
|
||||
return element.parent();
|
||||
}
|
||||
|
||||
@Comment("获取上级节点")
|
||||
@Example("${elementVar.parents()}")
|
||||
public static Elements parents(Element element){
|
||||
return element.parents();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,162 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class ElementsFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Elements.class;
|
||||
}
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${elementsVar.xpath('//title/text()')}")
|
||||
public static String xpath(Elements elements,String xpath){
|
||||
return ExtractUtils.getValueByXPath(elements, xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpath提取内容")
|
||||
@Example("${elementsVar.xpaths('//h2/text()')}")
|
||||
public static List<String> xpaths(Elements elements,String xpath){
|
||||
return ExtractUtils.getValuesByXPath(elements, xpath);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regx('<title>(.*?)</title>')}")
|
||||
public static String regx(Elements elements,String regx){
|
||||
return ExtractUtils.getFirstMatcher(elements.html(), regx, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regx('<title>(.*?)</title>',1)}")
|
||||
public static String regx(Elements elements,String regx,int groupIndex){
|
||||
return ExtractUtils.getFirstMatcher(elements.html(), regx, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<String> regx(Elements elements,String regx,List<Integer> groups){
|
||||
return ExtractUtils.getFirstMatcher(elements.html(), regx, groups);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regxs('<h2>(.*?)</h2>')}")
|
||||
public static List<String> regxs(Elements elements,String regx){
|
||||
return ExtractUtils.getMatchers(elements.html(), regx, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regxs('<h2>(.*?)</h2>',1)}")
|
||||
public static List<String> regxs(Elements elements,String regx,int groupIndex){
|
||||
return ExtractUtils.getMatchers(elements.html(), regx, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取内容")
|
||||
@Example("${elementsVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<List<String>> regxs(Elements elements,String regx,List<Integer> groups){
|
||||
return ExtractUtils.getMatchers(elements.html(), regx, groups);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${elementsVar.selector('div > a')}")
|
||||
public static Element selector(Elements elements,String selector){
|
||||
Elements foundElements = elements.select(selector);
|
||||
if(foundElements.size() > 0){
|
||||
return foundElements.get(0);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Comment("返回所有attr")
|
||||
@Example("${elementsVar.attrs('href')}")
|
||||
public static List<String> attrs(Elements elements,String key){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.attr(key));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有value")
|
||||
@Example("${elementsVar.vals()}")
|
||||
public static List<String> vals(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.val());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有text")
|
||||
@Example("${elementsVar.texts()}")
|
||||
public static List<String> texts(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.text());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有html")
|
||||
@Example("${elementsVar.htmls()}")
|
||||
public static List<String> htmls(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.html());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有outerHtml")
|
||||
@Example("${elementsVar.outerHtmls()}")
|
||||
public static List<String> outerHtmls(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.outerHtml());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有ownTexts")
|
||||
@Example("${elementsVar.ownTexts()}")
|
||||
public static List<String> ownTexts(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.ownText());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("返回所有wholeText")
|
||||
@Example("${elementsVar.wholeTexts()}")
|
||||
public static List<String> wholeTexts(Elements elements){
|
||||
List<String> list = new ArrayList<>(elements.size());
|
||||
for (Element element : elements) {
|
||||
list.add(element.wholeText());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取内容")
|
||||
@Example("${elementsVar.selectors('div > a')}")
|
||||
public static Elements selectors(Elements elements,String selector){
|
||||
return elements.select(selector);
|
||||
}
|
||||
|
||||
@Comment("获取上级节点")
|
||||
@Example("${elementsVar.parents()}")
|
||||
public static Elements parents(Elements elements){
|
||||
return elements.parents();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Component
|
||||
public class ListFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return List.class;
|
||||
}
|
||||
|
||||
@Comment("获取list的长度")
|
||||
@Example("${listVar.length()}")
|
||||
public static int length(List<?> list){
|
||||
return list.size();
|
||||
}
|
||||
|
||||
@Comment("将list拼接起来")
|
||||
@Example("${listVar.join()}")
|
||||
public static String join(List<?> list){
|
||||
return StringUtils.join(list.toArray());
|
||||
}
|
||||
|
||||
@Comment("将list用separator拼接起来")
|
||||
@Example("${listVar.join('-')}")
|
||||
public static String join(List<?> list,String separator){
|
||||
if(list.size() == 1){
|
||||
return list.get(0).toString();
|
||||
}else{
|
||||
return StringUtils.join(list.toArray(),separator);
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("将list<String>排序")
|
||||
@Example("${listVar.sort()}")
|
||||
public static List<String> sort(List<String> list){
|
||||
Collections.sort(list);
|
||||
return list;
|
||||
}
|
||||
|
||||
@Comment("将list打乱顺序")
|
||||
@Example("${listVar.shuffle()}")
|
||||
public static List<?> shuffle(List<?> list){
|
||||
Collections.shuffle(list);
|
||||
return list;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class MapFunctionExtension implements FunctionExtension {
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Map.class;
|
||||
}
|
||||
|
||||
@Comment("将map转换为List")
|
||||
@Example("${mapmVar.toList('=')}")
|
||||
public static List<String> toList(Map<?,?> map,String separator){
|
||||
return map.entrySet().stream().map(entry-> entry.getKey() + separator + entry.getValue()).collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
|
||||
@Component
|
||||
public class ObjectFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return Object.class;
|
||||
}
|
||||
|
||||
@Comment("将对象转为string类型")
|
||||
@Example("${objVar.string()}")
|
||||
public static String string(Object obj){
|
||||
if (obj instanceof String) {
|
||||
return (String) obj;
|
||||
}
|
||||
return Objects.toString(obj);
|
||||
}
|
||||
|
||||
@Comment("根据jsonpath提取内容")
|
||||
@Example("${objVar.jsonpath('$.code')}")
|
||||
public static Object jsonpath(Object obj,String path){
|
||||
if(obj instanceof String){
|
||||
return ExtractUtils.getValueByJsonPath(JSON.parse((String)obj), path);
|
||||
}
|
||||
return ExtractUtils.getValueByJsonPath(obj, path);
|
||||
}
|
||||
|
||||
@Comment("睡眠等待一段时间")
|
||||
@Example("${objVar.sleep(1000)}")
|
||||
public static Object sleep(Object obj, int millis) {
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
} catch (InterruptedException ignored) {
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,127 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.annotation.Return;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.spiderflow.io.SpiderResponse;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ResponseFunctionExtension implements FunctionExtension {
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return SpiderResponse.class;
|
||||
}
|
||||
|
||||
@Comment("将请求结果转为Element对象")
|
||||
@Example("${resp.element()}")
|
||||
public static Element element(SpiderResponse response) {
|
||||
return Jsoup.parse(response.getHtml(),response.getUrl());
|
||||
}
|
||||
|
||||
@Comment("根据xpath在请求结果中查找")
|
||||
@Example("${resp.xpath('//title/text()')}")
|
||||
@Return({Element.class, String.class})
|
||||
public static String xpath(SpiderResponse response, String xpath) {
|
||||
return ExtractUtils.getValueByXPath(element(response), xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpath在请求结果中查找")
|
||||
@Example("${resp.xpaths('//a/@href')}")
|
||||
public static List<String> xpaths(SpiderResponse response, String xpath) {
|
||||
return ExtractUtils.getValuesByXPath(element(response), xpath);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regx('<title>(.*?)</title>')}")
|
||||
public static String regx(SpiderResponse response, String pattern) {
|
||||
return ExtractUtils.getFirstMatcher(response.getHtml(), pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regx('<title>(.*?)</title>',1)}")
|
||||
public static String regx(SpiderResponse response, String pattern, int groupIndex) {
|
||||
return ExtractUtils.getFirstMatcher(response.getHtml(), pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<String> regx(SpiderResponse response, String pattern, List<Integer> groups) {
|
||||
return ExtractUtils.getFirstMatcher(response.getHtml(), pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regxs('<h2>(.*?)</h2>')}")
|
||||
public static List<String> regxs(SpiderResponse response, String pattern) {
|
||||
return ExtractUtils.getMatchers(response.getHtml(), pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regxs('<h2>(.*?)</h2>',1)}")
|
||||
public static List<String> regxs(SpiderResponse response, String pattern, int groupIndex) {
|
||||
return ExtractUtils.getMatchers(response.getHtml(), pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取请求结果中的内容")
|
||||
@Example("${resp.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<List<String>> regxs(SpiderResponse response, String pattern, List<Integer> groups) {
|
||||
return ExtractUtils.getMatchers(response.getHtml(), pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取请求结果")
|
||||
@Example("${resp.selector('div > a')}")
|
||||
public static Element selector(SpiderResponse response, String selector) {
|
||||
return ElementFunctionExtension.selector(element(response), selector);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取请求结果")
|
||||
@Example("${resp.selectors('div > a')}")
|
||||
public static Elements selectors(SpiderResponse response, String selector) {
|
||||
return ElementFunctionExtension.selectors(element(response), selector);
|
||||
}
|
||||
|
||||
@Comment("根据jsonpath提取请求结果")
|
||||
@Example("${resp.jsonpath('$.code')}")
|
||||
public static Object jsonpath(SpiderResponse response, String path) {
|
||||
return ExtractUtils.getValueByJsonPath(response.getJson(), path);
|
||||
}
|
||||
|
||||
@Comment("获取页面上的链接")
|
||||
@Example("${resp.links()}")
|
||||
public static List<String> links(SpiderResponse response) {
|
||||
return ExtractUtils.getAttrBySelector(element(response), "a", "abs:href")
|
||||
.stream()
|
||||
.filter(link -> StringUtils.isNotBlank(link))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Comment("获取页面上的链接")
|
||||
@Example("${resp.links('https://www\\.xxx\\.com/xxxx/(.*?)')}")
|
||||
public static List<String> links(SpiderResponse response, String regx) {
|
||||
Pattern pattern = Pattern.compile(regx);
|
||||
return links(response)
|
||||
.stream()
|
||||
.filter(link -> pattern.matcher(link).matches())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Comment("获取当前页面所有图片链接")
|
||||
@Example("${resp.images()}")
|
||||
public static List<String> images(SpiderResponse response) {
|
||||
return ExtractUtils.getAttrBySelector(element(response), "img", "src")
|
||||
.stream()
|
||||
.filter(link -> StringUtils.isNotBlank(link))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.jdbc.support.rowset.SqlRowSet;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class SqlRowSetExtension implements FunctionExtension {
|
||||
public static Map<String, String[]> tableMetaMap = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return SqlRowSet.class;
|
||||
}
|
||||
|
||||
@Example("${rs.nextToMap()}")
|
||||
public static Map<String, Object> nextToMap(SqlRowSet sqlRowSet) {
|
||||
try {
|
||||
if (!sqlRowSet.next()) {
|
||||
return null;
|
||||
}
|
||||
String[] columnNames = sqlRowSet.getMetaData().getColumnNames();
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
for (String columnName : columnNames) {
|
||||
result.put(columnName, sqlRowSet.getObject(columnName));
|
||||
}
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,143 @@
|
||||
package org.spiderflow.core.executor.function.extension;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.apache.commons.text.StringEscapeUtils;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.parser.Parser;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.annotation.Return;
|
||||
import org.spiderflow.core.executor.function.DateFunctionExecutor;
|
||||
import org.spiderflow.core.utils.ExtractUtils;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class StringFunctionExtension implements FunctionExtension{
|
||||
|
||||
@Override
|
||||
public Class<?> support() {
|
||||
return String.class;
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regx('<title>(.*?)</title>')}")
|
||||
public static String regx(String source,String pattern){
|
||||
return ExtractUtils.getFirstMatcher(source, pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regx('<title>(.*?)</title>',1)}")
|
||||
public static String regx(String source,String pattern,int groupIndex){
|
||||
return ExtractUtils.getFirstMatcher(source, pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<String> regx(String source,String pattern,List<Integer> groups){
|
||||
return ExtractUtils.getFirstMatcher(source, pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regxs('<h2>(.*?)</h2>')}")
|
||||
public static List<String> regxs(String source,String pattern){
|
||||
return ExtractUtils.getMatchers(source, pattern, true);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regxs('<h2>(.*?)</h2>',1)}")
|
||||
public static List<String> regxs(String source,String pattern,int groupIndex){
|
||||
return ExtractUtils.getMatchers(source, pattern, groupIndex);
|
||||
}
|
||||
|
||||
@Comment("根据正则表达式提取String中的内容")
|
||||
@Example("${strVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
|
||||
public static List<List<String>> regxs(String source,String pattern,List<Integer> groups){
|
||||
return ExtractUtils.getMatchers(source, pattern, groups);
|
||||
}
|
||||
|
||||
@Comment("根据xpath在String变量中查找")
|
||||
@Example("${strVar.xpath('//title/text()')}")
|
||||
@Return({Element.class,String.class})
|
||||
public static String xpath(String source,String xpath){
|
||||
return ExtractUtils.getValueByXPath(element(source), xpath);
|
||||
}
|
||||
|
||||
@Comment("根据xpath在String变量中查找")
|
||||
@Example("${strVar.xpaths('//a/@href')}")
|
||||
public static List<String> xpaths(String source,String xpath){
|
||||
return ExtractUtils.getValuesByXPath(element(source), xpath);
|
||||
}
|
||||
|
||||
@Comment("将String变量转为Element对象")
|
||||
@Example("${strVar.element()}")
|
||||
public static Element element(String source){
|
||||
return Parser.xmlParser().parseInput(source,"");
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取")
|
||||
@Example("${strVar.selector('div > a')}")
|
||||
public static Element selector(String source,String cssQuery){
|
||||
return element(source).selectFirst(cssQuery);
|
||||
}
|
||||
|
||||
@Comment("根据css选择器提取")
|
||||
@Example("${strVar.selector('div > a')}")
|
||||
public static Elements selectors(String source,String cssQuery){
|
||||
return element(source).select(cssQuery);
|
||||
}
|
||||
|
||||
@Comment("将string转为json对象")
|
||||
@Example("${strVar.json()}")
|
||||
public static Object json(String source){
|
||||
return JSON.parse(source);
|
||||
}
|
||||
|
||||
@Comment("根据jsonpath提取内容")
|
||||
@Example("${strVar.jsonpath('$.code')}")
|
||||
public static Object jsonpath(String source,String jsonPath){
|
||||
return ExtractUtils.getValueByJsonPath(json(source), jsonPath);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为int类型")
|
||||
@Example("${strVar.toInt(0)}")
|
||||
public static Integer toInt(String source,int defaultValue){
|
||||
return NumberUtils.toInt(source, defaultValue);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为int类型")
|
||||
@Example("${strVar.toInt()}")
|
||||
public static Integer toInt(String source){
|
||||
return NumberUtils.toInt(source);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为double类型")
|
||||
@Example("${strVar.toDouble()}")
|
||||
public static Double toDouble(String source){
|
||||
return NumberUtils.toDouble(source);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为long类型")
|
||||
@Example("${strVar.toLong()}")
|
||||
public static Long toLong(String source){
|
||||
return NumberUtils.toLong(source);
|
||||
}
|
||||
|
||||
@Comment("将字符串转为date类型")
|
||||
@Example("${strVar.toDate('yyyy-MM-dd HH:mm:ss')}")
|
||||
public static Date toDate(String source,String pattern) throws ParseException{
|
||||
return DateFunctionExecutor.parse(source, pattern);
|
||||
}
|
||||
|
||||
@Comment("反转义字符串")
|
||||
@Example("${strVar.unescape()}")
|
||||
public static String unescape(String source){
|
||||
return StringEscapeUtils.unescapeJava(source);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class CommentExecutor implements ShapeExecutor{
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "comment";
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 等待执行结束执行器
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class ForkJoinExecutor implements ShapeExecutor {
|
||||
|
||||
/**
|
||||
* 缓存已完成节点的变量
|
||||
*/
|
||||
private Map<String, Map<String, Object>> cachedVariables = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "forkJoin";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allowExecuteNext(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
|
||||
String key = context.getId() + "-" + node.getNodeId();
|
||||
synchronized (node){
|
||||
boolean isDone = node.isDone();
|
||||
Map<String, Object> cached = cachedVariables.get(key);
|
||||
if(!isDone){
|
||||
if(cached == null){
|
||||
cached = new HashMap<>();
|
||||
cachedVariables.put(key, cached);
|
||||
}
|
||||
cached.putAll(variables);
|
||||
}else if(cached != null){
|
||||
//将缓存的变量存入到当前变量中,传递给下一级
|
||||
variables.putAll(cached);
|
||||
cachedVariables.remove(key);
|
||||
}
|
||||
return isDone;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.core.utils.ExpressionUtils;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 函数执行器
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class FunctionExecutor implements ShapeExecutor{
|
||||
|
||||
public static final String FUNCTION = "function";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FunctionExecutor.class);
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
List<Map<String, String>> functions = node.getListJsonValue(FUNCTION);
|
||||
for (Map<String, String> item : functions) {
|
||||
String function = item.get(FUNCTION);
|
||||
if(StringUtils.isNotBlank(function)){
|
||||
try {
|
||||
logger.debug("执行函数{}",function);
|
||||
ExpressionUtils.execute(function, variables);
|
||||
} catch (Exception e) {
|
||||
logger.error("执行函数{}失败,异常信息:{}",function,e);
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "function";
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 循环执行器
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class LoopExecutor implements ShapeExecutor{
|
||||
|
||||
public static final String LOOP_ITEM = "loopItem";
|
||||
|
||||
public static final String LOOP_START = "loopStart";
|
||||
|
||||
public static final String LOOP_END = "loopEnd";
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "loop";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.core.Spider;
|
||||
import org.spiderflow.core.model.SpiderFlow;
|
||||
import org.spiderflow.core.service.SpiderFlowService;
|
||||
import org.spiderflow.core.utils.SpiderFlowUtils;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 子流程执行器
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class ProcessExecutor implements ShapeExecutor{
|
||||
|
||||
public static final String FLOW_ID = "flowId";
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(ProcessExecutor.class);
|
||||
|
||||
@Autowired
|
||||
private SpiderFlowService spiderFlowService;
|
||||
|
||||
@Autowired
|
||||
private Spider spider;
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
String flowId = node.getStringJsonValue("flowId");
|
||||
SpiderFlow spiderFlow = spiderFlowService.getById(flowId);
|
||||
if(spiderFlow != null){
|
||||
logger.info("执行子流程:{}", spiderFlow.getName());
|
||||
SpiderNode root = SpiderFlowUtils.loadXMLFromString(spiderFlow.getXml());
|
||||
spider.executeNode(null,root,context,variables);
|
||||
}else{
|
||||
logger.info("执行子流程:{}失败，找不到该子流程", flowId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "process";
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 开始执行器
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class StartExecutor implements ShapeExecutor{
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "start";
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.core.utils.ExpressionUtils;
|
||||
import org.spiderflow.executor.ShapeExecutor;
|
||||
import org.spiderflow.model.SpiderNode;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 定义变量执行器
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class VariableExecutor implements ShapeExecutor{
|
||||
|
||||
private static final String VARIABLE_NAME = "variable-name";
|
||||
|
||||
private static final String VARIABLE_VALUE = "variable-value";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VariableExecutor.class);
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
List<Map<String, String>> variableList = node.getListJsonValue(VARIABLE_NAME,VARIABLE_VALUE);
|
||||
for (Map<String, String> nameValue : variableList) {
|
||||
Object value = null;
|
||||
String variableName = nameValue.get(VARIABLE_NAME);
|
||||
String variableValue = nameValue.get(VARIABLE_VALUE);
|
||||
try {
|
||||
value = ExpressionUtils.execute(variableValue, variables);
|
||||
logger.debug("设置变量{}={}",variableName,value);
|
||||
context.pause(node.getNodeId(),"common",variableName,value);
|
||||
} catch (Exception e) {
|
||||
logger.error("设置变量{}出错，异常信息：{}",variableName,e);
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
}
|
||||
variables.put(variableName, value);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportShape() {
|
||||
return "variable";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isThread() {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
package org.spiderflow.core.expression;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.spiderflow.ExpressionEngine;
|
||||
import org.spiderflow.core.expression.interpreter.Reflection;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.spiderflow.executor.FunctionExtension;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class DefaultExpressionEngine implements ExpressionEngine{
|
||||
|
||||
@Autowired
|
||||
private List<FunctionExecutor> functionExecutors;
|
||||
|
||||
@Autowired
|
||||
private List<FunctionExtension> functionExtensions;
|
||||
|
||||
@PostConstruct
|
||||
private void init(){
|
||||
for (FunctionExtension extension : functionExtensions) {
|
||||
Reflection.getInstance().registerExtensionClass(extension.support(), extension.getClass());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object execute(String expression, Map<String, Object> variables) {
|
||||
if(StringUtils.isBlank(expression)){
|
||||
return expression;
|
||||
}
|
||||
ExpressionTemplateContext context = new ExpressionTemplateContext(variables);
|
||||
for (FunctionExecutor executor : functionExecutors) {
|
||||
context.set(executor.getFunctionPrefix(), executor);
|
||||
}
|
||||
ExpressionGlobalVariables.getVariables().entrySet().forEach(entry->{
|
||||
context.set(entry.getKey(),ExpressionTemplate.create(entry.getValue()).render(context));
|
||||
});
|
||||
try {
|
||||
ExpressionTemplateContext.set(context);
|
||||
return ExpressionTemplate.create(expression).render(context);
|
||||
} finally {
|
||||
ExpressionTemplateContext.remove();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,118 @@
|
||||
|
||||
package org.spiderflow.core.expression;
|
||||
|
||||
import org.spiderflow.core.expression.parsing.Span;
|
||||
import org.spiderflow.core.expression.parsing.Span.Line;
|
||||
import org.spiderflow.core.expression.parsing.TokenStream;
|
||||
|
||||
/** All errors reported by the library go through the static functions of this class. */
|
||||
public class ExpressionError {
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Create an error message based on the provided message and stream, highlighting the line on which the error happened. If the
|
||||
* stream has more tokens, the next token will be highlighted. Otherwise the end of the source of the stream will be
|
||||
* highlighted.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Throws a {@link RuntimeException}
|
||||
* </p>
|
||||
*/
|
||||
public static void error (String message, TokenStream stream) {
|
||||
if (stream.hasMore())
|
||||
error(message, stream.consume().getSpan());
|
||||
else {
|
||||
String source = stream.getSource();
|
||||
if (source == null)
|
||||
error(message, new Span(" ", 0, 1));
|
||||
else
|
||||
error(message, new Span(source, source.length() - 1, source.length()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Create an error message based on the provided message and location, highlighting the location in the line on which the
|
||||
* error happened. Throws a {@link TemplateException} **/
|
||||
public static void error (String message, Span location, Throwable cause) {
|
||||
|
||||
Line line = location.getLine();
|
||||
message = "Error (" + line.getLineNumber() + "): " + message + "\n\n";
|
||||
message += line.getText();
|
||||
message += "\n";
|
||||
|
||||
int errorStart = location.getStart() - line.getStart();
|
||||
int errorEnd = errorStart + location.getText().length() - 1;
|
||||
for (int i = 0, n = line.getText().length(); i < n; i++) {
|
||||
boolean useTab = line.getText().charAt(i) == '\t';
|
||||
message += i >= errorStart && i <= errorEnd ? "^" : useTab ? "\t" : " ";
|
||||
}
|
||||
|
||||
if (cause == null)
|
||||
throw new TemplateException(message, location);
|
||||
else
|
||||
throw new TemplateException(message, location, cause);
|
||||
}
|
||||
|
||||
/** Create an error message based on the provided message and location, highlighting the location in the line on which the
|
||||
* error happened. Throws a {@link TemplateException} **/
|
||||
public static void error (String message, Span location) {
|
||||
error(message, location, null);
|
||||
}
|
||||
|
||||
/** Exception thrown by all basis-template code via {@link ExpressionError#error(String, Span)}. In case an error happens deep inside a
|
||||
* list of included templates, the {@link #getMessage()} method will return a condensed error message. **/
|
||||
public static class TemplateException extends RuntimeException {
|
||||
private static final long serialVersionUID = 1L;
|
||||
private final Span location;
|
||||
private final String errorMessage;
|
||||
|
||||
private TemplateException (String message, Span location) {
|
||||
super(message);
|
||||
this.errorMessage = message;
|
||||
this.location = location;
|
||||
}
|
||||
|
||||
public TemplateException (String message, Span location, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.errorMessage = message;
|
||||
this.location = location;
|
||||
}
|
||||
|
||||
/** Returns the location in the template at which the error happened. **/
|
||||
public Span getLocation () {
|
||||
return location;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage () {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
|
||||
if (getCause() == null || getCause() == this) {
|
||||
return super.getMessage();
|
||||
}
|
||||
|
||||
builder.append(errorMessage.substring(0, errorMessage.indexOf('\n')));
|
||||
builder.append("\n");
|
||||
|
||||
Throwable cause = getCause();
|
||||
while (cause != null && cause != this) {
|
||||
if (cause instanceof TemplateException) {
|
||||
TemplateException ex = (TemplateException)cause;
|
||||
if (ex.getCause() == null || ex.getCause() == ex)
|
||||
builder.append(ex.errorMessage);
|
||||
else
|
||||
builder.append(ex.errorMessage.substring(0, ex.errorMessage.indexOf('\n')));
|
||||
builder.append("\n");
|
||||
}
|
||||
cause = cause.getCause();
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
||||
public static class StringLiteralException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
package org.spiderflow.core.expression;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
public class ExpressionGlobalVariables {
|
||||
|
||||
private static Map<String, String> variables = new HashMap<>();
|
||||
|
||||
private static ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
|
||||
|
||||
public static void reset(Map<String, String> map){
|
||||
Lock lock = readWriteLock.writeLock();
|
||||
lock.lock();
|
||||
try {
|
||||
variables.clear();
|
||||
variables.putAll(map);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String, String> getVariables(){
|
||||
Lock lock = readWriteLock.readLock();
|
||||
lock.lock();
|
||||
try {
|
||||
return variables;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
|
||||
package org.spiderflow.core.expression;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.util.List;
|
||||
|
||||
import org.spiderflow.core.expression.interpreter.AstInterpreter;
|
||||
import org.spiderflow.core.expression.parsing.Ast;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Node;
|
||||
import org.spiderflow.core.expression.parsing.Parser;
|
||||
|
||||
|
||||
/** A template is loaded by a {@link TemplateLoader} from a file marked up with the basis-template language. The template can be
|
||||
* rendered to a {@link String} or {@link OutputStream} by calling one of the <code>render()</code> methods. The
|
||||
* {@link ExpressionTemplateContext} passed to the <code>render()</code> methods is used to look up variable values referenced in the
|
||||
* template. */
|
||||
public class ExpressionTemplate {
|
||||
private final List<Node> nodes;
|
||||
|
||||
/** Internal. Created by {@link Parser}. **/
|
||||
private ExpressionTemplate (List<Node> nodes) {
|
||||
this.nodes = nodes;
|
||||
}
|
||||
|
||||
public static ExpressionTemplate create(String source){
|
||||
return new ExpressionTemplate(Parser.parse(source));
|
||||
}
|
||||
|
||||
/** Internal. The AST nodes representing this template after parsing. See {@link Ast}. Used by {@link AstInterpreter}. **/
|
||||
public List<Node> getNodes () {
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/** Renders the template using the TemplateContext to resolve variable values referenced in the template. **/
|
||||
public Object render (ExpressionTemplateContext context) {
|
||||
return AstInterpreter.interpret(this, context);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
|
||||
package org.spiderflow.core.expression;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.spiderflow.core.expression.interpreter.AstInterpreter;
|
||||
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* A template context stores mappings from variable names to user provided variable values. A {@link ExpressionTemplate} is given a context
|
||||
* for rendering to resolve variable values it references in template expressions.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Internally, a template context is a stack of these mappings, similar to scopes in a programming language, and used as such by
|
||||
* the {@link AstInterpreter}.
|
||||
* </p>
|
||||
*/
|
||||
public class ExpressionTemplateContext {
|
||||
private final List<Map<String, Object>> scopes = new ArrayList<Map<String, Object>>();
|
||||
|
||||
/** Keeps track of previously allocated, unused scopes. New scopes are first tried to be retrieved from this pool to avoid
|
||||
* generating garbage. **/
|
||||
private final List<Map<String, Object>> freeScopes = new ArrayList<Map<String, Object>>();
|
||||
|
||||
private final static ThreadLocal<ExpressionTemplateContext> CONTEXT_THREAD_LOCAL = new ThreadLocal<>();
|
||||
|
||||
public static ExpressionTemplateContext get(){
|
||||
return CONTEXT_THREAD_LOCAL.get();
|
||||
}
|
||||
|
||||
public static void remove(){
|
||||
CONTEXT_THREAD_LOCAL.remove();
|
||||
}
|
||||
|
||||
public static void set(ExpressionTemplateContext context){
|
||||
CONTEXT_THREAD_LOCAL.set(context);
|
||||
}
|
||||
|
||||
public ExpressionTemplateContext () {
|
||||
push();
|
||||
}
|
||||
|
||||
public ExpressionTemplateContext(Map<String,Object> variables) {
|
||||
this();
|
||||
if(variables != null){
|
||||
variables.forEach(this::set);
|
||||
}
|
||||
}
|
||||
|
||||
/** Sets the value of the variable with the given name. If the variable already exists in one of the scopes, that variable is
|
||||
* set. Otherwise the variable is set on the last pushed scope. */
|
||||
public ExpressionTemplateContext set (String name, Object value) {
|
||||
for (int i = scopes.size() - 1; i >= 0; i--) {
|
||||
Map<String, Object> ctx = scopes.get(i);
|
||||
if (ctx.isEmpty()) continue;
|
||||
if (ctx.containsKey(name)) {
|
||||
ctx.put(name, value);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
scopes.get(scopes.size() - 1).put(name, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Sets the value of the variable with the given name on the last pushed scope **/
|
||||
public ExpressionTemplateContext setOnCurrentScope (String name, Object value) {
|
||||
scopes.get(scopes.size() - 1).put(name, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Internal. Returns the value of the variable with the given name, walking the scope stack from top to bottom, similar to how
|
||||
* scopes in programming languages are searched for variables. */
|
||||
public Object get (String name) {
|
||||
for (int i = scopes.size() - 1; i >= 0; i--) {
|
||||
Map<String, Object> ctx = scopes.get(i);
|
||||
if (ctx.isEmpty()) continue;
|
||||
Object value = ctx.get(name);
|
||||
if (value != null) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Internal. Returns all variables currently defined in this context. */
|
||||
public Set<String> getVariables () {
|
||||
Set<String> variables = new HashSet<String>();
|
||||
for (int i = 0, n = scopes.size(); i < n; i++) {
|
||||
variables.addAll(scopes.get(i).keySet());
|
||||
}
|
||||
return variables;
|
||||
}
|
||||
|
||||
/** Internal. Pushes a new "scope" onto the stack. **/
|
||||
public void push () {
|
||||
Map<String, Object> newScope = freeScopes.size() > 0 ? freeScopes.remove(freeScopes.size() - 1) : new HashMap<String, Object>();
|
||||
scopes.add(newScope);
|
||||
}
|
||||
|
||||
/** Internal. Pops the top of the "scope" stack. **/
|
||||
public void pop () {
|
||||
Map<String, Object> oldScope = scopes.remove(scopes.size() - 1);
|
||||
oldScope.clear();
|
||||
freeScopes.add(oldScope);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
|
||||
package org.spiderflow.core.expression.interpreter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.spiderflow.core.expression.ExpressionError;
|
||||
import org.spiderflow.core.expression.ExpressionError.TemplateException;
|
||||
import org.spiderflow.core.expression.ExpressionTemplate;
|
||||
import org.spiderflow.core.expression.ExpressionTemplateContext;
|
||||
import org.spiderflow.core.expression.parsing.Ast;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Node;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Text;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Interprets a Template given a TemplateContext to lookup variable values in and writes the evaluation results to an output
|
||||
* stream. Uses the global {@link Reflection} instance as returned by {@link Reflection#getInstance()} to access members and call
|
||||
* methods.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The interpeter traverses the AST as stored in {@link ExpressionTemplate#getNodes()}. the interpeter has a method for each AST node type
|
||||
* (see {@link Ast} that evaluates that node. A node may return a value, to be used in the interpretation of a parent node or to
|
||||
* be written to the output stream.
|
||||
* </p>
|
||||
**/
|
||||
public class AstInterpreter {
|
||||
public static Object interpret (ExpressionTemplate template, ExpressionTemplateContext context) {
|
||||
try {
|
||||
return interpretNodeList(template.getNodes(), template, context);
|
||||
} catch (Throwable t) {
|
||||
if (t instanceof TemplateException)
|
||||
throw (TemplateException)t;
|
||||
else {
|
||||
ExpressionError.error("执行表达式出错 " + t.getMessage(), template.getNodes().get(0).getSpan(),t);
|
||||
return null; // never reached
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Object interpretNodeList (List<Node> nodes, ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
|
||||
String result = "";
|
||||
for (int i = 0, n = nodes.size(); i < n; i++) {
|
||||
Node node = nodes.get(i);
|
||||
Object value = node.evaluate(template, context);
|
||||
if(node instanceof Text){
|
||||
result += node.getSpan().getText();
|
||||
}else if(value == null){
|
||||
if(i == 0 && i + 1 == n){
|
||||
return null;
|
||||
}
|
||||
result += "null";
|
||||
}else if(value instanceof String || value instanceof Number || value instanceof Boolean){
|
||||
if(i ==0 && i + 1 ==n){
|
||||
return value;
|
||||
}
|
||||
result += value;
|
||||
}else if(i + 1 < n){
|
||||
ExpressionError.error("表达式执行错误", node.getSpan());
|
||||
}else{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,365 @@
|
||||
|
||||
package org.spiderflow.core.expression.interpreter;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class JavaReflection extends Reflection {
|
||||
private final Map<Class<?>, Map<String, Field>> fieldCache = new ConcurrentHashMap<Class<?>, Map<String, Field>>();
|
||||
private final Map<Class<?>, Map<JavaReflection.MethodSignature, Method>> methodCache = new ConcurrentHashMap<Class<?>, Map<JavaReflection.MethodSignature, Method>>();
|
||||
private final Map<Class<?>, Map<String,List<Method>>> extensionmethodCache = new ConcurrentHashMap<>();
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
@Override
|
||||
public Object getField (Object obj, String name) {
|
||||
Class cls = obj instanceof Class ? (Class)obj : obj.getClass();
|
||||
Map<String, Field> fields = fieldCache.get(cls);
|
||||
if (fields == null) {
|
||||
fields = new ConcurrentHashMap<String, Field>();
|
||||
fieldCache.put(cls, fields);
|
||||
}
|
||||
|
||||
Field field = fields.get(name);
|
||||
if (field == null) {
|
||||
try {
|
||||
field = cls.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
fields.put(name, field);
|
||||
} catch (Throwable t) {
|
||||
// fall through, try super classes
|
||||
}
|
||||
|
||||
if (field == null) {
|
||||
Class parentClass = cls.getSuperclass();
|
||||
while (parentClass != Object.class && parentClass != null) {
|
||||
try {
|
||||
field = parentClass.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
fields.put(name, field);
|
||||
} catch (NoSuchFieldException e) {
|
||||
// fall through
|
||||
}
|
||||
parentClass = parentClass.getSuperclass();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return field;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getFieldValue (Object obj, Object field) {
|
||||
Field javaField = (Field)field;
|
||||
try {
|
||||
return javaField.get(obj);
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException("Couldn't get value of field '" + javaField.getName() + "' from object of type '" + obj.getClass().getSimpleName() + "'");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerExtensionClass(Class<?> target,Class<?> clazz){
|
||||
Method[] methods = clazz.getDeclaredMethods();
|
||||
if(methods != null){
|
||||
Map<String, List<Method>> cachedMethodMap = extensionmethodCache.get(target);
|
||||
if(cachedMethodMap == null){
|
||||
cachedMethodMap = new HashMap<>();
|
||||
extensionmethodCache.put(target,cachedMethodMap);
|
||||
}
|
||||
for (Method method : methods) {
|
||||
if(Modifier.isStatic(method.getModifiers()) && method.getParameterCount() > 0){
|
||||
List<Method> cachedList = cachedMethodMap.get(method.getName());
|
||||
if(cachedList == null){
|
||||
cachedList = new ArrayList<>();
|
||||
cachedMethodMap.put(method.getName(), cachedList);
|
||||
}
|
||||
cachedList.add(method);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getExtensionMethod(Object obj, String name, Object... arguments) {
|
||||
Class<?> cls = obj instanceof Class ? (Class<?>)obj : obj.getClass();
|
||||
if(cls.isArray()){
|
||||
cls = Object[].class;
|
||||
}
|
||||
return getExtensionMethod(cls,name,arguments);
|
||||
}
|
||||
|
||||
private Object getExtensionMethod(Class<?> cls, String name, Object... arguments) {
|
||||
if(cls == null){
|
||||
cls = Object.class;
|
||||
}
|
||||
Map<String, List<Method>> methodMap = extensionmethodCache.get(cls);
|
||||
if(methodMap != null){
|
||||
List<Method> methodList = methodMap.get(name);
|
||||
if(methodList != null){
|
||||
Class<?>[] parameterTypes = new Class[arguments.length + 1];
|
||||
parameterTypes[0] = cls;
|
||||
for (int i = 0; i < arguments.length; i++) {
|
||||
parameterTypes[i + 1] = arguments[i] == null ? null : arguments[i].getClass();
|
||||
}
|
||||
return findMethod(methodList, parameterTypes);
|
||||
}
|
||||
}
|
||||
if(cls != Object.class){
|
||||
Class<?>[] interfaces = cls.getInterfaces();
|
||||
if(interfaces != null){
|
||||
for (Class<?> clazz : interfaces) {
|
||||
Object method = getExtensionMethod(clazz,name,arguments);
|
||||
if(method != null){
|
||||
return method;
|
||||
}
|
||||
}
|
||||
}
|
||||
return getExtensionMethod(cls.getSuperclass(),name,arguments);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getMethod (Object obj, String name, Object... arguments) {
|
||||
Class<?> cls = obj instanceof Class ? (Class<?>)obj : obj.getClass();
|
||||
Map<JavaReflection.MethodSignature, Method> methods = methodCache.get(cls);
|
||||
if (methods == null) {
|
||||
methods = new ConcurrentHashMap<JavaReflection.MethodSignature, Method>();
|
||||
methodCache.put(cls, methods);
|
||||
}
|
||||
|
||||
Class<?>[] parameterTypes = new Class[arguments.length];
|
||||
for (int i = 0; i < arguments.length; i++) {
|
||||
parameterTypes[i] = arguments[i] == null ? null : arguments[i].getClass();
|
||||
}
|
||||
|
||||
JavaReflection.MethodSignature signature = new MethodSignature(name, parameterTypes);
|
||||
Method method = methods.get(signature);
|
||||
|
||||
if (method == null) {
|
||||
try {
|
||||
if (name == null) {
|
||||
method = findApply(cls);
|
||||
} else {
|
||||
method = findMethod(cls, name, parameterTypes);
|
||||
if(method == null && parameterTypes != null){
|
||||
method = findMethod(cls, name, new Class<?>[]{Object[].class});
|
||||
}
|
||||
}
|
||||
method.setAccessible(true);
|
||||
methods.put(signature, method);
|
||||
} catch (Throwable e) {
|
||||
// fall through
|
||||
}
|
||||
|
||||
if (method == null) {
|
||||
Class<?> parentClass = cls.getSuperclass();
|
||||
while (parentClass != Object.class && parentClass != null) {
|
||||
try {
|
||||
if (name == null)
|
||||
method = findApply(parentClass);
|
||||
else {
|
||||
method = findMethod(parentClass, name, parameterTypes);
|
||||
}
|
||||
method.setAccessible(true);
|
||||
methods.put(signature, method);
|
||||
} catch (Throwable e) {
|
||||
// fall through
|
||||
}
|
||||
parentClass = parentClass.getSuperclass();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return method;
|
||||
}
|
||||
|
||||
/** Returns the <code>apply()</code> method of a functional interface. **/
|
||||
private static Method findApply (Class<?> cls) {
|
||||
for (Method method : cls.getDeclaredMethods()) {
|
||||
if (method.getName().equals("apply")) return method;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Method findMethod (List<Method> methods, Class<?>[] parameterTypes) {
|
||||
Method foundMethod = null;
|
||||
int foundScore = 0;
|
||||
for (Method method : methods) {
|
||||
// Check if the types match.
|
||||
Class<?>[] otherTypes = method.getParameterTypes();
|
||||
if(parameterTypes.length != otherTypes.length){
|
||||
continue;
|
||||
}
|
||||
boolean match = true;
|
||||
int score = 0;
|
||||
for (int ii = 0, nn = parameterTypes.length; ii < nn; ii++) {
|
||||
Class<?> type = parameterTypes[ii];
|
||||
Class<?> otherType = otherTypes[ii];
|
||||
|
||||
if (!otherType.isAssignableFrom(type)) {
|
||||
score++;
|
||||
if (!isPrimitiveAssignableFrom(type, otherType)) {
|
||||
score++;
|
||||
if (!isCoercible(type, otherType)) {
|
||||
match = false;
|
||||
break;
|
||||
} else {
|
||||
score++;
|
||||
}
|
||||
}
|
||||
}else if(type == null && otherType.isPrimitive()){
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
if (foundMethod == null) {
|
||||
foundMethod = method;
|
||||
foundScore = score;
|
||||
} else {
|
||||
if (score < foundScore) {
|
||||
foundScore = score;
|
||||
foundMethod = method;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return foundMethod;
|
||||
}
|
||||
|
||||
/** Returns the method best matching the given signature, including type coercion, or null. **/
|
||||
private static Method findMethod (Class<?> cls, String name, Class<?>[] parameterTypes) {
|
||||
Method[] methods = cls.getDeclaredMethods();
|
||||
List<Method> methodList = new ArrayList<>();
|
||||
for (int i = 0, n = methods.length; i < n; i++) {
|
||||
Method method = methods[i];
|
||||
|
||||
// if neither name or parameter list size match, bail on this method
|
||||
if (!method.getName().equals(name)) continue;
|
||||
if (method.getParameterTypes().length != parameterTypes.length) continue;
|
||||
methodList.add(method);
|
||||
}
|
||||
return findMethod(methodList,parameterTypes);
|
||||
}
|
||||
|
||||
/** Returns whether the from type can be assigned to the to type, assuming either type is a (boxed) primitive type. We can
|
||||
* relax the type constraint a little, as we'll invoke a method via reflection. That means the from type will always be boxed,
|
||||
* as the {@link Method#invoke(Object, Object...)} method takes objects. **/
|
||||
private static boolean isPrimitiveAssignableFrom (Class<?> from, Class<?> to) {
|
||||
if ((from == Boolean.class || from == boolean.class) && (to == boolean.class || to == Boolean.class)) return true;
|
||||
if ((from == Integer.class || from == int.class) && (to == int.class || to == Integer.class)) return true;
|
||||
if ((from == Float.class || from == float.class) && (to == float.class || to == Float.class)) return true;
|
||||
if ((from == Double.class || from == double.class) && (to == double.class || to == Double.class)) return true;
|
||||
if ((from == Byte.class || from == byte.class) && (to == byte.class || to == Byte.class)) return true;
|
||||
if ((from == Short.class || from == short.class) && (to == short.class || to == Short.class)) return true;
|
||||
if ((from == Long.class || from == long.class) && (to == long.class || to == Long.class)) return true;
|
||||
if ((from == Character.class || from == char.class) && (to == char.class || to == Character.class)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static String[] getStringTypes(Object[] objects){
|
||||
String[] parameterTypes = new String[objects == null ? 0: objects.length];
|
||||
if(objects != null){
|
||||
for(int i=0,len = objects.length;i<len;i++){
|
||||
Object value = objects[i];
|
||||
parameterTypes[i] = value == null ? "null" : value.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
return parameterTypes;
|
||||
}
|
||||
|
||||
/** Returns whether the from type can be coerced to the to type. The coercion rules follow those of Java. See JLS 5.1.2
|
||||
* https://docs.oracle.com/javase/specs/jls/se7/html/jls-5.html **/
|
||||
private static boolean isCoercible (Class<?> from, Class<?> to) {
|
||||
if (from == Integer.class || from == int.class) {
|
||||
return to == float.class || to == Float.class || to == double.class || to == Double.class || to == long.class || to == Long.class;
|
||||
}
|
||||
|
||||
if (from == Float.class || from == float.class) {
|
||||
return to == double.class || to == Double.class;
|
||||
}
|
||||
|
||||
if (from == Double.class || from == double.class) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (from == Character.class || from == char.class) {
|
||||
return to == int.class || to == Integer.class || to == float.class || to == Float.class || to == double.class || to == Double.class || to == long.class
|
||||
|| to == Long.class;
|
||||
}
|
||||
|
||||
if (from == Byte.class || from == byte.class) {
|
||||
return to == int.class || to == Integer.class || to == float.class || to == Float.class || to == double.class || to == Double.class || to == long.class
|
||||
|| to == Long.class || to == short.class || to == Short.class;
|
||||
}
|
||||
|
||||
if (from == Short.class || from == short.class) {
|
||||
return to == int.class || to == Integer.class || to == float.class || to == Float.class || to == double.class || to == Double.class || to == long.class
|
||||
|| to == Long.class;
|
||||
}
|
||||
|
||||
if (from == Long.class || from == long.class) {
|
||||
return to == float.class || to == Float.class || to == double.class || to == Double.class;
|
||||
}
|
||||
|
||||
if(from == int[].class || from == Integer[].class){
|
||||
return to == Object[].class || to == float[].class || to == Float[].class || to == double[].class || to == Double[].class || to == long[].class || to == Long[].class;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object callMethod (Object obj, Object method, Object... arguments) {
|
||||
Method javaMethod = (Method)method;
|
||||
try {
|
||||
return javaMethod.invoke(obj, arguments);
|
||||
} catch (Throwable t) {
|
||||
throw new RuntimeException("Couldn't call method '" + javaMethod.getName() + "' with arguments '" + Arrays.toString(arguments)
|
||||
+ "' on object of type '" + obj.getClass().getSimpleName() + "'.", t);
|
||||
}
|
||||
}
|
||||
|
||||
private static class MethodSignature {
|
||||
private final String name;
|
||||
@SuppressWarnings("rawtypes") private final Class[] parameters;
|
||||
private final int hashCode;
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
public MethodSignature (String name, Class[] parameters) {
|
||||
this.name = name;
|
||||
this.parameters = parameters;
|
||||
final int prime = 31;
|
||||
int hash = 1;
|
||||
hash = prime * hash + ((name == null) ? 0 : name.hashCode());
|
||||
hash = prime * hash + Arrays.hashCode(parameters);
|
||||
hashCode = hash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode () {
|
||||
return hashCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals (Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
JavaReflection.MethodSignature other = (JavaReflection.MethodSignature)obj;
|
||||
if (name == null) {
|
||||
if (other.name != null) return false;
|
||||
} else if (!name.equals(other.name)) return false;
|
||||
if (!Arrays.equals(parameters, other.parameters)) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
|
||||
package org.spiderflow.core.expression.interpreter;
|
||||
|
||||
/** Used by {@link AstInterpreter} to access fields and methods of objects. This is a singleton class used by all
|
||||
* {@link AstInterpreter} instances. Replace the default implementation via {@link #setInstance(Reflection)}. The implementation
|
||||
* must be thread-safe. */
|
||||
public abstract class Reflection {
|
||||
private static Reflection instance = new JavaReflection();
|
||||
|
||||
/** Sets the Reflection instance to be used by all Template interpreters **/
|
||||
public synchronized static void setInstance (Reflection reflection) {
|
||||
instance = reflection;
|
||||
}
|
||||
|
||||
/** Returns the Reflection instance used to fetch field and call methods **/
|
||||
public synchronized static Reflection getInstance () {
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Returns an opaque handle to a field with the given name or null if the field could not be found **/
|
||||
public abstract Object getField (Object obj, String name);
|
||||
|
||||
/** Returns an opaque handle to the method with the given name best matching the signature implied by the given arguments, or
|
||||
* null if the method could not be found. If obj is an instance of Class, the matching static method is returned. If the name
|
||||
* is null and the object is a {@link FunctionalInterface}, the first declared method on the object is returned. **/
|
||||
public abstract Object getMethod (Object obj, String name, Object... arguments);
|
||||
|
||||
public abstract Object getExtensionMethod (Object obj, String name,Object ... arguments);
|
||||
|
||||
public abstract void registerExtensionClass(Class<?> target,Class<?> clazz);
|
||||
|
||||
/** Returns the value of the field from the object. The field must have been previously retrieved via
|
||||
* {@link #getField(Object, String)}. **/
|
||||
public abstract Object getFieldValue (Object obj, Object field);
|
||||
|
||||
/** Calls the method on the object with the given arguments. The method must have been previously retrieved via
|
||||
* {@link #getMethod(Object, String, Object...)}. **/
|
||||
public abstract Object callMethod (Object obj, Object method, Object... arguments);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,129 @@
|
||||
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
import javax.xml.transform.Source;
|
||||
|
||||
/** Wraps a the content of a {@link Source} and handles traversing the contained characters. Manages a current {@link Span} via
|
||||
* the {@link #startSpan()} and {@link #endSpan()} methods. */
|
||||
public class CharacterStream {
|
||||
private final String source;
|
||||
private int index = 0;
|
||||
private final int end;
|
||||
|
||||
private int spanStart = 0;
|
||||
|
||||
public CharacterStream (String source) {
|
||||
this(source, 0, source.length());
|
||||
}
|
||||
|
||||
public CharacterStream (String source, int start, int end) {
|
||||
if (start > end) throw new IllegalArgumentException("Start must be <= end.");
|
||||
if (start < 0) throw new IndexOutOfBoundsException("Start must be >= 0.");
|
||||
if (start > Math.max(0, source.length() - 1)) throw new IndexOutOfBoundsException("Start outside of string.");
|
||||
if (end > source.length()) throw new IndexOutOfBoundsException("End outside of string.");
|
||||
|
||||
this.source = source;
|
||||
this.index = start;
|
||||
this.end = end;
|
||||
}
|
||||
|
||||
/** Returns whether there are more characters in the stream **/
|
||||
public boolean hasMore () {
|
||||
return index < end;
|
||||
}
|
||||
|
||||
/** Returns the next character without advancing the stream **/
|
||||
public char peek () {
|
||||
if (!hasMore()) throw new RuntimeException("No more characters in stream.");
|
||||
return source.charAt(index++);
|
||||
}
|
||||
|
||||
/** Returns the next character and advance the stream **/
|
||||
public char consume () {
|
||||
if (!hasMore()) throw new RuntimeException("No more characters in stream.");
|
||||
return source.charAt(index++);
|
||||
}
|
||||
|
||||
/** Matches the given needle with the next characters. Returns true if the needle is matched, false otherwise. If there's a
|
||||
* match and consume is true, the stream is advanced by the needle's length. */
|
||||
public boolean match (String needle, boolean consume) {
|
||||
int needleLength = needle.length();
|
||||
if(needleLength + index >end){
|
||||
return false;
|
||||
}
|
||||
for (int i = 0, j = index; i < needleLength; i++, j++) {
|
||||
if (index >= end) return false;
|
||||
if (needle.charAt(i) != source.charAt(j)) return false;
|
||||
}
|
||||
if (consume) index += needleLength;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns whether the next character is a digit and optionally consumes it. **/
|
||||
public boolean matchDigit (boolean consume) {
|
||||
if (index >= end) return false;
|
||||
char c = source.charAt(index);
|
||||
if (Character.isDigit(c)) {
|
||||
if (consume) index++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Returns whether the next character is the start of an identifier and optionally consumes it. Adheres to
|
||||
* {@link Character#isJavaIdentifierStart(char)}. **/
|
||||
public boolean matchIdentifierStart (boolean consume) {
|
||||
if (index >= end) return false;
|
||||
char c = source.charAt(index);
|
||||
if (Character.isJavaIdentifierStart(c) || c == '@') {
|
||||
if (consume) index++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Returns whether the next character is the start of an identifier and optionally consumes it. Adheres to
|
||||
* {@link Character#isJavaIdentifierPart(char)}. **/
|
||||
public boolean matchIdentifierPart (boolean consume) {
|
||||
if (index >= end) return false;
|
||||
char c = source.charAt(index);
|
||||
if (Character.isJavaIdentifierPart(c)) {
|
||||
if (consume) index++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Skips any number of successive whitespace characters. **/
|
||||
public void skipWhiteSpace () {
|
||||
while (true) {
|
||||
if (index >= end) return;
|
||||
char c = source.charAt(index);
|
||||
if (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
|
||||
index++;
|
||||
continue;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Start a new Span at the current stream position. Call {@link #endSpan()} to complete the span. **/
|
||||
public void startSpan () {
|
||||
spanStart = index;
|
||||
}
|
||||
|
||||
/** Completes the span started with {@link #startSpan()} at the current stream position. **/
|
||||
public Span endSpan () {
|
||||
return new Span(source, spanStart, index);
|
||||
}
|
||||
|
||||
public boolean isSpanEmpty () {
|
||||
return spanStart == this.index;
|
||||
}
|
||||
|
||||
/** Returns the current character position in the stream. **/
|
||||
public int getPosition () {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,242 @@
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.xml.transform.Source;
|
||||
|
||||
import org.spiderflow.core.expression.ExpressionError;
|
||||
import org.spiderflow.core.expression.ExpressionTemplate;
|
||||
import org.spiderflow.core.expression.parsing.Ast.BinaryOperation;
|
||||
import org.spiderflow.core.expression.parsing.Ast.BooleanLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.ByteLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.CharacterLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.DoubleLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Expression;
|
||||
import org.spiderflow.core.expression.parsing.Ast.FloatLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.FunctionCall;
|
||||
import org.spiderflow.core.expression.parsing.Ast.IntegerLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.ListLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.LongLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.MapLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.MapOrArrayAccess;
|
||||
import org.spiderflow.core.expression.parsing.Ast.MemberAccess;
|
||||
import org.spiderflow.core.expression.parsing.Ast.MethodCall;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Node;
|
||||
import org.spiderflow.core.expression.parsing.Ast.NullLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.ShortLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.StringLiteral;
|
||||
import org.spiderflow.core.expression.parsing.Ast.TernaryOperation;
|
||||
import org.spiderflow.core.expression.parsing.Ast.Text;
|
||||
import org.spiderflow.core.expression.parsing.Ast.UnaryOperation;
|
||||
import org.spiderflow.core.expression.parsing.Ast.VariableAccess;
|
||||
|
||||
|
||||
/** Parses a {@link Source} into a {@link ExpressionTemplate}. The implementation is a simple recursive descent parser with a lookahead of
|
||||
* 1. **/
|
||||
public class Parser {
|
||||
|
||||
/** Parses a {@link Source} into a {@link ExpressionTemplate}. **/
|
||||
public static List<Node> parse (String source) {
|
||||
List<Node> nodes = new ArrayList<Node>();
|
||||
TokenStream stream = new TokenStream(new Tokenizer().tokenize(source));
|
||||
while (stream.hasMore()) {
|
||||
nodes.add(parseStatement(stream));
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/** Parse a statement, which may either be a text block, if statement, for statement, while statement, macro definition,
|
||||
* include statement or an expression. **/
|
||||
private static Node parseStatement (TokenStream tokens) {
|
||||
Node result = null;
|
||||
|
||||
if (tokens.match(TokenType.TextBlock, false))
|
||||
result = new Text(tokens.consume().getSpan());
|
||||
else
|
||||
result = parseExpression(tokens);
|
||||
|
||||
// consume semi-colons as statement delimiters
|
||||
while (tokens.match(";", true))
|
||||
;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private static Expression parseExpression (TokenStream stream) {
|
||||
return parseTernaryOperator(stream);
|
||||
}
|
||||
|
||||
private static Expression parseTernaryOperator (TokenStream stream) {
|
||||
Expression condition = parseBinaryOperator(stream, 0);
|
||||
if (stream.match(TokenType.Questionmark, true)) {
|
||||
Expression trueExpression = parseTernaryOperator(stream);
|
||||
stream.expect(TokenType.Colon);
|
||||
Expression falseExpression = parseTernaryOperator(stream);
|
||||
return new TernaryOperation(condition, trueExpression, falseExpression);
|
||||
} else {
|
||||
return condition;
|
||||
}
|
||||
}
|
||||
|
||||
private static final TokenType[][] binaryOperatorPrecedence = new TokenType[][] {new TokenType[] {TokenType.Assignment},
|
||||
new TokenType[] {TokenType.Or, TokenType.And, TokenType.Xor}, new TokenType[] {TokenType.Equal, TokenType.NotEqual},
|
||||
new TokenType[] {TokenType.Less, TokenType.LessEqual, TokenType.Greater, TokenType.GreaterEqual}, new TokenType[] {TokenType.Plus, TokenType.Minus},
|
||||
new TokenType[] {TokenType.ForwardSlash, TokenType.Asterisk, TokenType.Percentage}};
|
||||
|
||||
private static Expression parseBinaryOperator (TokenStream stream, int level) {
|
||||
int nextLevel = level + 1;
|
||||
Expression left = nextLevel == binaryOperatorPrecedence.length ? parseUnaryOperator(stream) : parseBinaryOperator(stream, nextLevel);
|
||||
|
||||
TokenType[] operators = binaryOperatorPrecedence[level];
|
||||
while (stream.hasMore() && stream.match(false, operators)) {
|
||||
Token operator = stream.consume();
|
||||
Expression right = nextLevel == binaryOperatorPrecedence.length ? parseUnaryOperator(stream) : parseBinaryOperator(stream, nextLevel);
|
||||
left = new BinaryOperation(left, operator, right);
|
||||
}
|
||||
|
||||
return left;
|
||||
}
|
||||
|
||||
private static final TokenType[] unaryOperators = new TokenType[] {TokenType.Not, TokenType.Plus, TokenType.Minus};
|
||||
|
||||
private static Expression parseUnaryOperator (TokenStream stream) {
|
||||
if (stream.match(false, unaryOperators)) {
|
||||
return new UnaryOperation(stream.consume(), parseUnaryOperator(stream));
|
||||
} else {
|
||||
if (stream.match(TokenType.LeftParantheses, true)) {
|
||||
Expression expression = parseExpression(stream);
|
||||
stream.expect(TokenType.RightParantheses);
|
||||
return expression;
|
||||
} else {
|
||||
return parseAccessOrCallOrLiteral(stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Expression parseAccessOrCallOrLiteral (TokenStream stream) {
|
||||
if (stream.match(TokenType.Identifier, false)) {
|
||||
return parseAccessOrCall(stream,TokenType.Identifier);
|
||||
} else if (stream.match(TokenType.LeftCurly, false)) {
|
||||
return parseMapLiteral(stream);
|
||||
} else if (stream.match(TokenType.LeftBracket, false)) {
|
||||
return parseListLiteral(stream);
|
||||
} else if (stream.match(TokenType.StringLiteral, false)) {
|
||||
if(stream.hasNext()){
|
||||
if(stream.next().getType() == TokenType.Period){
|
||||
stream.prev();
|
||||
return parseAccessOrCall(stream,TokenType.StringLiteral);
|
||||
}
|
||||
stream.prev();
|
||||
}
|
||||
|
||||
return new StringLiteral(stream.expect(TokenType.StringLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.BooleanLiteral, false)) {
|
||||
return new BooleanLiteral(stream.expect(TokenType.BooleanLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.DoubleLiteral, false)) {
|
||||
return new DoubleLiteral(stream.expect(TokenType.DoubleLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.FloatLiteral, false)) {
|
||||
return new FloatLiteral(stream.expect(TokenType.FloatLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.ByteLiteral, false)) {
|
||||
return new ByteLiteral(stream.expect(TokenType.ByteLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.ShortLiteral, false)) {
|
||||
return new ShortLiteral(stream.expect(TokenType.ShortLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.IntegerLiteral, false)) {
|
||||
return new IntegerLiteral(stream.expect(TokenType.IntegerLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.LongLiteral, false)) {
|
||||
return new LongLiteral(stream.expect(TokenType.LongLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.CharacterLiteral, false)) {
|
||||
return new CharacterLiteral(stream.expect(TokenType.CharacterLiteral).getSpan());
|
||||
} else if (stream.match(TokenType.NullLiteral, false)) {
|
||||
return new NullLiteral(stream.expect(TokenType.NullLiteral).getSpan());
|
||||
} else {
|
||||
ExpressionError.error("Expected a variable, field, map, array, function or method call, or literal.", stream);
|
||||
return null; // not reached
|
||||
}
|
||||
}
|
||||
|
||||
private static Expression parseMapLiteral (TokenStream stream) {
|
||||
Span openCurly = stream.expect(TokenType.LeftCurly).getSpan();
|
||||
|
||||
List<Token> keys = new ArrayList<>();
|
||||
List<Expression> values = new ArrayList<>();
|
||||
while (stream.hasMore() && !stream.match("}", false)) {
|
||||
if(stream.match(TokenType.StringLiteral, false)){
|
||||
keys.add(stream.expect(TokenType.StringLiteral));
|
||||
}else{
|
||||
keys.add(stream.expect(TokenType.Identifier));
|
||||
}
|
||||
|
||||
stream.expect(":");
|
||||
values.add(parseExpression(stream));
|
||||
if (!stream.match("}", false)) stream.expect(TokenType.Comma);
|
||||
}
|
||||
Span closeCurly = stream.expect("}").getSpan();
|
||||
return new MapLiteral(new Span(openCurly, closeCurly), keys, values);
|
||||
}
|
||||
|
||||
private static Expression parseListLiteral (TokenStream stream) {
|
||||
Span openBracket = stream.expect(TokenType.LeftBracket).getSpan();
|
||||
|
||||
List<Expression> values = new ArrayList<>();
|
||||
while (stream.hasMore() && !stream.match(TokenType.RightBracket, false)) {
|
||||
values.add(parseExpression(stream));
|
||||
if (!stream.match(TokenType.RightBracket, false)) stream.expect(TokenType.Comma);
|
||||
}
|
||||
|
||||
Span closeBracket = stream.expect(TokenType.RightBracket).getSpan();
|
||||
return new ListLiteral(new Span(openBracket, closeBracket), values);
|
||||
}
|
||||
|
||||
private static Expression parseAccessOrCall (TokenStream stream,TokenType tokenType) {
|
||||
//Span identifier = stream.expect(TokenType.Identifier);
|
||||
//Expression result = new VariableAccess(identifier);
|
||||
Span identifier = stream.expect(tokenType).getSpan();
|
||||
Expression result = tokenType == TokenType.StringLiteral ? new StringLiteral(identifier) :new VariableAccess(identifier);
|
||||
|
||||
while (stream.hasMore() && stream.match(false, TokenType.LeftParantheses, TokenType.LeftBracket, TokenType.Period)) {
|
||||
|
||||
// function or method call
|
||||
if (stream.match(TokenType.LeftParantheses, false)) {
|
||||
List<Expression> arguments = parseArguments(stream);
|
||||
Span closingSpan = stream.expect(TokenType.RightParantheses).getSpan();
|
||||
if (result instanceof VariableAccess || result instanceof MapOrArrayAccess)
|
||||
result = new FunctionCall(new Span(result.getSpan(), closingSpan), result, arguments);
|
||||
else if (result instanceof MemberAccess) {
|
||||
result = new MethodCall(new Span(result.getSpan(), closingSpan), (MemberAccess)result, arguments);
|
||||
} else {
|
||||
ExpressionError.error("Expected a variable, field or method.", stream);
|
||||
}
|
||||
}
|
||||
|
||||
// map or array access
|
||||
else if (stream.match(TokenType.LeftBracket, true)) {
|
||||
Expression keyOrIndex = parseExpression(stream);
|
||||
Span closingSpan = stream.expect(TokenType.RightBracket).getSpan();
|
||||
result = new MapOrArrayAccess(new Span(result.getSpan(), closingSpan), result, keyOrIndex);
|
||||
}
|
||||
|
||||
// field or method access
|
||||
else if (stream.match(TokenType.Period, true)) {
|
||||
identifier = stream.expect(TokenType.Identifier).getSpan();
|
||||
result = new MemberAccess(result, identifier);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Does not consume the closing parentheses. **/
|
||||
private static List<Expression> parseArguments (TokenStream stream) {
|
||||
stream.expect(TokenType.LeftParantheses);
|
||||
List<Expression> arguments = new ArrayList<Expression>();
|
||||
while (stream.hasMore() && !stream.match(TokenType.RightParantheses, false)) {
|
||||
arguments.add(parseExpression(stream));
|
||||
if (!stream.match(TokenType.RightParantheses, false)) stream.expect(TokenType.Comma);
|
||||
}
|
||||
return arguments;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,145 @@
|
||||
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
/** A span within a source string denoted by start and end index, with the latter being exclusive. */
|
||||
public class Span {
|
||||
/** the source string this span refers to **/
|
||||
private final String source;
|
||||
|
||||
/** start index in source string, starting at 0 **/
|
||||
private int start;
|
||||
|
||||
/** end index in source string, exclusive, starting at 0 **/
|
||||
private int end;
|
||||
|
||||
/** Cached String instance to reduce pressure on GC **/
|
||||
private final String cachedText;
|
||||
|
||||
public Span (String source) {
|
||||
this(source, 0, source.length());
|
||||
}
|
||||
|
||||
public Span (String source, int start, int end) {
|
||||
if (start > end) throw new IllegalArgumentException("Start must be <= end.");
|
||||
if (start < 0) throw new IndexOutOfBoundsException("Start must be >= 0.");
|
||||
if (start > source.length() - 1)
|
||||
throw new IndexOutOfBoundsException("Start outside of string.");
|
||||
if (end >source.length()) throw new IndexOutOfBoundsException("End outside of string.");
|
||||
|
||||
this.source = source;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.cachedText = source.substring(start, end);
|
||||
}
|
||||
|
||||
public Span (Span start, Span end) {
|
||||
if (!start.source.equals(end.source)) throw new IllegalArgumentException("The two spans do not reference the same source.");
|
||||
if (start.start > end.end) throw new IllegalArgumentException("Start must be <= end.");
|
||||
if (start.start < 0) throw new IndexOutOfBoundsException("Start must be >= 0.");
|
||||
if (start.start > start.source.length() - 1) throw new IndexOutOfBoundsException("Start outside of string.");
|
||||
if (end.end > start.source.length()) throw new IndexOutOfBoundsException("End outside of string.");
|
||||
|
||||
this.source = start.source;
|
||||
this.start = start.start;
|
||||
this.end = end.end;
|
||||
this.cachedText = source.substring(this.start, this.end);
|
||||
}
|
||||
|
||||
/** Returns the text referenced by this span **/
|
||||
public String getText () {
|
||||
return cachedText;
|
||||
}
|
||||
|
||||
/** Returns the index of the first character of this span. **/
|
||||
public int getStart () {
|
||||
return start;
|
||||
}
|
||||
|
||||
/** Returns the index of the last character of this span plus 1. **/
|
||||
public int getEnd () {
|
||||
return end;
|
||||
}
|
||||
|
||||
/** Returns the source string this span references. **/
|
||||
public String getSource () {
|
||||
return source;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString () {
|
||||
return "Span [text=" + getText() + ", start=" + start + ", end=" + end + "]";
|
||||
}
|
||||
|
||||
/** Returns the line this span is on. Does not return a correct result for spans across multiple lines. **/
|
||||
public Line getLine () {
|
||||
int lineStart = start;
|
||||
while (true) {
|
||||
if (lineStart < 0) break;
|
||||
char c = source.charAt(lineStart);
|
||||
if (c == '\n') {
|
||||
lineStart = lineStart + 1;
|
||||
break;
|
||||
}
|
||||
lineStart--;
|
||||
}
|
||||
if (lineStart < 0) lineStart = 0;
|
||||
|
||||
int lineEnd = end;
|
||||
while (true) {
|
||||
if (lineEnd > source.length() - 1) break;
|
||||
char c = source.charAt(lineEnd);
|
||||
if (c == '\n') {
|
||||
break;
|
||||
}
|
||||
lineEnd++;
|
||||
}
|
||||
|
||||
int lineNumber = 0;
|
||||
int idx = lineStart;
|
||||
while (idx > 0) {
|
||||
char c = source.charAt(idx);
|
||||
if (c == '\n') {
|
||||
lineNumber++;
|
||||
}
|
||||
idx--;
|
||||
}
|
||||
lineNumber++;
|
||||
|
||||
return new Line(source, lineStart, lineEnd, lineNumber);
|
||||
}
|
||||
|
||||
/** A line within a Source **/
|
||||
public static class Line {
|
||||
private final String source;
|
||||
private final int start;
|
||||
private final int end;
|
||||
private final int lineNumber;
|
||||
|
||||
public Line (String source, int start, int end, int lineNumber) {
|
||||
this.source = source;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.lineNumber = lineNumber;
|
||||
}
|
||||
|
||||
public String getSource () {
|
||||
return source;
|
||||
}
|
||||
|
||||
public int getStart () {
|
||||
return start;
|
||||
}
|
||||
|
||||
public int getEnd () {
|
||||
return end;
|
||||
}
|
||||
|
||||
public int getLineNumber () {
|
||||
return lineNumber;
|
||||
}
|
||||
|
||||
public String getText () {
|
||||
return source.substring(start, end);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
/** A token produced by the {@link Tokenizer}. */
|
||||
public class Token {
|
||||
private final TokenType type;
|
||||
private final Span span;
|
||||
|
||||
public Token (TokenType type, Span span) {
|
||||
this.type = type;
|
||||
this.span = span;
|
||||
}
|
||||
|
||||
public TokenType getType () {
|
||||
return type;
|
||||
}
|
||||
|
||||
public Span getSpan () {
|
||||
return span;
|
||||
}
|
||||
|
||||
public String getText () {
|
||||
return span.getText();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString () {
|
||||
return "Token [type=" + type + ", span=" + span + "]";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.xml.transform.Source;
|
||||
|
||||
import org.spiderflow.core.expression.ExpressionError;
|
||||
|
||||
|
||||
/** Iterates over a list of {@link Token} instances, provides methods to match expected tokens and throw errors in case of a
|
||||
* mismatch. */
|
||||
public class TokenStream {
|
||||
private final List<Token> tokens;
|
||||
private int index;
|
||||
private final int end;
|
||||
|
||||
public TokenStream (List<Token> tokens) {
|
||||
this.tokens = tokens;
|
||||
this.index = 0;
|
||||
this.end = tokens.size();
|
||||
}
|
||||
|
||||
/** Returns whether there are more tokens in the stream. **/
|
||||
public boolean hasMore () {
|
||||
return index < end;
|
||||
}
|
||||
|
||||
public boolean hasNext(){
|
||||
return index + 1 < end;
|
||||
}
|
||||
|
||||
public boolean hasPrev(){
|
||||
return index > 0;
|
||||
}
|
||||
|
||||
/** Consumes the next token and returns it. **/
|
||||
public Token consume () {
|
||||
if (!hasMore()) throw new RuntimeException("Reached the end of the source.");
|
||||
return tokens.get(index++);
|
||||
}
|
||||
|
||||
public Token next(){
|
||||
if (!hasMore()) throw new RuntimeException("Reached the end of the source.");
|
||||
return tokens.get(++index);
|
||||
}
|
||||
|
||||
public Token prev(){
|
||||
if(index == 0){
|
||||
throw new RuntimeException("Reached the end of the source.");
|
||||
}
|
||||
return tokens.get(--index);
|
||||
}
|
||||
|
||||
/** Checks if the next token has the give type and optionally consumes, or throws an error if the next token did not match the
|
||||
* type. */
|
||||
public Token expect (TokenType type) {
|
||||
boolean result = match(type, true);
|
||||
if (!result) {
|
||||
Token token = index < tokens.size() ? tokens.get(index) : null;
|
||||
Span span = token != null ? token.getSpan() : null;
|
||||
if (span == null)
|
||||
ExpressionError.error("Expected '" + type.getError() + "', but reached the end of the source.", this);
|
||||
else
|
||||
ExpressionError.error("Expected '" + type.getError() + "', but got '" + token.getText() + "'", span);
|
||||
return null; // never reached
|
||||
} else {
|
||||
return tokens.get(index - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks if the next token matches the given text and optionally consumes, or throws an error if the next token did not match
|
||||
* the text. */
|
||||
public Token expect (String text) {
|
||||
boolean result = match(text, true);
|
||||
if (!result) {
|
||||
Token token = index < tokens.size() ? tokens.get(index) : null;
|
||||
Span span = token != null ? token.getSpan() : null;
|
||||
if (span == null)
|
||||
ExpressionError.error("Expected '" + text + "', but reached the end of the source.", this);
|
||||
else
|
||||
ExpressionError.error("Expected '" + text + "', but got '" + token.getText() + "'", span);
|
||||
return null; // never reached
|
||||
} else {
|
||||
return tokens.get(index - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/** Matches and optionally consumes the next token in case of a match. Returns whether the token matched. */
|
||||
public boolean match (TokenType type, boolean consume) {
|
||||
if (index >= end) return false;
|
||||
if (tokens.get(index).getType() == type) {
|
||||
if (consume) index++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Matches and optionally consumes the next token in case of a match. Returns whether the token matched. */
|
||||
public boolean match (String text, boolean consume) {
|
||||
if (index >= end) return false;
|
||||
if (tokens.get(index).getText().equals(text)) {
|
||||
if (consume) index++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Matches any of the token types and optionally consumes the next token in case of a match. Returns whether the token
|
||||
* matched. */
|
||||
public boolean match (boolean consume, TokenType... types) {
|
||||
for (TokenType type : types) {
|
||||
if (match(type, consume)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Matches any of the token texts and optionally consumes the next token in case of a match. Returns whether the token
|
||||
* matched. */
|
||||
public boolean match (boolean consume, String... tokenTexts) {
|
||||
for (String text : tokenTexts) {
|
||||
if (match(text, consume)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Returns the {@link Source} this stream wraps. */
|
||||
public String getSource () {
|
||||
if (tokens.size() == 0) return null;
|
||||
return tokens.get(0).getSpan().getSource();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,101 @@
|
||||
|
||||
package org.spiderflow.core.expression.parsing;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
|
||||
/** Enumeration of token types. A token type consists of a representation for error messages, and may optionally specify a literal
|
||||
* to be used by the {@link CharacterStream} to recognize the token. Token types are sorted by their literal length to easy
|
||||
* matching of token types with common prefixes, e.g. "<" and "<=". Token types with longer literals are matched first. */
|
||||
public enum TokenType {
|
||||
// @off
|
||||
TextBlock("a text block"),
|
||||
Period(".", "."),
|
||||
Comma(",", ","),
|
||||
Semicolon(";", ";"),
|
||||
Colon(":", ":"),
|
||||
Plus("+", "+"),
|
||||
Minus("-", "-"),
|
||||
Asterisk("*", "*"),
|
||||
ForwardSlash("/", "/"),
|
||||
PostSlash("\\", "\\"),
|
||||
Percentage("%", "%"),
|
||||
LeftParantheses("(", ")"),
|
||||
RightParantheses(")", ")"),
|
||||
LeftBracket("[", "["),
|
||||
RightBracket("]", "]"),
|
||||
LeftCurly("{", "{"),
|
||||
RightCurly("}"), // special treatment!
|
||||
Less("<", "<"),
|
||||
Greater(">", ">"),
|
||||
LessEqual("<=", "<="),
|
||||
GreaterEqual(">=", ">="),
|
||||
Equal("==", "=="),
|
||||
NotEqual("!=", "!="),
|
||||
Assignment("=", "="),
|
||||
And("&&", "&&"),
|
||||
Or("||", "||"),
|
||||
Xor("^", "^"),
|
||||
Not("!", "!"),
|
||||
Questionmark("?", "?"),
|
||||
DoubleQuote("\"", "\""),
|
||||
SingleQuote("'", "'"),
|
||||
BooleanLiteral("true or false"),
|
||||
DoubleLiteral("a double floating point number"),
|
||||
FloatLiteral("a floating point number"),
|
||||
LongLiteral("a long integer number"),
|
||||
IntegerLiteral("an integer number"),
|
||||
ShortLiteral("a short integer number"),
|
||||
ByteLiteral("a byte integer number"),
|
||||
CharacterLiteral("a character"),
|
||||
StringLiteral("a string"),
|
||||
NullLiteral("null"),
|
||||
Identifier("an identifier");
|
||||
// @on
|
||||
|
||||
private static TokenType[] values;
|
||||
|
||||
static {
|
||||
// Sort the token types by their literal length. The character stream uses this
|
||||
// this order to match tokens with the longest length first.
|
||||
values = TokenType.values();
|
||||
Arrays.sort(values, new Comparator<TokenType>() {
|
||||
@Override
|
||||
public int compare (TokenType o1, TokenType o2) {
|
||||
if (o1.literal == null && o2.literal == null) return 0;
|
||||
if (o1.literal == null && o2.literal != null) return 1;
|
||||
if (o1.literal != null && o2.literal == null) return -1;
|
||||
return o2.literal.length() - o1.literal.length();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private final String literal;
|
||||
private final String error;
|
||||
|
||||
TokenType (String error) {
|
||||
this.literal = null;
|
||||
this.error = error;
|
||||
}
|
||||
|
||||
TokenType (String literal, String error) {
|
||||
this.literal = literal;
|
||||
this.error = error;
|
||||
}
|
||||
|
||||
/** The literal to match, may be null. **/
|
||||
public String getLiteral () {
|
||||
return literal;
|
||||
}
|
||||
|
||||
/** The error string to use when reporting this token type in an error message. **/
|
||||
public String getError () {
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Returns an array of token types, sorted in descending order based on their literal length. This is used by the
|
||||
* {@link CharacterStream} to match token types with the longest literal first. E.g. "<=" will be matched before "<". **/
|
||||
public static TokenType[] getSortedValues () {
|
||||
return values;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
package org.spiderflow.core.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import org.jsoup.Connection;
|
||||
import org.jsoup.Connection.Method;
|
||||
import org.jsoup.Connection.Response;
|
||||
import org.jsoup.Jsoup;
|
||||
|
||||
/**
|
||||
* 请求对象包装类
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
public class HttpRequest {
|
||||
|
||||
private Connection connection = null;
|
||||
|
||||
public static HttpRequest create(){
|
||||
return new HttpRequest();
|
||||
}
|
||||
|
||||
public HttpRequest url(String url){
|
||||
this.connection = Jsoup.connect(url);
|
||||
this.connection.method(Method.GET);
|
||||
this.connection.timeout(60000);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest headers(Map<String,String> headers){
|
||||
this.connection.headers(headers);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest header(String key,String value){
|
||||
this.connection.header(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest header(String key,Object value){
|
||||
if(value != null){
|
||||
this.connection.header(key,value.toString());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest cookies(Map<String,String> cookies){
|
||||
this.connection.cookies(cookies);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest cookie(String name, String value) {
|
||||
if (value != null) {
|
||||
this.connection.cookie(name, value);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest contentType(String contentType){
|
||||
this.connection.header("Content-Type", contentType);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest data(String key,String value){
|
||||
this.connection.data(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest data(String key,Object value){
|
||||
if(value != null){
|
||||
this.connection.data(key, value.toString());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest data(String key,String filename,InputStream is){
|
||||
this.connection.data(key, filename, is);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest data(Object body){
|
||||
if(body != null){
|
||||
this.connection.requestBody(body.toString());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest data(Map<String,String> data){
|
||||
this.connection.data(data);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest method(String method){
|
||||
this.connection.method(Method.valueOf(method));
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest followRedirect(boolean followRedirects){
|
||||
this.connection.followRedirects(followRedirects);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest timeout(int timeout){
|
||||
this.connection.timeout(timeout);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest proxy(String host,int port){
|
||||
this.connection.proxy(host, port);
|
||||
return this;
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public HttpRequest validateTLSCertificates(boolean value){
|
||||
this.connection.validateTLSCertificates(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpResponse execute() throws IOException{
|
||||
this.connection.ignoreContentType(true);
|
||||
this.connection.ignoreHttpErrors(true);
|
||||
this.connection.maxBodySize(0);
|
||||
|
||||
Response response = connection.execute();
|
||||
return new HttpResponse(response);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,104 @@
|
||||
package org.spiderflow.core.io;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import org.jsoup.Connection.Response;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.spiderflow.io.SpiderResponse;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 响应对象包装类
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
public class HttpResponse implements SpiderResponse{
|
||||
|
||||
private Response response;
|
||||
|
||||
private int statusCode;
|
||||
|
||||
private String urlLink;
|
||||
|
||||
private String htmlValue;
|
||||
|
||||
private String titleName;
|
||||
|
||||
private Object jsonValue;
|
||||
|
||||
public HttpResponse(Response response){
|
||||
super();
|
||||
this.response = response;
|
||||
this.statusCode = response.statusCode();
|
||||
this.urlLink = response.url().toExternalForm();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getStatusCode(){
|
||||
return statusCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTitle() {
|
||||
if (titleName == null) {
|
||||
synchronized (this){
|
||||
titleName = Jsoup.parse(getHtml()).title();
|
||||
}
|
||||
}
|
||||
return titleName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHtml(){
|
||||
if(htmlValue == null){
|
||||
synchronized (this){
|
||||
htmlValue = response.body();
|
||||
}
|
||||
}
|
||||
return htmlValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getJson(){
|
||||
if(jsonValue == null){
|
||||
jsonValue = JSON.parse(getHtml());
|
||||
}
|
||||
return jsonValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,String> getCookies(){
|
||||
return response.cookies();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,String> getHeaders(){
|
||||
return response.headers();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getBytes(){
|
||||
return response.bodyAsBytes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType(){
|
||||
return response.contentType();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCharset(String charset) {
|
||||
this.response.charset(charset);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() {
|
||||
return urlLink;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getStream() {
|
||||
return response.bodyStream();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
package org.spiderflow.core.job;
|
||||
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.quartz.JobDataMap;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.context.SpiderContextHolder;
|
||||
import org.spiderflow.core.Spider;
|
||||
import org.spiderflow.core.model.SpiderFlow;
|
||||
import org.spiderflow.core.model.Task;
|
||||
import org.spiderflow.core.service.SpiderFlowService;
|
||||
import org.spiderflow.core.service.TaskService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.quartz.QuartzJobBean;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 爬虫定时执行
|
||||
*
|
||||
* @author Administrator
|
||||
*/
|
||||
@Component
|
||||
public class SpiderJob extends QuartzJobBean {
|
||||
|
||||
@Autowired
|
||||
private Spider spider;
|
||||
|
||||
@Autowired
|
||||
private SpiderFlowService spiderFlowService;
|
||||
|
||||
@Autowired
|
||||
private TaskService taskService;
|
||||
|
||||
private static Map<Integer, SpiderContext> contextMap = new HashMap<>();
|
||||
|
||||
@Value("${spider.job.enable:true}")
|
||||
private boolean spiderJobEnable;
|
||||
|
||||
@Value("${spider.workspace}")
|
||||
private String workspace;
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(SpiderJob.class);
|
||||
|
||||
@Override
|
||||
protected void executeInternal(JobExecutionContext context) {
|
||||
if (!spiderJobEnable) {
|
||||
return;
|
||||
}
|
||||
JobDataMap dataMap = context.getMergedJobDataMap();
|
||||
SpiderFlow spiderFlow = (SpiderFlow) dataMap.get(SpiderJobManager.JOB_PARAM_NAME);
|
||||
if("1".equalsIgnoreCase(spiderFlow.getEnabled())){
|
||||
run(spiderFlow, context.getNextFireTime());
|
||||
}
|
||||
}
|
||||
|
||||
public void run(String id) {
|
||||
run(spiderFlowService.getById(id), null);
|
||||
}
|
||||
|
||||
public void run(SpiderFlow spiderFlow, Date nextExecuteTime) {
|
||||
Task task = new Task();
|
||||
task.setFlowId(spiderFlow.getId());
|
||||
task.setBeginTime(new Date());
|
||||
taskService.save(task);
|
||||
run(spiderFlow,task,nextExecuteTime);
|
||||
}
|
||||
|
||||
public void run(SpiderFlow spiderFlow, Task task,Date nextExecuteTime) {
|
||||
SpiderJobContext context = null;
|
||||
Date now = new Date();
|
||||
try {
|
||||
context = SpiderJobContext.create(this.workspace, spiderFlow.getId(),task.getId(),false);
|
||||
SpiderContextHolder.set(context);
|
||||
contextMap.put(task.getId(), context);
|
||||
logger.info("开始执行任务{}", spiderFlow.getName());
|
||||
spider.run(spiderFlow, context);
|
||||
logger.info("执行任务{}完毕，下次执行时间：{}", spiderFlow.getName(), nextExecuteTime == null ? null : DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
|
||||
} catch (Exception e) {
|
||||
logger.error("执行任务{}出错", spiderFlow.getName(), e);
|
||||
} finally {
|
||||
if (context != null) {
|
||||
context.close();
|
||||
}
|
||||
task.setEndTime(new Date());
|
||||
taskService.saveOrUpdate(task);
|
||||
contextMap.remove(task.getId());
|
||||
SpiderContextHolder.remove();
|
||||
}
|
||||
spiderFlowService.executeCountIncrement(spiderFlow.getId(), now, nextExecuteTime);
|
||||
}
|
||||
|
||||
public static SpiderContext getSpiderContext(Integer taskId) {
|
||||
return contextMap.get(taskId);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
package org.spiderflow.core.job;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.model.SpiderOutput;
|
||||
|
||||
public class SpiderJobContext extends SpiderContext{
|
||||
|
||||
private static final long serialVersionUID = 9099787449108938453L;
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(SpiderJobContext.class);
|
||||
|
||||
private OutputStream outputstream;
|
||||
|
||||
private List<SpiderOutput> outputs = new ArrayList<>();
|
||||
|
||||
private boolean output;
|
||||
|
||||
public SpiderJobContext(OutputStream outputstream,boolean output) {
|
||||
super();
|
||||
this.outputstream = outputstream;
|
||||
this.output = output;
|
||||
}
|
||||
|
||||
public void close(){
|
||||
try {
|
||||
this.outputstream.close();
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addOutput(SpiderOutput output) {
|
||||
if(this.output){
|
||||
synchronized (this.outputs){
|
||||
this.outputs.add(output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SpiderOutput> getOutputs() {
|
||||
return outputs;
|
||||
}
|
||||
|
||||
public OutputStream getOutputstream(){
|
||||
return this.outputstream;
|
||||
}
|
||||
|
||||
public static SpiderJobContext create(String directory,String id,Integer taskId,boolean output){
|
||||
OutputStream os = null;
|
||||
try {
|
||||
File file = new File(new File(directory),id + File.separator + "logs" + File.separator + taskId + ".log");
|
||||
File dirFile = file.getParentFile();
|
||||
if(!dirFile.exists()){
|
||||
dirFile.mkdirs();
|
||||
}
|
||||
os = new FileOutputStream(file, true);
|
||||
} catch (Exception e) {
|
||||
logger.error("创建日志文件出错",e);
|
||||
}
|
||||
SpiderJobContext context = new SpiderJobContext(os, output);
|
||||
context.setFlowId(id);
|
||||
return context;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package org.spiderflow.core.job;
|
||||
|
||||
import org.quartz.CronScheduleBuilder;
|
||||
import org.quartz.CronTrigger;
|
||||
import org.quartz.JobBuilder;
|
||||
import org.quartz.JobDetail;
|
||||
import org.quartz.JobKey;
|
||||
import org.quartz.Scheduler;
|
||||
import org.quartz.SchedulerException;
|
||||
import org.quartz.TriggerBuilder;
|
||||
import org.quartz.TriggerKey;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.spiderflow.core.Spider;
|
||||
import org.spiderflow.core.model.SpiderFlow;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 爬虫定时执行管理
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
public class SpiderJobManager {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(SpiderJobManager.class);
|
||||
|
||||
private final static String JOB_NAME = "SPIDER_TASK_";
|
||||
|
||||
public final static String JOB_PARAM_NAME = "SPIDER_FLOW";
|
||||
|
||||
@Autowired
|
||||
private SpiderJob spiderJob;
|
||||
|
||||
/**
|
||||
* 调度器
|
||||
*/
|
||||
@Autowired
|
||||
private Scheduler scheduler;
|
||||
|
||||
private JobKey getJobKey(String id){
|
||||
return JobKey.jobKey(JOB_NAME + id);
|
||||
}
|
||||
|
||||
private TriggerKey getTriggerKey(String id){
|
||||
return TriggerKey.triggerKey(JOB_NAME + id);
|
||||
}
|
||||
|
||||
/**
|
||||
* 新建定时任务
|
||||
* @param spiderFlow 爬虫流程图
|
||||
* @return boolean true/false
|
||||
*/
|
||||
public Date addJob(SpiderFlow spiderFlow){
|
||||
try {
|
||||
JobDetail job = JobBuilder.newJob(SpiderJob.class).withIdentity(getJobKey(spiderFlow.getId())).build();
|
||||
job.getJobDataMap().put(JOB_PARAM_NAME, spiderFlow);
|
||||
|
||||
CronScheduleBuilder cronScheduleBuilder = CronScheduleBuilder.cronSchedule(spiderFlow.getCron()).withMisfireHandlingInstructionDoNothing();
|
||||
|
||||
CronTrigger trigger = TriggerBuilder.newTrigger().withIdentity(getTriggerKey(spiderFlow.getId())).withSchedule(cronScheduleBuilder).build();
|
||||
|
||||
return scheduler.scheduleJob(job,trigger);
|
||||
} catch (SchedulerException e) {
|
||||
logger.error("创建定时任务出错",e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void run(String id){
|
||||
Spider.executorInstance.submit(()->{
|
||||
spiderJob.run(id);
|
||||
});
|
||||
}
|
||||
|
||||
public boolean remove(String id){
|
||||
try {
|
||||
scheduler.deleteJob(getJobKey(id));
|
||||
return true;
|
||||
} catch (SchedulerException e) {
|
||||
logger.error("删除定时任务失败",e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package org.spiderflow.core.mapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.spiderflow.core.model.DataSource;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
|
||||
public interface DataSourceMapper extends BaseMapper<DataSource>{
|
||||
|
||||
@Select("select id,name from sp_datasource")
|
||||
List<DataSource> selectAll();
|
||||
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package org.spiderflow.core.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.spiderflow.core.model.FlowNotice;
|
||||
|
||||
@Mapper
|
||||
public interface FlowNoticeMapper extends BaseMapper<FlowNotice> {
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package org.spiderflow.core.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.spiderflow.core.model.Function;
|
||||
|
||||
@Mapper
|
||||
public interface FunctionMapper extends BaseMapper<Function> {
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue