Compare commits
54 Commits
Wangyanyix
...
main
Author | SHA1 | Date |
---|---|---|
Lin | f67d3e9130 | 5 days ago |
Lin | 315c9246d2 | 5 days ago |
Lin | 41becde0aa | 5 days ago |
Lin | ca15429378 | 5 days ago |
Lin | 71f8221b0f | 5 days ago |
Lin | 40524b3fec | 5 days ago |
Lin | 21cf165ac6 | 5 days ago |
Lin | ad3796b17e | 5 days ago |
Lin | c5c7925352 | 5 days ago |
Lin | bd2d2f8b38 | 5 days ago |
Lin | 27fa2329f9 | 5 days ago |
Lin | 11af1fff19 | 5 days ago |
Lin | 87594b987b | 5 days ago |
Lin | edb6f27f88 | 5 days ago |
Lin | a29f33b493 | 5 days ago |
Lin | f1052b180b | 5 days ago |
Lin | c648389959 | 5 days ago |
Lin | 8188fb18cb | 5 days ago |
RichardWang | 902661b7aa | 7 days ago |
RichardWang | f909563daa | 1 week ago |
RichardWang | 91beb1a23b | 3 weeks ago |
Lin | 9fb5fd2ad9 | 3 weeks ago |
Lin | cb174b44d0 | 1 month ago |
Lin | 303ea59c2b | 1 month ago |
Lin | 1c9ec9b167 | 1 month ago |
Lin | 3eecc9ed60 | 1 month ago |
Lin | 9e6d9eafc6 | 1 month ago |
Lin | 1c8a00fbbb | 1 month ago |
Lin | d4e76dba28 | 1 month ago |
Lin | 9ce47b9e8a | 1 month ago |
Lin | 663de7af66 | 1 month ago |
Lin | 3f9776c4b1 | 1 month ago |
Lin | 37a072fadc | 2 months ago |
Lin | 280e11d2ce | 2 months ago |
lyy | 2ec4211749 | 2 months ago |
truly | a1686700a0 | 2 months ago |
sgqt | 1feef3faa4 | 2 months ago |
sgqt | 3b56399573 | 2 months ago |
RichardWang | 2007d9b75b | 2 months ago |
RichardWang | 29919f697e | 2 months ago |
Lin | 799de75e51 | 2 months ago |
Lin | 688572c38d | 2 months ago |
truly | f6f270b902 | 2 months ago |
truly | 0d97a4f645 | 2 months ago |
Lin | e997aee47f | 2 months ago |
Lin | e3889aa4d2 | 2 months ago |
Lin | 7203ca831c | 2 months ago |
Lin | a5bbb05e5e | 2 months ago |
Lin | 4666433316 | 2 months ago |
Lin | 639478f03b | 2 months ago |
Lin | e85c342f0f | 2 months ago |
Lin | ac37961a89 | 2 months ago |
Lin | f8b0e72751 | 2 months ago |
puc8kt9je | 89f50fa58d | 2 months ago |
After Width: | Height: | Size: 180 KiB |
After Width: | Height: | Size: 96 KiB |
After Width: | Height: | Size: 67 KiB |
After Width: | Height: | Size: 67 KiB |
After Width: | Height: | Size: 56 KiB |
After Width: | Height: | Size: 36 KiB |
After Width: | Height: | Size: 464 KiB |
After Width: | Height: | Size: 57 KiB |
After Width: | Height: | Size: 126 KiB |
After Width: | Height: | Size: 75 KiB |
After Width: | Height: | Size: 56 KiB |
After Width: | Height: | Size: 72 KiB |
After Width: | Height: | Size: 81 KiB |
After Width: | Height: | Size: 61 KiB |
After Width: | Height: | Size: 76 KiB |
After Width: | Height: | Size: 65 KiB |
After Width: | Height: | Size: 248 KiB |
After Width: | Height: | Size: 64 KiB |
After Width: | Height: | Size: 88 KiB |
After Width: | Height: | Size: 7.8 MiB |
After Width: | Height: | Size: 62 KiB |
After Width: | Height: | Size: 5.5 MiB |
After Width: | Height: | Size: 22 KiB |
After Width: | Height: | Size: 42 KiB |
After Width: | Height: | Size: 177 KiB |
After Width: | Height: | Size: 39 KiB |
After Width: | Height: | Size: 144 KiB |
After Width: | Height: | Size: 186 KiB |
After Width: | Height: | Size: 127 KiB |
After Width: | Height: | Size: 34 KiB |
After Width: | Height: | Size: 142 KiB |
After Width: | Height: | Size: 263 KiB |
@ -0,0 +1,6 @@
|
||||
projectKey=clawer
|
||||
serverUrl=http://localhost:9000
|
||||
serverVersion=7.8.0.26217
|
||||
dashboardUrl=http://localhost:9000/dashboard?id=clawer
|
||||
ceTaskId=AZMv5JVBnAUFl5pPDUTm
|
||||
ceTaskUrl=http://localhost:9000/api/ce/task?id=AZMv5JVBnAUFl5pPDUTm
|
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Suysker
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
@ -0,0 +1,90 @@
|
||||
import pandas as pd
|
||||
import mysql.connector
|
||||
from mysql.connector import Error
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# 数据库连接配置
|
||||
db_config = {
|
||||
'host': '152.136.166.253', # 修改这里,去掉端口号
|
||||
'port': 8989, # 单独指定端口号
|
||||
'database': 'fly_ticket',
|
||||
'user': 'root',
|
||||
'password': 'Cauc@2024'
|
||||
}
|
||||
|
||||
def import_csv_to_db(file_path, cursor):
|
||||
df = pd.read_csv(file_path)
|
||||
for index, row in df.iterrows():
|
||||
sql = """INSERT INTO flight (f_n, f_s_p, f_a_p, f_s_a, f_a_a, f_s_t, f_a_t, f_Date, f_Delay, f_p, f_food, f_wide, f_depcode, f_dstcode)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
f_s_p = VALUES(f_s_p),
|
||||
f_a_p = VALUES(f_a_p),
|
||||
f_s_a = VALUES(f_s_a),
|
||||
f_a_a = VALUES(f_a_a),
|
||||
f_s_t = VALUES(f_s_t),
|
||||
f_a_t = VALUES(f_a_t),
|
||||
f_Delay = VALUES(f_Delay),
|
||||
f_p = VALUES(f_p),
|
||||
f_food = VALUES(f_food),
|
||||
f_wide = VALUES(f_wide),
|
||||
f_depcode = VALUES(f_depcode),
|
||||
f_dstcode = VALUES(f_dstcode);"""
|
||||
|
||||
values = (
|
||||
row['航班号'],
|
||||
row['出发城市'],
|
||||
row['到达城市'],
|
||||
row['出发机场'],
|
||||
row['到达机场'],
|
||||
row['出发时间'],
|
||||
row['到达时间'],
|
||||
row['出发日期'],
|
||||
row['出发延误时间'],
|
||||
row['economy_origin'],
|
||||
row['经济舱餐食信息'],
|
||||
row['经济舱座椅间距'],
|
||||
row['出发机场三字码'],
|
||||
row['到达机场三字码']
|
||||
)
|
||||
|
||||
cursor.execute(sql, values)
|
||||
|
||||
try:
|
||||
# 连接到数据库
|
||||
conn = mysql.connector.connect(**db_config)
|
||||
|
||||
if conn.is_connected():
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 设置日期范围
|
||||
start_date = datetime(2024, 11, 12)
|
||||
end_date = datetime(2024, 11, 20)
|
||||
current_date = start_date
|
||||
|
||||
while current_date <= end_date:
|
||||
folder_name = current_date.strftime("%Y-%m-%d")
|
||||
folder_path = os.path.join("D:\college\SE2\Ctrip-Crawler-main\Ctrip-Crawler-withComfortInfo", folder_name, "2024-11-12")
|
||||
|
||||
if os.path.exists(folder_path):
|
||||
for file_name in os.listdir(folder_path):
|
||||
if file_name.endswith('.csv'):
|
||||
file_path = os.path.join(folder_path, file_name)
|
||||
import_csv_to_db(file_path, cursor)
|
||||
print(f"已导入文件: {file_path}")
|
||||
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
# 提交更改
|
||||
conn.commit()
|
||||
print("所有数据成功插入到数据库")
|
||||
|
||||
except Error as e:
|
||||
print(f"连接数据库时出错: {e}")
|
||||
|
||||
finally:
|
||||
if 'conn' in locals() and conn.is_connected():
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print("数据库连接已关闭")
|
@ -0,0 +1,412 @@
|
||||
import io
|
||||
import os
|
||||
import gzip
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import requests
|
||||
import threading
|
||||
import pandas as pd
|
||||
from seleniumwire import webdriver
|
||||
from datetime import datetime as dt,timedelta
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.common.exceptions import TimeoutException,StaleElementReferenceException,ElementNotInteractableException,ElementClickInterceptedException # 加载异常
|
||||
|
||||
|
||||
def getcitycode():
|
||||
cityname,code=[],[]
|
||||
#采用携程的api接口
|
||||
city_url='https://flights.ctrip.com/online/api/poi/get?v='+str(random.random())
|
||||
headers={
|
||||
'dnt':'1',
|
||||
'referer':'https://verify.ctrip.com/',
|
||||
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'
|
||||
}
|
||||
r=requests.get(city_url,headers=headers)
|
||||
citys=json.loads(r.text).get('data')
|
||||
for city in citys:
|
||||
if city =='热门':
|
||||
continue
|
||||
for key in city:
|
||||
try:
|
||||
for k in citys[city][key]:
|
||||
cityname.append(k['display'])
|
||||
code.append(k['data'])
|
||||
except:
|
||||
continue
|
||||
citycode=dict(zip(cityname,code))
|
||||
|
||||
return cityname,citycode
|
||||
|
||||
|
||||
|
||||
class FLIGHT(object):
|
||||
def __init__(self):
|
||||
self.url = 'https://flights.ctrip.com/online/list/oneway' #携程机票查询页面
|
||||
self.chromeDriverPath = 'C:/Program Files/Google/Chrome/Application/chromedriver' #chromedriver位置
|
||||
self.options = webdriver.ChromeOptions() # 创建一个配置对象
|
||||
#self.options.add_argument('--incognito') # 隐身模式(无痕模式)
|
||||
#self.options.add_argument('User-Agent=%s'%UserAgent().random) # 替换User-Agent
|
||||
self.options.add_argument("--disable-blink-features")
|
||||
self.options.add_argument("--disable-blink-features=AutomationControlled")
|
||||
self.options.add_experimental_option("excludeSwitches", ['enable-automation'])# 不显示正在受自动化软件控制
|
||||
self.driver = webdriver.Chrome(executable_path=self.chromeDriverPath,chrome_options=self.options)
|
||||
self.driver.maximize_window()
|
||||
self.err=0#错误重试次数
|
||||
|
||||
|
||||
def getpage(self):
|
||||
##############获取地区码
|
||||
self.startcode=self.citycode[self.city[0]][-3:]
|
||||
self.endcode=self.citycode[self.city[1]][-3:]
|
||||
|
||||
##############生成访问链接
|
||||
flights_url=self.url+'-'+self.startcode+'-'+self.endcode+'?&depdate='+self.date
|
||||
print(flights_url)
|
||||
##############设置加载超时阈值
|
||||
self.driver.set_page_load_timeout(300)
|
||||
try:
|
||||
self.driver.get(flights_url)
|
||||
except:
|
||||
print('页面连接失败')
|
||||
self.driver.close()
|
||||
self.getpage()
|
||||
else:
|
||||
try:
|
||||
##############判断是否存在验证码
|
||||
self.driver.find_element(By.CLASS_NAME,"basic-alert.alert-giftinfo")
|
||||
print('等待2小时后重试')
|
||||
time.sleep(7200)
|
||||
self.getpage()
|
||||
except:
|
||||
##############不存在验证码,执行下一步
|
||||
self.remove_btn()
|
||||
|
||||
def remove_btn(self):
|
||||
try:
|
||||
js_remove="$('.notice-box').remove();"
|
||||
self.driver.execute_script(js_remove)
|
||||
except Exception as e:
|
||||
print('防疫移除失败',e)
|
||||
else:
|
||||
self.changecity()
|
||||
|
||||
|
||||
|
||||
def changecity(self):
|
||||
try:
|
||||
#获取出发地与目的地元素位置
|
||||
its=self.driver.find_elements(By.CLASS_NAME,'form-input-v3')
|
||||
|
||||
#若出发地与目标值不符,则更改出发地
|
||||
while self.city[0] not in its[0].get_attribute('value'):
|
||||
its[0].click()
|
||||
time.sleep(0.5)
|
||||
its[0].send_keys(Keys.CONTROL + 'a')
|
||||
time.sleep(0.5)
|
||||
its[0].send_keys(self.city[0])
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
#若目的地与目标值不符,则更改目的地
|
||||
while self.city[1] not in its[1].get_attribute('value'):
|
||||
its[1].click()
|
||||
time.sleep(0.5)
|
||||
its[1].send_keys(Keys.CONTROL + 'a')
|
||||
time.sleep(0.5)
|
||||
its[1].send_keys(self.city[1])
|
||||
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
#通过低价提醒按钮实现enter键换页
|
||||
self.driver.implicitly_wait(5) # seconds
|
||||
self.driver.find_elements(By.CLASS_NAME,'low-price-remind')[0].click()
|
||||
except IndexError as e:
|
||||
print('\n更换城市错误 找不到元素',e)
|
||||
#以防万一
|
||||
its[1].send_keys(Keys.ENTER)
|
||||
|
||||
print('\n更换城市成功',self.city[0]+'-'+self.city[1])
|
||||
except (ElementNotInteractableException,StaleElementReferenceException,ElementClickInterceptedException,ElementClickInterceptedException) as e:
|
||||
print('\n更换城市错误 元素错误',e)
|
||||
self.err+=1
|
||||
if self.err<=5:
|
||||
self.click_btn()
|
||||
else:
|
||||
self.err=0
|
||||
del self.driver.requests
|
||||
self.getpage()
|
||||
except Exception as e:
|
||||
print('\n更换城市错误',e)
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#从头开始重新执行程序
|
||||
self.getpage()
|
||||
else:
|
||||
#若无错误,执行下一步
|
||||
self.err=0
|
||||
self.getdata()
|
||||
|
||||
|
||||
|
||||
def getdata(self):
|
||||
try:
|
||||
#等待响应加载完成
|
||||
self.predata = self.driver.wait_for_request('/international/search/api/search/batchSearch?.*', timeout=60)
|
||||
|
||||
rb=dict(json.loads(self.predata.body).get('flightSegments')[0])
|
||||
|
||||
except TimeoutException as e:
|
||||
print('\获取数据错误',e)
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#从头开始重新执行程序
|
||||
self.getpage()
|
||||
else:
|
||||
#检查数据获取正确性
|
||||
if rb['departureCityName'] == self.city[0] and rb['arrivalCityName'] == self.city[1]:
|
||||
print('城市获取正确')
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#若无错误,执行下一步
|
||||
self.decode_data()
|
||||
else:
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#重新更换城市
|
||||
self.changecity()
|
||||
|
||||
|
||||
|
||||
def decode_data(self):
|
||||
try:
|
||||
buf = io.BytesIO(self.predata.response.body)
|
||||
gf = gzip.GzipFile(fileobj = buf)
|
||||
self.dedata = gf.read().decode('UTF-8')
|
||||
self.dedata=json.loads(self.dedata)
|
||||
except:
|
||||
print('重新获取数据')
|
||||
self.getpage()
|
||||
else:
|
||||
#若无错误,执行下一步
|
||||
self.check_data()
|
||||
|
||||
|
||||
|
||||
def check_data(self):
|
||||
try:
|
||||
self.flightItineraryList=self.dedata['data']['flightItineraryList']
|
||||
#倒序遍历,删除转机航班
|
||||
for i in range(len(self.flightItineraryList)-1, -1, -1):
|
||||
if self.flightItineraryList[i]['flightSegments'][0]['transferCount'] !=0:
|
||||
self.flightItineraryList.pop(i)
|
||||
if len(self.flightItineraryList):
|
||||
#存在直航航班,执行下一步
|
||||
self.muti_process()
|
||||
else:
|
||||
print('不存在直航航班')
|
||||
return 0
|
||||
except:
|
||||
print('不存在直航航班')
|
||||
return 0
|
||||
|
||||
|
||||
def muti_process(self):
|
||||
processes = []
|
||||
|
||||
self.flights = pd.DataFrame()
|
||||
self.prices = pd.DataFrame()
|
||||
#处理航班信息
|
||||
processes.append(threading.Thread(target=self.proc_flightSegments))
|
||||
#处理票价信息
|
||||
processes.append(threading.Thread(target=self.proc_priceList))
|
||||
|
||||
for pro in processes:
|
||||
pro.start()
|
||||
for pro in processes:
|
||||
pro.join()
|
||||
|
||||
#若无错误,执行下一步
|
||||
self.mergedata()
|
||||
|
||||
def proc_flightSegments(self):
|
||||
for flightlist in self.flightItineraryList:
|
||||
flightlist=flightlist['flightSegments'][0]['flightList']
|
||||
flightUnitList=dict(flightlist[0])
|
||||
|
||||
|
||||
departureday=flightUnitList['departureDateTime'].split(' ')[0]
|
||||
departuretime=flightUnitList['departureDateTime'].split(' ')[1]
|
||||
|
||||
arrivalday=flightUnitList['arrivalDateTime'].split(' ')[0]
|
||||
arrivaltime=flightUnitList['arrivalDateTime'].split(' ')[1]
|
||||
|
||||
#删除一些不重要的信息
|
||||
dellist=['sequenceNo', 'marketAirlineCode',
|
||||
'departureProvinceId','departureCityId','departureCityCode','departureAirportShortName','departureTerminal',
|
||||
'arrivalProvinceId','arrivalCityId','arrivalCityCode','arrivalAirportShortName','arrivalTerminal',
|
||||
'transferDuration','stopList','leakedVisaTagSwitch','trafficType','highLightPlaneNo','mealType',
|
||||
'operateAirlineCode','arrivalDateTime','departureDateTime','operateFlightNo','operateAirlineName']
|
||||
for value in dellist:
|
||||
try:
|
||||
flightUnitList.pop(value)
|
||||
except:
|
||||
continue
|
||||
|
||||
#更新日期格式
|
||||
flightUnitList.update({'departureday': departureday, 'departuretime': departuretime,
|
||||
'arrivalday': arrivalday, 'arrivaltime': arrivaltime})
|
||||
|
||||
|
||||
self.flights=pd.concat([self.flights,pd.DataFrame(flightUnitList,index=[0])],ignore_index=True)
|
||||
|
||||
|
||||
|
||||
def proc_priceList(self):
|
||||
for flightlist in self.flightItineraryList:
|
||||
flightNo=flightlist['itineraryId'].split('_')[0]
|
||||
priceList=flightlist['priceList']
|
||||
|
||||
#经济舱,经济舱折扣
|
||||
economy,economy_discount=[],[]
|
||||
#商务舱,商务舱折扣
|
||||
bussiness,bussiness_discount=[],[]
|
||||
|
||||
for price in priceList:
|
||||
adultPrice=price['adultPrice']
|
||||
cabin=price['cabin']
|
||||
priceUnitList=dict(price['priceUnitList'][0]['flightSeatList'][0])
|
||||
discountRate=priceUnitList['discountRate']
|
||||
#经济舱
|
||||
if cabin=='Y':
|
||||
economy.append(adultPrice)
|
||||
economy_discount.append(discountRate)
|
||||
#商务舱
|
||||
elif cabin=='C':
|
||||
bussiness.append(adultPrice)
|
||||
bussiness_discount.append(discountRate)
|
||||
|
||||
if economy !=[]:
|
||||
try:
|
||||
economy_origin=economy[economy_discount.index(1)]
|
||||
except:
|
||||
economy_origin=int(max(economy)/max(economy_discount))
|
||||
|
||||
if min(economy_discount) !=1:
|
||||
economy_low=min(economy)
|
||||
economy_cut=min(economy_discount)
|
||||
else:
|
||||
economy_low=''
|
||||
economy_cut=''
|
||||
|
||||
else:
|
||||
economy_origin=''
|
||||
economy_low=''
|
||||
economy_cut=''
|
||||
|
||||
|
||||
if bussiness !=[]:
|
||||
try:
|
||||
bussiness_origin=bussiness[bussiness_discount.index(1)]
|
||||
except:
|
||||
bussiness_origin=int(max(bussiness)/max(bussiness_discount))
|
||||
|
||||
if min(bussiness_discount) !=1:
|
||||
bussiness_low=min(bussiness)
|
||||
bussiness_cut=min(bussiness_discount)
|
||||
else:
|
||||
bussiness_low=''
|
||||
bussiness_cut=''
|
||||
|
||||
else:
|
||||
bussiness_origin=''
|
||||
bussiness_low=''
|
||||
bussiness_cut=''
|
||||
|
||||
price_info={'flightNo':flightNo,
|
||||
'economy_origin':economy_origin,'economy_low':economy_low,'economy_cut':economy_cut,
|
||||
'bussiness_origin':bussiness_origin,'bussiness_low':bussiness_low,'bussiness_cut':bussiness_cut}
|
||||
|
||||
#self.prices=self.prices.append(price_info,ignore_index=True)
|
||||
self.prices=pd.concat([self.prices,pd.DataFrame(price_info,index=[0])],ignore_index=True)
|
||||
|
||||
|
||||
|
||||
def mergedata(self):
|
||||
try:
|
||||
self.df = self.flights.merge(self.prices,on=['flightNo'])
|
||||
|
||||
self.df['数据获取日期']=dt.now().strftime('%Y-%m-%d')
|
||||
|
||||
#对pandas的columns进行重命名
|
||||
order=['数据获取日期','航班号','航空公司',
|
||||
'出发日期','出发时间','到达日期','到达时间','飞行时长','出发国家','出发城市','出发机场','出发机场三字码',
|
||||
'到达国家','到达城市','到达机场','到达机场三字码','飞机型号','飞机尺寸','飞机型号三字码',
|
||||
'经济舱原价','经济舱最低价','经济舱折扣','商务舱原价','商务舱最低价','商务舱折扣',
|
||||
'到达准点率','停留次数']
|
||||
|
||||
origin=['数据获取日期','flightNo','marketAirlineName',
|
||||
'departureday','departuretime','arrivalday','arrivaltime','duration',
|
||||
'departureCountryName','departureCityName','departureAirportName','departureAirportCode',
|
||||
'arrivalCountryName','arrivalCityName','arrivalAirportName','arrivalAirportCode',
|
||||
'aircraftName','aircraftSize','aircraftCode',
|
||||
'economy_origin','economy_low','economy_cut',
|
||||
'bussiness_origin','bussiness_low','bussiness_cut',
|
||||
'arrivalPunctuality','stopCount']
|
||||
|
||||
columns=dict(zip(origin,order))
|
||||
|
||||
self.df=self.df.rename(columns=columns)
|
||||
|
||||
self.df = self.df[order]
|
||||
|
||||
|
||||
if not os.path.exists(self.date):
|
||||
os.makedirs(self.date)
|
||||
|
||||
filename=os.getcwd()+'\\'+self.date+'\\'+self.date+'-'+self.city[0]+'-'+self.city[1]+'.csv'
|
||||
|
||||
self.df.to_csv(filename,encoding='GB18030',index=False)
|
||||
|
||||
print('\n数据爬取完成',filename)
|
||||
except Exception as e:
|
||||
print('合并数据失败',e)
|
||||
|
||||
|
||||
def demain(self,citys,citycode):
|
||||
self.citycode=citycode
|
||||
#设置出发日期
|
||||
self.date=dt.now()+timedelta(days=7)
|
||||
self.date=self.date.strftime('%Y-%m-%d')
|
||||
|
||||
for city in citys:
|
||||
self.city=city
|
||||
|
||||
if citys.index(city)==0:
|
||||
#第一次运行
|
||||
self.getpage()
|
||||
else:
|
||||
#后续运行只需更换出发与目的地
|
||||
self.changecity()
|
||||
|
||||
#运行结束退出
|
||||
self.driver.quit()
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
citys=[]
|
||||
cityname,citycode=getcitycode()
|
||||
city=['上海','广州','深圳','北京']
|
||||
ytic=list(reversed(city))
|
||||
for m in city:
|
||||
for n in ytic:
|
||||
if m==n:
|
||||
continue
|
||||
else:
|
||||
citys.append([m,n])
|
||||
fly = FLIGHT()
|
||||
fly.demain(citys,citycode)
|
||||
print('\n程序运行完成!!!!')
|
||||
|
@ -0,0 +1,397 @@
|
||||
import io
|
||||
import os
|
||||
import gzip
|
||||
import time
|
||||
import json
|
||||
import threading
|
||||
import pandas as pd
|
||||
from seleniumwire import webdriver
|
||||
from datetime import datetime as dt,timedelta
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.common.exceptions import TimeoutException,StaleElementReferenceException,ElementNotInteractableException,ElementClickInterceptedException # 加载异常
|
||||
|
||||
|
||||
|
||||
class FLIGHT(object):
|
||||
def __init__(self):
|
||||
self.chromeDriverPath = 'C:/Program Files/Google/Chrome/Application/chromedriver' #chromedriver位置
|
||||
self.options = webdriver.ChromeOptions() # 创建一个配置对象
|
||||
self.options.add_argument('--incognito') # 隐身模式(无痕模式)
|
||||
self.options.add_argument("--disable-blink-features")
|
||||
self.options.add_argument("--disable-blink-features=AutomationControlled")
|
||||
self.options.add_experimental_option("excludeSwitches", ['enable-automation'])# 不显示正在受自动化软件控制
|
||||
self.driver = webdriver.Chrome(executable_path=self.chromeDriverPath,chrome_options=self.options)
|
||||
self.driver.set_page_load_timeout(300)#设置加载超时阈值
|
||||
self.driver.maximize_window()
|
||||
self.err=0#错误重试次数
|
||||
#前往首页
|
||||
self.driver.get('https://flights.ctrip.com/online/channel/domestic')
|
||||
|
||||
|
||||
|
||||
def getpage(self):
|
||||
try:
|
||||
self.driver.find_element(By.CLASS_NAME,'pc_home-jipiao').click()#点击飞机图标,返回主界面
|
||||
self.driver.implicitly_wait(5) # seconds
|
||||
self.driver.find_elements(By.CLASS_NAME,'radio-label')[0].click()#单程
|
||||
|
||||
while self.driver.find_elements(By.CSS_SELECTOR,"[aria-label=请选择日期]")[0].get_attribute("value") != self.date:
|
||||
|
||||
self.driver.find_element(By.CLASS_NAME,'modifyDate.depart-date').click()#点击日期选择
|
||||
|
||||
for m in self.driver.find_elements(By.CLASS_NAME,'date-picker.date-picker-block'):
|
||||
|
||||
if int(m.find_element(By.CLASS_NAME,'month').text[:-1]) != int(self.date[5:7]):
|
||||
continue
|
||||
|
||||
for d in m.find_elements(By.CLASS_NAME,'date-d'):
|
||||
if int(d.text) == int(self.date[-2:]):
|
||||
d.click()
|
||||
break
|
||||
|
||||
self.driver.find_element(By.CLASS_NAME,'search-btn').click()#搜索
|
||||
|
||||
except:
|
||||
print('页面连接失败')
|
||||
self.driver.close()
|
||||
self.getpage()
|
||||
else:
|
||||
try:
|
||||
##############判断是否存在验证码
|
||||
self.driver.find_element(By.ID,"verification-code")
|
||||
print('等待2小时后重试')
|
||||
time.sleep(7200)
|
||||
self.getpage()
|
||||
except:
|
||||
##############不存在验证码,执行下一步
|
||||
self.changecity()
|
||||
|
||||
def remove_btn(self):
|
||||
try:
|
||||
js_remove="$('.notice-box').remove();"
|
||||
self.driver.execute_script(js_remove)
|
||||
except Exception as e:
|
||||
print('防疫移除失败',e)
|
||||
|
||||
|
||||
def changecity(self):
|
||||
|
||||
#移除防疫提醒
|
||||
self.remove_btn()
|
||||
|
||||
try:
|
||||
#获取出发地与目的地元素位置
|
||||
its=self.driver.find_elements(By.CLASS_NAME,'form-input-v3')
|
||||
|
||||
#若出发地与目标值不符,则更改出发地
|
||||
while self.city[0] not in its[0].get_attribute('value'):
|
||||
its[0].click()
|
||||
time.sleep(0.5)
|
||||
its[0].send_keys(Keys.CONTROL + 'a')
|
||||
time.sleep(0.5)
|
||||
its[0].send_keys(self.city[0])
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
#若目的地与目标值不符,则更改目的地
|
||||
while self.city[1] not in its[1].get_attribute('value'):
|
||||
its[1].click()
|
||||
time.sleep(0.5)
|
||||
its[1].send_keys(Keys.CONTROL + 'a')
|
||||
time.sleep(0.5)
|
||||
its[1].send_keys(self.city[1])
|
||||
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
#通过低价提醒按钮实现enter键换页
|
||||
self.driver.implicitly_wait(5) # seconds
|
||||
self.driver.find_elements(By.CLASS_NAME,'low-price-remind')[0].click()
|
||||
except IndexError as e:
|
||||
print('\n更换城市错误 找不到元素',e)
|
||||
#以防万一
|
||||
its[1].send_keys(Keys.ENTER)
|
||||
|
||||
print('\n更换城市成功',self.city[0]+'-'+self.city[1])
|
||||
#捕获错误
|
||||
except (IndexError,ElementNotInteractableException,StaleElementReferenceException,ElementClickInterceptedException,ElementClickInterceptedException) as e:
|
||||
print('\n更换城市错误 元素错误',e)
|
||||
self.err+=1
|
||||
if self.err<=5:
|
||||
self.changecity()
|
||||
else:
|
||||
self.err=0
|
||||
del self.driver.requests
|
||||
self.getpage()
|
||||
except Exception as e:
|
||||
print('\n更换城市错误',e)
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#从头开始重新执行程序
|
||||
self.getpage()
|
||||
else:
|
||||
#若无错误,执行下一步
|
||||
self.err=0
|
||||
self.getdata()
|
||||
|
||||
|
||||
|
||||
def getdata(self):
|
||||
try:
|
||||
#等待响应加载完成
|
||||
self.predata = self.driver.wait_for_request('/international/search/api/search/batchSearch?.*', timeout=30)
|
||||
|
||||
rb=dict(json.loads(self.predata.body).get('flightSegments')[0])
|
||||
|
||||
except TimeoutException as e:
|
||||
print('\获取数据错误',e)
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#从头开始重新执行程序
|
||||
self.getpage()
|
||||
else:
|
||||
#检查数据获取正确性
|
||||
if rb['departureCityName'] == self.city[0] and rb['arrivalCityName'] == self.city[1]:
|
||||
print('城市获取正确')
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#若无错误,执行下一步
|
||||
self.decode_data()
|
||||
else:
|
||||
#删除本次请求
|
||||
del self.driver.requests
|
||||
#重新更换城市
|
||||
self.changecity()
|
||||
|
||||
|
||||
|
||||
def decode_data(self):
|
||||
try:
|
||||
buf = io.BytesIO(self.predata.response.body)
|
||||
gf = gzip.GzipFile(fileobj = buf)
|
||||
self.dedata = gf.read().decode('UTF-8')
|
||||
self.dedata=json.loads(self.dedata)
|
||||
except:
|
||||
print('重新获取数据')
|
||||
self.getpage()
|
||||
else:
|
||||
#若无错误,执行下一步
|
||||
self.check_data()
|
||||
|
||||
|
||||
|
||||
def check_data(self):
|
||||
try:
|
||||
self.flightItineraryList=self.dedata['data']['flightItineraryList']
|
||||
#倒序遍历,删除转机航班
|
||||
for i in range(len(self.flightItineraryList)-1, -1, -1):
|
||||
if self.flightItineraryList[i]['flightSegments'][0]['transferCount'] !=0:
|
||||
self.flightItineraryList.pop(i)
|
||||
if len(self.flightItineraryList):
|
||||
#存在直航航班,执行下一步
|
||||
self.muti_process()
|
||||
else:
|
||||
print('不存在直航航班')
|
||||
return 0
|
||||
except:
|
||||
print('不存在直航航班')
|
||||
return 0
|
||||
|
||||
|
||||
def muti_process(self):
|
||||
processes = []
|
||||
|
||||
self.flights = pd.DataFrame()
|
||||
self.prices = pd.DataFrame()
|
||||
#处理航班信息
|
||||
processes.append(threading.Thread(target=self.proc_flightSegments))
|
||||
#处理票价信息
|
||||
processes.append(threading.Thread(target=self.proc_priceList))
|
||||
|
||||
for pro in processes:
|
||||
pro.start()
|
||||
for pro in processes:
|
||||
pro.join()
|
||||
|
||||
#若无错误,执行下一步
|
||||
self.mergedata()
|
||||
|
||||
def proc_flightSegments(self):
|
||||
for flightlist in self.flightItineraryList:
|
||||
flightlist=flightlist['flightSegments'][0]['flightList']
|
||||
flightUnitList=dict(flightlist[0])
|
||||
|
||||
|
||||
departureday=flightUnitList['departureDateTime'].split(' ')[0]
|
||||
departuretime=flightUnitList['departureDateTime'].split(' ')[1]
|
||||
|
||||
arrivalday=flightUnitList['arrivalDateTime'].split(' ')[0]
|
||||
arrivaltime=flightUnitList['arrivalDateTime'].split(' ')[1]
|
||||
|
||||
#删除一些不重要的信息
|
||||
dellist=['sequenceNo', 'marketAirlineCode',
|
||||
'departureProvinceId','departureCityId','departureCityCode','departureAirportShortName','departureTerminal',
|
||||
'arrivalProvinceId','arrivalCityId','arrivalCityCode','arrivalAirportShortName','arrivalTerminal',
|
||||
'transferDuration','stopList','leakedVisaTagSwitch','trafficType','highLightPlaneNo','mealType',
|
||||
'operateAirlineCode','arrivalDateTime','departureDateTime','operateFlightNo','operateAirlineName']
|
||||
for value in dellist:
|
||||
try:
|
||||
flightUnitList.pop(value)
|
||||
except:
|
||||
continue
|
||||
|
||||
#更新日期格式
|
||||
flightUnitList.update({'departureday': departureday, 'departuretime': departuretime,
|
||||
'arrivalday': arrivalday, 'arrivaltime': arrivaltime})
|
||||
|
||||
|
||||
self.flights=pd.concat([self.flights,pd.DataFrame(flightUnitList,index=[0])],ignore_index=True)
|
||||
|
||||
|
||||
|
||||
def proc_priceList(self):
|
||||
for flightlist in self.flightItineraryList:
|
||||
flightNo=flightlist['itineraryId'].split('_')[0]
|
||||
priceList=flightlist['priceList']
|
||||
|
||||
#经济舱,经济舱折扣
|
||||
economy,economy_discount=[],[]
|
||||
#商务舱,商务舱折扣
|
||||
bussiness,bussiness_discount=[],[]
|
||||
|
||||
for price in priceList:
|
||||
adultPrice=price['adultPrice']
|
||||
cabin=price['cabin']
|
||||
priceUnitList=dict(price['priceUnitList'][0]['flightSeatList'][0])
|
||||
discountRate=priceUnitList['discountRate']
|
||||
#经济舱
|
||||
if cabin=='Y':
|
||||
economy.append(adultPrice)
|
||||
economy_discount.append(discountRate)
|
||||
#商务舱
|
||||
elif cabin=='C':
|
||||
bussiness.append(adultPrice)
|
||||
bussiness_discount.append(discountRate)
|
||||
|
||||
if economy !=[]:
|
||||
try:
|
||||
economy_origin=economy[economy_discount.index(1)]
|
||||
except:
|
||||
economy_origin=int(max(economy)/max(economy_discount))
|
||||
|
||||
if min(economy_discount) !=1:
|
||||
economy_low=min(economy)
|
||||
economy_cut=min(economy_discount)
|
||||
else:
|
||||
economy_low=''
|
||||
economy_cut=''
|
||||
|
||||
else:
|
||||
economy_origin=''
|
||||
economy_low=''
|
||||
economy_cut=''
|
||||
|
||||
|
||||
if bussiness !=[]:
|
||||
try:
|
||||
bussiness_origin=bussiness[bussiness_discount.index(1)]
|
||||
except:
|
||||
bussiness_origin=int(max(bussiness)/max(bussiness_discount))
|
||||
|
||||
if min(bussiness_discount) !=1:
|
||||
bussiness_low=min(bussiness)
|
||||
bussiness_cut=min(bussiness_discount)
|
||||
else:
|
||||
bussiness_low=''
|
||||
bussiness_cut=''
|
||||
|
||||
else:
|
||||
bussiness_origin=''
|
||||
bussiness_low=''
|
||||
bussiness_cut=''
|
||||
|
||||
price_info={'flightNo':flightNo,
|
||||
'economy_origin':economy_origin,'economy_low':economy_low,'economy_cut':economy_cut,
|
||||
'bussiness_origin':bussiness_origin,'bussiness_low':bussiness_low,'bussiness_cut':bussiness_cut}
|
||||
|
||||
#self.prices=self.prices.append(price_info,ignore_index=True)
|
||||
self.prices=pd.concat([self.prices,pd.DataFrame(price_info,index=[0])],ignore_index=True)
|
||||
|
||||
|
||||
|
||||
def mergedata(self):
|
||||
try:
|
||||
self.df = self.flights.merge(self.prices,on=['flightNo'])
|
||||
|
||||
self.df['数据获取日期']=dt.now().strftime('%Y-%m-%d')
|
||||
|
||||
#对pandas的columns进行重命名
|
||||
order=['数据获取日期','航班号','航空公司',
|
||||
'出发日期','出发时间','到达日期','到达时间','飞行时长','出发国家','出发城市','出发机场','出发机场三字码',
|
||||
'到达国家','到达城市','到达机场','到达机场三字码','飞机型号','飞机尺寸','飞机型号三字码',
|
||||
'经济舱原价','经济舱最低价','经济舱折扣','商务舱原价','商务舱最低价','商务舱折扣',
|
||||
'到达准点率','停留次数']
|
||||
|
||||
origin=['数据获取日期','flightNo','marketAirlineName',
|
||||
'departureday','departuretime','arrivalday','arrivaltime','duration',
|
||||
'departureCountryName','departureCityName','departureAirportName','departureAirportCode',
|
||||
'arrivalCountryName','arrivalCityName','arrivalAirportName','arrivalAirportCode',
|
||||
'aircraftName','aircraftSize','aircraftCode',
|
||||
'economy_origin','economy_low','economy_cut',
|
||||
'bussiness_origin','bussiness_low','bussiness_cut',
|
||||
'arrivalPunctuality','stopCount']
|
||||
|
||||
columns=dict(zip(origin,order))
|
||||
|
||||
self.df=self.df.rename(columns=columns)
|
||||
|
||||
self.df = self.df[order]
|
||||
|
||||
|
||||
if not os.path.exists(self.date):
|
||||
os.makedirs(self.date)
|
||||
|
||||
filename=os.getcwd()+'\\'+self.date+'\\'+self.date+'-'+self.city[0]+'-'+self.city[1]+'.csv'
|
||||
|
||||
self.df.to_csv(filename,encoding='GB18030',index=False)
|
||||
|
||||
print('\n数据爬取完成',filename)
|
||||
except Exception as e:
|
||||
print('合并数据失败',e)
|
||||
|
||||
|
||||
def demain(self,citys):
|
||||
#设置出发日期
|
||||
self.date=dt.now()+timedelta(days=1)
|
||||
self.date=self.date.strftime('%Y-%m-%d')
|
||||
|
||||
for city in citys:
|
||||
self.city=city
|
||||
|
||||
if citys.index(city)==0:
|
||||
#第一次运行
|
||||
self.getpage()
|
||||
else:
|
||||
#后续运行只需更换出发与目的地
|
||||
self.changecity()
|
||||
|
||||
#运行结束退出
|
||||
self.driver.quit()
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
citys=[]
|
||||
city=['上海','广州','深圳','北京']
|
||||
#形成城市对
|
||||
ytic=list(reversed(city))
|
||||
for m in city:
|
||||
for n in ytic:
|
||||
if m==n:
|
||||
continue
|
||||
else:
|
||||
citys.append([m,n])
|
||||
fly = FLIGHT()
|
||||
fly.demain(citys)
|
||||
print('\n程序运行完成!!!!')
|
||||
|
@ -0,0 +1,143 @@
|
||||
import requests
|
||||
import datetime
|
||||
import re
|
||||
import demjson
|
||||
import time
|
||||
import pandas as pd
|
||||
|
||||
def create_assist_date(datestart = None,dateend = None):
|
||||
# 创建日期辅助表
|
||||
if datestart is None:
|
||||
datestart = '2020-01-01'
|
||||
if dateend is None:
|
||||
dateend = (datetime.datetime.now()+datetime.timedelta(days=-1)).strftime('%Y-%m-%d')
|
||||
|
||||
# 转为日期格式
|
||||
datestart=datetime.datetime.strptime(datestart,'%Y-%m-%d')
|
||||
dateend=datetime.datetime.strptime(dateend,'%Y-%m-%d')
|
||||
date_list = []
|
||||
date_list.append(datestart.strftime('%Y-%m-%d'))
|
||||
while datestart<dateend:
|
||||
# 日期叠加一天
|
||||
datestart+=datetime.timedelta(days=+1)
|
||||
# 日期转字符串存入列表
|
||||
date_list.append(datestart.strftime('%Y-%m-%d'))
|
||||
return date_list
|
||||
|
||||
def getdata(citys,dateseries):
|
||||
url='https://www.lsjpjg.com/getthis.php'
|
||||
|
||||
headers={
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
'Host': 'www.lsjpjg.com',
|
||||
'Origin': 'https://www.lsjpjg.com',
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4647.116 Safari/537.36',
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
}
|
||||
|
||||
for city in citys:
|
||||
df=pd.DataFrame()
|
||||
err=0
|
||||
|
||||
for date in dateseries:
|
||||
|
||||
data={'dep_dt': date,'dep_ct': city[0],'arr_ct': city[1]}
|
||||
res=requests.post(url, headers=headers,data=data)
|
||||
#判断航线是否一直不存在
|
||||
if res.text=='\ufeff[]' :
|
||||
print(city,'无航班',date)
|
||||
err+=1
|
||||
#数量超过阈值则中断该航线
|
||||
if err>30:
|
||||
break
|
||||
continue
|
||||
else:
|
||||
err-=1
|
||||
print(city,date)
|
||||
|
||||
res.encoding=res.apparent_encoding
|
||||
NewResponse = re.sub(r"/","",res.text)
|
||||
try:
|
||||
r=NewResponse.encode('utf-8')
|
||||
j=demjson.decode(r)
|
||||
except:
|
||||
continue
|
||||
temp=pd.DataFrame(j)
|
||||
try:
|
||||
temp.drop('icon',axis=1,inplace=True)
|
||||
temp['出发日期']=date
|
||||
except:
|
||||
continue
|
||||
df=pd.concat([df,temp])
|
||||
time.sleep(0.5)
|
||||
|
||||
filename=city[0]+'-'+city[1]
|
||||
#处理原始数据
|
||||
proc_data(filename,df,interval=8)
|
||||
|
||||
|
||||
def proc_data(filename,df,interval=8):
|
||||
#保存原始数据至本地
|
||||
df.to_csv(filename+'.csv',encoding='GB18030')
|
||||
df['全票价']=0
|
||||
df['日期差']=None
|
||||
|
||||
for i in df.index:
|
||||
try:
|
||||
if not '经济' in df['discount'][i]:
|
||||
df.drop(index=i,inplace=True)
|
||||
elif '折' in df['discount'][i]:
|
||||
#判断出发日期与查询日期之间的间隔是否大于阈值
|
||||
delta=datetime.datetime.strptime(df['出发日期'][i],'%Y-%m-%d')-datetime.datetime.strptime(df['qry_dt'][i],'%Y-%m-%d')
|
||||
if delta.days >interval:
|
||||
df.drop(index=i,inplace=True)
|
||||
continue
|
||||
else:
|
||||
df.loc[i,'日期差']=delta.days
|
||||
#通过折扣率计算全票价
|
||||
discount=float(re.findall('\d+\.?\d*',df['discount'][i])[0])
|
||||
full_price=df['price'][i]/discount*10
|
||||
df.loc[i,'全票价']=full_price
|
||||
|
||||
elif ('全价'or'经典') in df['discount'][i]:
|
||||
#判断出发日期与查询日期之间的间隔是否大于阈值
|
||||
delta=datetime.datetime.strptime(df['出发日期'][i],'%Y-%m-%d')-datetime.datetime.strptime(df['qry_dt'][i],'%Y-%m-%d')
|
||||
if delta.days >interval:
|
||||
df.drop(index=i,inplace=True)
|
||||
continue
|
||||
else:
|
||||
df.loc[i,'日期差']=delta.days
|
||||
#全票价
|
||||
full_price=df['price'][i]
|
||||
df.loc[i,'全票价']=full_price
|
||||
except:
|
||||
df.drop(index=i,inplace=True)
|
||||
|
||||
avg_full_price=df[df['全票价']!=0].groupby(['出发日期'])[['全票价']].mean()
|
||||
avg_price=df[df['全票价']!=df['price']].groupby(['出发日期'])[['price']].mean()
|
||||
result=pd.concat([avg_price,avg_full_price],axis=1)
|
||||
|
||||
result['折扣']=result['price']/result['全票价']
|
||||
|
||||
#将处理后的数据保存至本地
|
||||
result.to_csv(result+'-'+filename+'.csv',encoding='GB18030')
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
citys=[]
|
||||
#设置开始与结束日期
|
||||
dateseries=create_assist_date(datestart = None,dateend = None)
|
||||
|
||||
city=['上海','广州','深圳','北京']
|
||||
ytic=list(reversed(city))
|
||||
for m in city:
|
||||
for n in ytic:
|
||||
if m==n:
|
||||
continue
|
||||
else:
|
||||
citys.append([m,n])
|
||||
|
||||
getdata(citys,dateseries)
|
@ -0,0 +1,17 @@
|
||||
# must be unique in a given SonarQube instance
|
||||
sonar.projectKey=clawer
|
||||
|
||||
# --- optional properties ---
|
||||
|
||||
# defaults to project key
|
||||
sonar.projectName=clawer
|
||||
# defaults to 'not provided'
|
||||
#sonar.projectVersion=1.0
|
||||
|
||||
# Path is relative to the sonar-project.properties file. Defaults to .
|
||||
#sonar.sources=src,WebContent
|
||||
|
||||
# Encoding of the source code. Default is default system encoding
|
||||
sonar.sourceEncoding=UTF-8
|
||||
|
||||
#sonar.java.binaries=target/classes/javabean,target/classes/servlet
|
@ -0,0 +1 @@
|
||||
../atob/bin/atob.js
|
@ -0,0 +1 @@
|
||||
../autoprefixer/bin/autoprefixer
|
@ -0,0 +1 @@
|
||||
../browserslist/cli.js
|
@ -0,0 +1 @@
|
||||
../esprima/bin/esparse.js
|
@ -0,0 +1 @@
|
||||
../esprima/bin/esvalidate.js
|
@ -0,0 +1 @@
|
||||
../gonzales-pe/bin/gonzales.js
|
@ -0,0 +1 @@
|
||||
../js-yaml/bin/js-yaml.js
|
@ -0,0 +1 @@
|
||||
../jsesc/bin/jsesc
|
@ -0,0 +1 @@
|
||||
../json5/lib/cli.js
|
@ -0,0 +1 @@
|
||||
../mkdirp/bin/cmd.js
|
@ -0,0 +1 @@
|
||||
../@babel/parser/bin/babel-parser.js
|
@ -0,0 +1 @@
|
||||
../rimraf/bin.js
|
@ -0,0 +1 @@
|
||||
../semver/bin/semver
|
@ -0,0 +1 @@
|
||||
../specificity/bin/specificity
|
@ -0,0 +1 @@
|
||||
../stylelint/bin/stylelint.js
|
@ -0,0 +1 @@
|
||||
../which/bin/which
|
22
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/code-frame/LICENSE
generated
vendored
@ -0,0 +1,22 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2014-2018 Sebastian McKenzie <sebmck@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
19
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/code-frame/README.md
generated
vendored
@ -0,0 +1,19 @@
|
||||
# @babel/code-frame
|
||||
|
||||
> Generate errors that contain a code frame that point to source locations.
|
||||
|
||||
See our website [@babel/code-frame](https://babeljs.io/docs/en/next/babel-code-frame.html) for more information.
|
||||
|
||||
## Install
|
||||
|
||||
Using npm:
|
||||
|
||||
```sh
|
||||
npm install --save-dev @babel/code-frame
|
||||
```
|
||||
|
||||
or using yarn:
|
||||
|
||||
```sh
|
||||
yarn add @babel/code-frame --dev
|
||||
```
|
173
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/code-frame/lib/index.js
generated
vendored
@ -0,0 +1,173 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.codeFrameColumns = codeFrameColumns;
|
||||
exports.default = _default;
|
||||
|
||||
function _highlight() {
|
||||
const data = _interopRequireWildcard(require("@babel/highlight"));
|
||||
|
||||
_highlight = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = Object.defineProperty && Object.getOwnPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : {}; if (desc.get || desc.set) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } } newObj.default = obj; return newObj; } }
|
||||
|
||||
let deprecationWarningShown = false;
|
||||
|
||||
function getDefs(chalk) {
|
||||
return {
|
||||
gutter: chalk.grey,
|
||||
marker: chalk.red.bold,
|
||||
message: chalk.red.bold
|
||||
};
|
||||
}
|
||||
|
||||
const NEWLINE = /\r\n|[\n\r\u2028\u2029]/;
|
||||
|
||||
function getMarkerLines(loc, source, opts) {
|
||||
const startLoc = Object.assign({
|
||||
column: 0,
|
||||
line: -1
|
||||
}, loc.start);
|
||||
const endLoc = Object.assign({}, startLoc, loc.end);
|
||||
const {
|
||||
linesAbove = 2,
|
||||
linesBelow = 3
|
||||
} = opts || {};
|
||||
const startLine = startLoc.line;
|
||||
const startColumn = startLoc.column;
|
||||
const endLine = endLoc.line;
|
||||
const endColumn = endLoc.column;
|
||||
let start = Math.max(startLine - (linesAbove + 1), 0);
|
||||
let end = Math.min(source.length, endLine + linesBelow);
|
||||
|
||||
if (startLine === -1) {
|
||||
start = 0;
|
||||
}
|
||||
|
||||
if (endLine === -1) {
|
||||
end = source.length;
|
||||
}
|
||||
|
||||
const lineDiff = endLine - startLine;
|
||||
const markerLines = {};
|
||||
|
||||
if (lineDiff) {
|
||||
for (let i = 0; i <= lineDiff; i++) {
|
||||
const lineNumber = i + startLine;
|
||||
|
||||
if (!startColumn) {
|
||||
markerLines[lineNumber] = true;
|
||||
} else if (i === 0) {
|
||||
const sourceLength = source[lineNumber - 1].length;
|
||||
markerLines[lineNumber] = [startColumn, sourceLength - startColumn];
|
||||
} else if (i === lineDiff) {
|
||||
markerLines[lineNumber] = [0, endColumn];
|
||||
} else {
|
||||
const sourceLength = source[lineNumber - i].length;
|
||||
markerLines[lineNumber] = [0, sourceLength];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (startColumn === endColumn) {
|
||||
if (startColumn) {
|
||||
markerLines[startLine] = [startColumn, 0];
|
||||
} else {
|
||||
markerLines[startLine] = true;
|
||||
}
|
||||
} else {
|
||||
markerLines[startLine] = [startColumn, endColumn - startColumn];
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
end,
|
||||
markerLines
|
||||
};
|
||||
}
|
||||
|
||||
function codeFrameColumns(rawLines, loc, opts = {}) {
|
||||
const highlighted = (opts.highlightCode || opts.forceColor) && (0, _highlight().shouldHighlight)(opts);
|
||||
const chalk = (0, _highlight().getChalk)(opts);
|
||||
const defs = getDefs(chalk);
|
||||
|
||||
const maybeHighlight = (chalkFn, string) => {
|
||||
return highlighted ? chalkFn(string) : string;
|
||||
};
|
||||
|
||||
if (highlighted) rawLines = (0, _highlight().default)(rawLines, opts);
|
||||
const lines = rawLines.split(NEWLINE);
|
||||
const {
|
||||
start,
|
||||
end,
|
||||
markerLines
|
||||
} = getMarkerLines(loc, lines, opts);
|
||||
const hasColumns = loc.start && typeof loc.start.column === "number";
|
||||
const numberMaxWidth = String(end).length;
|
||||
let frame = lines.slice(start, end).map((line, index) => {
|
||||
const number = start + 1 + index;
|
||||
const paddedNumber = ` ${number}`.slice(-numberMaxWidth);
|
||||
const gutter = ` ${paddedNumber} | `;
|
||||
const hasMarker = markerLines[number];
|
||||
const lastMarkerLine = !markerLines[number + 1];
|
||||
|
||||
if (hasMarker) {
|
||||
let markerLine = "";
|
||||
|
||||
if (Array.isArray(hasMarker)) {
|
||||
const markerSpacing = line.slice(0, Math.max(hasMarker[0] - 1, 0)).replace(/[^\t]/g, " ");
|
||||
const numberOfMarkers = hasMarker[1] || 1;
|
||||
markerLine = ["\n ", maybeHighlight(defs.gutter, gutter.replace(/\d/g, " ")), markerSpacing, maybeHighlight(defs.marker, "^").repeat(numberOfMarkers)].join("");
|
||||
|
||||
if (lastMarkerLine && opts.message) {
|
||||
markerLine += " " + maybeHighlight(defs.message, opts.message);
|
||||
}
|
||||
}
|
||||
|
||||
return [maybeHighlight(defs.marker, ">"), maybeHighlight(defs.gutter, gutter), line, markerLine].join("");
|
||||
} else {
|
||||
return ` ${maybeHighlight(defs.gutter, gutter)}${line}`;
|
||||
}
|
||||
}).join("\n");
|
||||
|
||||
if (opts.message && !hasColumns) {
|
||||
frame = `${" ".repeat(numberMaxWidth + 1)}${opts.message}\n${frame}`;
|
||||
}
|
||||
|
||||
if (highlighted) {
|
||||
return chalk.reset(frame);
|
||||
} else {
|
||||
return frame;
|
||||
}
|
||||
}
|
||||
|
||||
function _default(rawLines, lineNumber, colNumber, opts = {}) {
|
||||
if (!deprecationWarningShown) {
|
||||
deprecationWarningShown = true;
|
||||
const message = "Passing lineNumber and colNumber is deprecated to @babel/code-frame. Please use `codeFrameColumns`.";
|
||||
|
||||
if (process.emitWarning) {
|
||||
process.emitWarning(message, "DeprecationWarning");
|
||||
} else {
|
||||
const deprecationError = new Error(message);
|
||||
deprecationError.name = "DeprecationWarning";
|
||||
console.warn(new Error(message));
|
||||
}
|
||||
}
|
||||
|
||||
colNumber = Math.max(colNumber, 0);
|
||||
const location = {
|
||||
start: {
|
||||
column: colNumber,
|
||||
line: lineNumber
|
||||
}
|
||||
};
|
||||
return codeFrameColumns(rawLines, location, opts);
|
||||
}
|
54
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/code-frame/package.json
generated
vendored
@ -0,0 +1,54 @@
|
||||
{
|
||||
"_args": [
|
||||
[
|
||||
"@babel/code-frame@7.0.0",
|
||||
"/home/travis/build/SonarSource/sonar-css/sonar-css-plugin/css-bundle"
|
||||
]
|
||||
],
|
||||
"_from": "@babel/code-frame@7.0.0",
|
||||
"_id": "@babel/code-frame@7.0.0",
|
||||
"_inBundle": false,
|
||||
"_integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==",
|
||||
"_location": "/@babel/code-frame",
|
||||
"_phantomChildren": {},
|
||||
"_requested": {
|
||||
"type": "version",
|
||||
"registry": true,
|
||||
"raw": "@babel/code-frame@7.0.0",
|
||||
"name": "@babel/code-frame",
|
||||
"escapedName": "@babel%2fcode-frame",
|
||||
"scope": "@babel",
|
||||
"rawSpec": "7.0.0",
|
||||
"saveSpec": null,
|
||||
"fetchSpec": "7.0.0"
|
||||
},
|
||||
"_requiredBy": [
|
||||
"/@babel/core",
|
||||
"/@babel/template",
|
||||
"/@babel/traverse"
|
||||
],
|
||||
"_resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz",
|
||||
"_spec": "7.0.0",
|
||||
"_where": "/home/travis/build/SonarSource/sonar-css/sonar-css-plugin/css-bundle",
|
||||
"author": {
|
||||
"name": "Sebastian McKenzie",
|
||||
"email": "sebmck@gmail.com"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/highlight": "^7.0.0"
|
||||
},
|
||||
"description": "Generate errors that contain a code frame that point to source locations.",
|
||||
"devDependencies": {
|
||||
"chalk": "^2.0.0",
|
||||
"strip-ansi": "^4.0.0"
|
||||
},
|
||||
"homepage": "https://babeljs.io/",
|
||||
"license": "MIT",
|
||||
"main": "lib/index.js",
|
||||
"name": "@babel/code-frame",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/babel/babel/tree/master/packages/babel-code-frame"
|
||||
},
|
||||
"version": "7.0.0"
|
||||
}
|
@ -0,0 +1,22 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2014-present Sebastian McKenzie and other contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@ -0,0 +1,19 @@
|
||||
# @babel/core
|
||||
|
||||
> Babel compiler core.
|
||||
|
||||
See our website [@babel/core](https://babeljs.io/docs/en/next/babel-core.html) for more information or the [issues](https://github.com/babel/babel/issues?utf8=%E2%9C%93&q=is%3Aissue+label%3A%22pkg%3A%20core%22+is%3Aopen) associated with this package.
|
||||
|
||||
## Install
|
||||
|
||||
Using npm:
|
||||
|
||||
```sh
|
||||
npm install --save-dev @babel/core
|
||||
```
|
||||
|
||||
or using yarn:
|
||||
|
||||
```sh
|
||||
yarn add @babel/core --dev
|
||||
```
|
199
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/core/lib/config/caching.js
generated
vendored
@ -0,0 +1,199 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.makeStrongCache = makeStrongCache;
|
||||
exports.makeWeakCache = makeWeakCache;
|
||||
exports.assertSimpleType = assertSimpleType;
|
||||
|
||||
function makeStrongCache(handler) {
|
||||
return makeCachedFunction(new Map(), handler);
|
||||
}
|
||||
|
||||
function makeWeakCache(handler) {
|
||||
return makeCachedFunction(new WeakMap(), handler);
|
||||
}
|
||||
|
||||
function makeCachedFunction(callCache, handler) {
|
||||
return function cachedFunction(arg, data) {
|
||||
let cachedValue = callCache.get(arg);
|
||||
|
||||
if (cachedValue) {
|
||||
for (const _ref of cachedValue) {
|
||||
const {
|
||||
value,
|
||||
valid
|
||||
} = _ref;
|
||||
if (valid(data)) return value;
|
||||
}
|
||||
}
|
||||
|
||||
const cache = new CacheConfigurator(data);
|
||||
const value = handler(arg, cache);
|
||||
if (!cache.configured()) cache.forever();
|
||||
cache.deactivate();
|
||||
|
||||
switch (cache.mode()) {
|
||||
case "forever":
|
||||
cachedValue = [{
|
||||
value,
|
||||
valid: () => true
|
||||
}];
|
||||
callCache.set(arg, cachedValue);
|
||||
break;
|
||||
|
||||
case "invalidate":
|
||||
cachedValue = [{
|
||||
value,
|
||||
valid: cache.validator()
|
||||
}];
|
||||
callCache.set(arg, cachedValue);
|
||||
break;
|
||||
|
||||
case "valid":
|
||||
if (cachedValue) {
|
||||
cachedValue.push({
|
||||
value,
|
||||
valid: cache.validator()
|
||||
});
|
||||
} else {
|
||||
cachedValue = [{
|
||||
value,
|
||||
valid: cache.validator()
|
||||
}];
|
||||
callCache.set(arg, cachedValue);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return value;
|
||||
};
|
||||
}
|
||||
|
||||
class CacheConfigurator {
|
||||
constructor(data) {
|
||||
this._active = true;
|
||||
this._never = false;
|
||||
this._forever = false;
|
||||
this._invalidate = false;
|
||||
this._configured = false;
|
||||
this._pairs = [];
|
||||
this._data = data;
|
||||
}
|
||||
|
||||
simple() {
|
||||
return makeSimpleConfigurator(this);
|
||||
}
|
||||
|
||||
mode() {
|
||||
if (this._never) return "never";
|
||||
if (this._forever) return "forever";
|
||||
if (this._invalidate) return "invalidate";
|
||||
return "valid";
|
||||
}
|
||||
|
||||
forever() {
|
||||
if (!this._active) {
|
||||
throw new Error("Cannot change caching after evaluation has completed.");
|
||||
}
|
||||
|
||||
if (this._never) {
|
||||
throw new Error("Caching has already been configured with .never()");
|
||||
}
|
||||
|
||||
this._forever = true;
|
||||
this._configured = true;
|
||||
}
|
||||
|
||||
never() {
|
||||
if (!this._active) {
|
||||
throw new Error("Cannot change caching after evaluation has completed.");
|
||||
}
|
||||
|
||||
if (this._forever) {
|
||||
throw new Error("Caching has already been configured with .forever()");
|
||||
}
|
||||
|
||||
this._never = true;
|
||||
this._configured = true;
|
||||
}
|
||||
|
||||
using(handler) {
|
||||
if (!this._active) {
|
||||
throw new Error("Cannot change caching after evaluation has completed.");
|
||||
}
|
||||
|
||||
if (this._never || this._forever) {
|
||||
throw new Error("Caching has already been configured with .never or .forever()");
|
||||
}
|
||||
|
||||
this._configured = true;
|
||||
const key = handler(this._data);
|
||||
|
||||
this._pairs.push([key, handler]);
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
invalidate(handler) {
|
||||
if (!this._active) {
|
||||
throw new Error("Cannot change caching after evaluation has completed.");
|
||||
}
|
||||
|
||||
if (this._never || this._forever) {
|
||||
throw new Error("Caching has already been configured with .never or .forever()");
|
||||
}
|
||||
|
||||
this._invalidate = true;
|
||||
this._configured = true;
|
||||
const key = handler(this._data);
|
||||
|
||||
this._pairs.push([key, handler]);
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
validator() {
|
||||
const pairs = this._pairs;
|
||||
return data => pairs.every(([key, fn]) => key === fn(data));
|
||||
}
|
||||
|
||||
deactivate() {
|
||||
this._active = false;
|
||||
}
|
||||
|
||||
configured() {
|
||||
return this._configured;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function makeSimpleConfigurator(cache) {
|
||||
function cacheFn(val) {
|
||||
if (typeof val === "boolean") {
|
||||
if (val) cache.forever();else cache.never();
|
||||
return;
|
||||
}
|
||||
|
||||
return cache.using(() => assertSimpleType(val()));
|
||||
}
|
||||
|
||||
cacheFn.forever = () => cache.forever();
|
||||
|
||||
cacheFn.never = () => cache.never();
|
||||
|
||||
cacheFn.using = cb => cache.using(() => assertSimpleType(cb()));
|
||||
|
||||
cacheFn.invalidate = cb => cache.invalidate(() => assertSimpleType(cb()));
|
||||
|
||||
return cacheFn;
|
||||
}
|
||||
|
||||
function assertSimpleType(value) {
|
||||
if (value != null && typeof value !== "string" && typeof value !== "boolean" && typeof value !== "number") {
|
||||
throw new Error("Cache keys must be either string, boolean, number, null, or undefined.");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
439
src/GenFlightRec/.scannerwork/css-bundle/node_modules/@babel/core/lib/config/config-chain.js
generated
vendored
@ -0,0 +1,439 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.buildPresetChain = buildPresetChain;
|
||||
exports.buildRootChain = buildRootChain;
|
||||
exports.buildPresetChainWalker = void 0;
|
||||
|
||||
function _path() {
|
||||
const data = _interopRequireDefault(require("path"));
|
||||
|
||||
_path = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _debug() {
|
||||
const data = _interopRequireDefault(require("debug"));
|
||||
|
||||
_debug = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
var _options = require("./validation/options");
|
||||
|
||||
var _patternToRegex = _interopRequireDefault(require("./pattern-to-regex"));
|
||||
|
||||
var _files = require("./files");
|
||||
|
||||
var _caching = require("./caching");
|
||||
|
||||
var _configDescriptors = require("./config-descriptors");
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
const debug = (0, _debug().default)("babel:config:config-chain");
|
||||
|
||||
function buildPresetChain(arg, context) {
|
||||
const chain = buildPresetChainWalker(arg, context);
|
||||
if (!chain) return null;
|
||||
return {
|
||||
plugins: dedupDescriptors(chain.plugins),
|
||||
presets: dedupDescriptors(chain.presets),
|
||||
options: chain.options.map(o => normalizeOptions(o))
|
||||
};
|
||||
}
|
||||
|
||||
const buildPresetChainWalker = makeChainWalker({
|
||||
init: arg => arg,
|
||||
root: preset => loadPresetDescriptors(preset),
|
||||
env: (preset, envName) => loadPresetEnvDescriptors(preset)(envName),
|
||||
overrides: (preset, index) => loadPresetOverridesDescriptors(preset)(index),
|
||||
overridesEnv: (preset, index, envName) => loadPresetOverridesEnvDescriptors(preset)(index)(envName)
|
||||
});
|
||||
exports.buildPresetChainWalker = buildPresetChainWalker;
|
||||
const loadPresetDescriptors = (0, _caching.makeWeakCache)(preset => buildRootDescriptors(preset, preset.alias, _configDescriptors.createUncachedDescriptors));
|
||||
const loadPresetEnvDescriptors = (0, _caching.makeWeakCache)(preset => (0, _caching.makeStrongCache)(envName => buildEnvDescriptors(preset, preset.alias, _configDescriptors.createUncachedDescriptors, envName)));
|
||||
const loadPresetOverridesDescriptors = (0, _caching.makeWeakCache)(preset => (0, _caching.makeStrongCache)(index => buildOverrideDescriptors(preset, preset.alias, _configDescriptors.createUncachedDescriptors, index)));
|
||||
const loadPresetOverridesEnvDescriptors = (0, _caching.makeWeakCache)(preset => (0, _caching.makeStrongCache)(index => (0, _caching.makeStrongCache)(envName => buildOverrideEnvDescriptors(preset, preset.alias, _configDescriptors.createUncachedDescriptors, index, envName))));
|
||||
|
||||
function buildRootChain(opts, context) {
|
||||
const programmaticChain = loadProgrammaticChain({
|
||||
options: opts,
|
||||
dirname: context.cwd
|
||||
}, context);
|
||||
if (!programmaticChain) return null;
|
||||
let configFile;
|
||||
|
||||
if (typeof opts.configFile === "string") {
|
||||
configFile = (0, _files.loadConfig)(opts.configFile, context.cwd, context.envName, context.caller);
|
||||
} else if (opts.configFile !== false) {
|
||||
configFile = (0, _files.findRootConfig)(context.root, context.envName, context.caller);
|
||||
}
|
||||
|
||||
let {
|
||||
babelrc,
|
||||
babelrcRoots
|
||||
} = opts;
|
||||
let babelrcRootsDirectory = context.cwd;
|
||||
const configFileChain = emptyChain();
|
||||
|
||||
if (configFile) {
|
||||
const validatedFile = validateConfigFile(configFile);
|
||||
const result = loadFileChain(validatedFile, context);
|
||||
if (!result) return null;
|
||||
|
||||
if (babelrc === undefined) {
|
||||
babelrc = validatedFile.options.babelrc;
|
||||
}
|
||||
|
||||
if (babelrcRoots === undefined) {
|
||||
babelrcRootsDirectory = validatedFile.dirname;
|
||||
babelrcRoots = validatedFile.options.babelrcRoots;
|
||||
}
|
||||
|
||||
mergeChain(configFileChain, result);
|
||||
}
|
||||
|
||||
const pkgData = typeof context.filename === "string" ? (0, _files.findPackageData)(context.filename) : null;
|
||||
let ignoreFile, babelrcFile;
|
||||
const fileChain = emptyChain();
|
||||
|
||||
if ((babelrc === true || babelrc === undefined) && pkgData && babelrcLoadEnabled(context, pkgData, babelrcRoots, babelrcRootsDirectory)) {
|
||||
({
|
||||
ignore: ignoreFile,
|
||||
config: babelrcFile
|
||||
} = (0, _files.findRelativeConfig)(pkgData, context.envName, context.caller));
|
||||
|
||||
if (ignoreFile && shouldIgnore(context, ignoreFile.ignore, null, ignoreFile.dirname)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (babelrcFile) {
|
||||
const result = loadFileChain(validateBabelrcFile(babelrcFile), context);
|
||||
if (!result) return null;
|
||||
mergeChain(fileChain, result);
|
||||
}
|
||||
}
|
||||
|
||||
const chain = mergeChain(mergeChain(mergeChain(emptyChain(), configFileChain), fileChain), programmaticChain);
|
||||
return {
|
||||
plugins: dedupDescriptors(chain.plugins),
|
||||
presets: dedupDescriptors(chain.presets),
|
||||
options: chain.options.map(o => normalizeOptions(o)),
|
||||
ignore: ignoreFile || undefined,
|
||||
babelrc: babelrcFile || undefined,
|
||||
config: configFile || undefined
|
||||
};
|
||||
}
|
||||
|
||||
function babelrcLoadEnabled(context, pkgData, babelrcRoots, babelrcRootsDirectory) {
|
||||
if (typeof babelrcRoots === "boolean") return babelrcRoots;
|
||||
const absoluteRoot = context.root;
|
||||
|
||||
if (babelrcRoots === undefined) {
|
||||
return pkgData.directories.indexOf(absoluteRoot) !== -1;
|
||||
}
|
||||
|
||||
let babelrcPatterns = babelrcRoots;
|
||||
if (!Array.isArray(babelrcPatterns)) babelrcPatterns = [babelrcPatterns];
|
||||
babelrcPatterns = babelrcPatterns.map(pat => {
|
||||
return typeof pat === "string" ? _path().default.resolve(babelrcRootsDirectory, pat) : pat;
|
||||
});
|
||||
|
||||
if (babelrcPatterns.length === 1 && babelrcPatterns[0] === absoluteRoot) {
|
||||
return pkgData.directories.indexOf(absoluteRoot) !== -1;
|
||||
}
|
||||
|
||||
return babelrcPatterns.some(pat => {
|
||||
if (typeof pat === "string") {
|
||||
pat = (0, _patternToRegex.default)(pat, babelrcRootsDirectory);
|
||||
}
|
||||
|
||||
return pkgData.directories.some(directory => {
|
||||
return matchPattern(pat, babelrcRootsDirectory, directory, context);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const validateConfigFile = (0, _caching.makeWeakCache)(file => ({
|
||||
filepath: file.filepath,
|
||||
dirname: file.dirname,
|
||||
options: (0, _options.validate)("configfile", file.options)
|
||||
}));
|
||||
const validateBabelrcFile = (0, _caching.makeWeakCache)(file => ({
|
||||
filepath: file.filepath,
|
||||
dirname: file.dirname,
|
||||
options: (0, _options.validate)("babelrcfile", file.options)
|
||||
}));
|
||||
const validateExtendFile = (0, _caching.makeWeakCache)(file => ({
|
||||
filepath: file.filepath,
|
||||
dirname: file.dirname,
|
||||
options: (0, _options.validate)("extendsfile", file.options)
|
||||
}));
|
||||
const loadProgrammaticChain = makeChainWalker({
|
||||
root: input => buildRootDescriptors(input, "base", _configDescriptors.createCachedDescriptors),
|
||||
env: (input, envName) => buildEnvDescriptors(input, "base", _configDescriptors.createCachedDescriptors, envName),
|
||||
overrides: (input, index) => buildOverrideDescriptors(input, "base", _configDescriptors.createCachedDescriptors, index),
|
||||
overridesEnv: (input, index, envName) => buildOverrideEnvDescriptors(input, "base", _configDescriptors.createCachedDescriptors, index, envName)
|
||||
});
|
||||
const loadFileChain = makeChainWalker({
|
||||
root: file => loadFileDescriptors(file),
|
||||
env: (file, envName) => loadFileEnvDescriptors(file)(envName),
|
||||
overrides: (file, index) => loadFileOverridesDescriptors(file)(index),
|
||||
overridesEnv: (file, index, envName) => loadFileOverridesEnvDescriptors(file)(index)(envName)
|
||||
});
|
||||
const loadFileDescriptors = (0, _caching.makeWeakCache)(file => buildRootDescriptors(file, file.filepath, _configDescriptors.createUncachedDescriptors));
|
||||
const loadFileEnvDescriptors = (0, _caching.makeWeakCache)(file => (0, _caching.makeStrongCache)(envName => buildEnvDescriptors(file, file.filepath, _configDescriptors.createUncachedDescriptors, envName)));
|
||||
const loadFileOverridesDescriptors = (0, _caching.makeWeakCache)(file => (0, _caching.makeStrongCache)(index => buildOverrideDescriptors(file, file.filepath, _configDescriptors.createUncachedDescriptors, index)));
|
||||
const loadFileOverridesEnvDescriptors = (0, _caching.makeWeakCache)(file => (0, _caching.makeStrongCache)(index => (0, _caching.makeStrongCache)(envName => buildOverrideEnvDescriptors(file, file.filepath, _configDescriptors.createUncachedDescriptors, index, envName))));
|
||||
|
||||
function buildRootDescriptors({
|
||||
dirname,
|
||||
options
|
||||
}, alias, descriptors) {
|
||||
return descriptors(dirname, options, alias);
|
||||
}
|
||||
|
||||
function buildEnvDescriptors({
|
||||
dirname,
|
||||
options
|
||||
}, alias, descriptors, envName) {
|
||||
const opts = options.env && options.env[envName];
|
||||
return opts ? descriptors(dirname, opts, `${alias}.env["${envName}"]`) : null;
|
||||
}
|
||||
|
||||
function buildOverrideDescriptors({
|
||||
dirname,
|
||||
options
|
||||
}, alias, descriptors, index) {
|
||||
const opts = options.overrides && options.overrides[index];
|
||||
if (!opts) throw new Error("Assertion failure - missing override");
|
||||
return descriptors(dirname, opts, `${alias}.overrides[${index}]`);
|
||||
}
|
||||
|
||||
function buildOverrideEnvDescriptors({
|
||||
dirname,
|
||||
options
|
||||
}, alias, descriptors, index, envName) {
|
||||
const override = options.overrides && options.overrides[index];
|
||||
if (!override) throw new Error("Assertion failure - missing override");
|
||||
const opts = override.env && override.env[envName];
|
||||
return opts ? descriptors(dirname, opts, `${alias}.overrides[${index}].env["${envName}"]`) : null;
|
||||
}
|
||||
|
||||
function makeChainWalker({
|
||||
root,
|
||||
env,
|
||||
overrides,
|
||||
overridesEnv
|
||||
}) {
|
||||
return (input, context, files = new Set()) => {
|
||||
const {
|
||||
dirname
|
||||
} = input;
|
||||
const flattenedConfigs = [];
|
||||
const rootOpts = root(input);
|
||||
|
||||
if (configIsApplicable(rootOpts, dirname, context)) {
|
||||
flattenedConfigs.push(rootOpts);
|
||||
const envOpts = env(input, context.envName);
|
||||
|
||||
if (envOpts && configIsApplicable(envOpts, dirname, context)) {
|
||||
flattenedConfigs.push(envOpts);
|
||||
}
|
||||
|
||||
(rootOpts.options.overrides || []).forEach((_, index) => {
|
||||
const overrideOps = overrides(input, index);
|
||||
|
||||
if (configIsApplicable(overrideOps, dirname, context)) {
|
||||
flattenedConfigs.push(overrideOps);
|
||||
const overrideEnvOpts = overridesEnv(input, index, context.envName);
|
||||
|
||||
if (overrideEnvOpts && configIsApplicable(overrideEnvOpts, dirname, context)) {
|
||||
flattenedConfigs.push(overrideEnvOpts);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (flattenedConfigs.some(({
|
||||
options: {
|
||||
ignore,
|
||||
only
|
||||
}
|
||||
}) => shouldIgnore(context, ignore, only, dirname))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const chain = emptyChain();
|
||||
|
||||
for (const op of flattenedConfigs) {
|
||||
if (!mergeExtendsChain(chain, op.options, dirname, context, files)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
mergeChainOpts(chain, op);
|
||||
}
|
||||
|
||||
return chain;
|
||||
};
|
||||
}
|
||||
|
||||
function mergeExtendsChain(chain, opts, dirname, context, files) {
|
||||
if (opts.extends === undefined) return true;
|
||||
const file = (0, _files.loadConfig)(opts.extends, dirname, context.envName, context.caller);
|
||||
|
||||
if (files.has(file)) {
|
||||
throw new Error(`Configuration cycle detected loading ${file.filepath}.\n` + `File already loaded following the config chain:\n` + Array.from(files, file => ` - ${file.filepath}`).join("\n"));
|
||||
}
|
||||
|
||||
files.add(file);
|
||||
const fileChain = loadFileChain(validateExtendFile(file), context, files);
|
||||
files.delete(file);
|
||||
if (!fileChain) return false;
|
||||
mergeChain(chain, fileChain);
|
||||
return true;
|
||||
}
|
||||
|
||||
function mergeChain(target, source) {
|
||||
target.options.push(...source.options);
|
||||
target.plugins.push(...source.plugins);
|
||||
target.presets.push(...source.presets);
|
||||
return target;
|
||||
}
|
||||
|
||||
function mergeChainOpts(target, {
|
||||
options,
|
||||
plugins,
|
||||
presets
|
||||
}) {
|
||||
target.options.push(options);
|
||||
target.plugins.push(...plugins());
|
||||
target.presets.push(...presets());
|
||||
return target;
|
||||
}
|
||||
|
||||
function emptyChain() {
|
||||
return {
|
||||
options: [],
|
||||
presets: [],
|
||||
plugins: []
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeOptions(opts) {
|
||||
const options = Object.assign({}, opts);
|
||||
delete options.extends;
|
||||
delete options.env;
|
||||
delete options.overrides;
|
||||
delete options.plugins;
|
||||
delete options.presets;
|
||||
delete options.passPerPreset;
|
||||
delete options.ignore;
|
||||
delete options.only;
|
||||
delete options.test;
|
||||
delete options.include;
|
||||
delete options.exclude;
|
||||
|
||||
if (options.hasOwnProperty("sourceMap")) {
|
||||
options.sourceMaps = options.sourceMap;
|
||||
delete options.sourceMap;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function dedupDescriptors(items) {
|
||||
const map = new Map();
|
||||
const descriptors = [];
|
||||
|
||||
for (const item of items) {
|
||||
if (typeof item.value === "function") {
|
||||
const fnKey = item.value;
|
||||
let nameMap = map.get(fnKey);
|
||||
|
||||
if (!nameMap) {
|
||||
nameMap = new Map();
|
||||
map.set(fnKey, nameMap);
|
||||
}
|
||||
|
||||
let desc = nameMap.get(item.name);
|
||||
|
||||
if (!desc) {
|
||||
desc = {
|
||||
value: item
|
||||
};
|
||||
descriptors.push(desc);
|
||||
if (!item.ownPass) nameMap.set(item.name, desc);
|
||||
} else {
|
||||
desc.value = item;
|
||||
}
|
||||
} else {
|
||||
descriptors.push({
|
||||
value: item
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return descriptors.reduce((acc, desc) => {
|
||||
acc.push(desc.value);
|
||||
return acc;
|
||||
}, []);
|
||||
}
|
||||
|
||||
function configIsApplicable({
|
||||
options
|
||||
}, dirname, context) {
|
||||
return (options.test === undefined || configFieldIsApplicable(context, options.test, dirname)) && (options.include === undefined || configFieldIsApplicable(context, options.include, dirname)) && (options.exclude === undefined || !configFieldIsApplicable(context, options.exclude, dirname));
|
||||
}
|
||||
|
||||
function configFieldIsApplicable(context, test, dirname) {
|
||||
const patterns = Array.isArray(test) ? test : [test];
|
||||
return matchesPatterns(context, patterns, dirname);
|
||||
}
|
||||
|
||||
function shouldIgnore(context, ignore, only, dirname) {
|
||||
if (ignore && matchesPatterns(context, ignore, dirname)) {
|
||||
debug("Ignored %o because it matched one of %O from %o", context.filename, ignore, dirname);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (only && !matchesPatterns(context, only, dirname)) {
|
||||
debug("Ignored %o because it failed to match one of %O from %o", context.filename, only, dirname);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function matchesPatterns(context, patterns, dirname) {
|
||||
return patterns.some(pattern => matchPattern(pattern, dirname, context.filename, context));
|
||||
}
|
||||
|
||||
function matchPattern(pattern, dirname, pathToTest, context) {
|
||||
if (typeof pattern === "function") {
|
||||
return !!pattern(pathToTest, {
|
||||
dirname,
|
||||
envName: context.envName,
|
||||
caller: context.caller
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof pathToTest !== "string") {
|
||||
throw new Error(`Configuration contains string/RegExp pattern, but no filename was passed to Babel`);
|
||||
}
|
||||
|
||||
if (typeof pattern === "string") {
|
||||
pattern = (0, _patternToRegex.default)(pattern, dirname);
|
||||
}
|
||||
|
||||
return pattern.test(pathToTest);
|
||||
}
|
@ -0,0 +1,210 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.createCachedDescriptors = createCachedDescriptors;
|
||||
exports.createUncachedDescriptors = createUncachedDescriptors;
|
||||
exports.createDescriptor = createDescriptor;
|
||||
|
||||
var _files = require("./files");
|
||||
|
||||
var _item = require("./item");
|
||||
|
||||
var _caching = require("./caching");
|
||||
|
||||
function isEqualDescriptor(a, b) {
|
||||
return a.name === b.name && a.value === b.value && a.options === b.options && a.dirname === b.dirname && a.alias === b.alias && a.ownPass === b.ownPass && (a.file && a.file.request) === (b.file && b.file.request) && (a.file && a.file.resolved) === (b.file && b.file.resolved);
|
||||
}
|
||||
|
||||
function createCachedDescriptors(dirname, options, alias) {
|
||||
const {
|
||||
plugins,
|
||||
presets,
|
||||
passPerPreset
|
||||
} = options;
|
||||
return {
|
||||
options,
|
||||
plugins: plugins ? () => createCachedPluginDescriptors(plugins, dirname)(alias) : () => [],
|
||||
presets: presets ? () => createCachedPresetDescriptors(presets, dirname)(alias)(!!passPerPreset) : () => []
|
||||
};
|
||||
}
|
||||
|
||||
function createUncachedDescriptors(dirname, options, alias) {
|
||||
let plugins;
|
||||
let presets;
|
||||
return {
|
||||
options,
|
||||
plugins: () => {
|
||||
if (!plugins) {
|
||||
plugins = createPluginDescriptors(options.plugins || [], dirname, alias);
|
||||
}
|
||||
|
||||
return plugins;
|
||||
},
|
||||
presets: () => {
|
||||
if (!presets) {
|
||||
presets = createPresetDescriptors(options.presets || [], dirname, alias, !!options.passPerPreset);
|
||||
}
|
||||
|
||||
return presets;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const PRESET_DESCRIPTOR_CACHE = new WeakMap();
|
||||
const createCachedPresetDescriptors = (0, _caching.makeWeakCache)((items, cache) => {
|
||||
const dirname = cache.using(dir => dir);
|
||||
return (0, _caching.makeStrongCache)(alias => (0, _caching.makeStrongCache)(passPerPreset => createPresetDescriptors(items, dirname, alias, passPerPreset).map(desc => loadCachedDescriptor(PRESET_DESCRIPTOR_CACHE, desc))));
|
||||
});
|
||||
const PLUGIN_DESCRIPTOR_CACHE = new WeakMap();
|
||||
const createCachedPluginDescriptors = (0, _caching.makeWeakCache)((items, cache) => {
|
||||
const dirname = cache.using(dir => dir);
|
||||
return (0, _caching.makeStrongCache)(alias => createPluginDescriptors(items, dirname, alias).map(desc => loadCachedDescriptor(PLUGIN_DESCRIPTOR_CACHE, desc)));
|
||||
});
|
||||
const DEFAULT_OPTIONS = {};
|
||||
|
||||
function loadCachedDescriptor(cache, desc) {
|
||||
const {
|
||||
value,
|
||||
options = DEFAULT_OPTIONS
|
||||
} = desc;
|
||||
if (options === false) return desc;
|
||||
let cacheByOptions = cache.get(value);
|
||||
|
||||
if (!cacheByOptions) {
|
||||
cacheByOptions = new WeakMap();
|
||||
cache.set(value, cacheByOptions);
|
||||
}
|
||||
|
||||
let possibilities = cacheByOptions.get(options);
|
||||
|
||||
if (!possibilities) {
|
||||
possibilities = [];
|
||||
cacheByOptions.set(options, possibilities);
|
||||
}
|
||||
|
||||
if (possibilities.indexOf(desc) === -1) {
|
||||
const matches = possibilities.filter(possibility => isEqualDescriptor(possibility, desc));
|
||||
|
||||
if (matches.length > 0) {
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
possibilities.push(desc);
|
||||
}
|
||||
|
||||
return desc;
|
||||
}
|
||||
|
||||
function createPresetDescriptors(items, dirname, alias, passPerPreset) {
|
||||
return createDescriptors("preset", items, dirname, alias, passPerPreset);
|
||||
}
|
||||
|
||||
function createPluginDescriptors(items, dirname, alias) {
|
||||
return createDescriptors("plugin", items, dirname, alias);
|
||||
}
|
||||
|
||||
function createDescriptors(type, items, dirname, alias, ownPass) {
|
||||
const descriptors = items.map((item, index) => createDescriptor(item, dirname, {
|
||||
type,
|
||||
alias: `${alias}$${index}`,
|
||||
ownPass: !!ownPass
|
||||
}));
|
||||
assertNoDuplicates(descriptors);
|
||||
return descriptors;
|
||||
}
|
||||
|
||||
function createDescriptor(pair, dirname, {
|
||||
type,
|
||||
alias,
|
||||
ownPass
|
||||
}) {
|
||||
const desc = (0, _item.getItemDescriptor)(pair);
|
||||
|
||||
if (desc) {
|
||||
return desc;
|
||||
}
|
||||
|
||||
let name;
|
||||
let options;
|
||||
let value = pair;
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 3) {
|
||||
[value, options, name] = value;
|
||||
} else {
|
||||
[value, options] = value;
|
||||
}
|
||||
}
|
||||
|
||||
let file = undefined;
|
||||
let filepath = null;
|
||||
|
||||
if (typeof value === "string") {
|
||||
if (typeof type !== "string") {
|
||||
throw new Error("To resolve a string-based item, the type of item must be given");
|
||||
}
|
||||
|
||||
const resolver = type === "plugin" ? _files.loadPlugin : _files.loadPreset;
|
||||
const request = value;
|
||||
({
|
||||
filepath,
|
||||
value
|
||||
} = resolver(value, dirname));
|
||||
file = {
|
||||
request,
|
||||
resolved: filepath
|
||||
};
|
||||
}
|
||||
|
||||
if (!value) {
|
||||
throw new Error(`Unexpected falsy value: ${String(value)}`);
|
||||
}
|
||||
|
||||
if (typeof value === "object" && value.__esModule) {
|
||||
if (value.default) {
|
||||
value = value.default;
|
||||
} else {
|
||||
throw new Error("Must export a default export when using ES6 modules.");
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof value !== "object" && typeof value !== "function") {
|
||||
throw new Error(`Unsupported format: ${typeof value}. Expected an object or a function.`);
|
||||
}
|
||||
|
||||
if (filepath !== null && typeof value === "object" && value) {
|
||||
throw new Error(`Plugin/Preset files are not allowed to export objects, only functions. In ${filepath}`);
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
alias: filepath || alias,
|
||||
value,
|
||||
options,
|
||||
dirname,
|
||||
ownPass,
|
||||
file
|
||||
};
|
||||
}
|
||||
|
||||
function assertNoDuplicates(items) {
|
||||
const map = new Map();
|
||||
|
||||
for (const item of items) {
|
||||
if (typeof item.value !== "function") continue;
|
||||
let nameMap = map.get(item.value);
|
||||
|
||||
if (!nameMap) {
|
||||
nameMap = new Set();
|
||||
map.set(item.value, nameMap);
|
||||
}
|
||||
|
||||
if (nameMap.has(item.name)) {
|
||||
throw new Error([`Duplicate plugin/preset detected.`, `If you'd like to use two separate instances of a plugin,`, `they need separate names, e.g.`, ``, ` plugins: [`, ` ['some-plugin', {}],`, ` ['some-plugin', {}, 'some unique name'],`, ` ]`].join("\n"));
|
||||
}
|
||||
|
||||
nameMap.add(item.name);
|
||||
}
|
||||
}
|
@ -0,0 +1,323 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.findConfigUpwards = findConfigUpwards;
|
||||
exports.findRelativeConfig = findRelativeConfig;
|
||||
exports.findRootConfig = findRootConfig;
|
||||
exports.loadConfig = loadConfig;
|
||||
|
||||
function _debug() {
|
||||
const data = _interopRequireDefault(require("debug"));
|
||||
|
||||
_debug = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _path() {
|
||||
const data = _interopRequireDefault(require("path"));
|
||||
|
||||
_path = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _fs() {
|
||||
const data = _interopRequireDefault(require("fs"));
|
||||
|
||||
_fs = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _json() {
|
||||
const data = _interopRequireDefault(require("json5"));
|
||||
|
||||
_json = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function _resolve() {
|
||||
const data = _interopRequireDefault(require("resolve"));
|
||||
|
||||
_resolve = function () {
|
||||
return data;
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
var _caching = require("../caching");
|
||||
|
||||
var _configApi = _interopRequireDefault(require("../helpers/config-api"));
|
||||
|
||||
var _utils = require("./utils");
|
||||
|
||||
var _patternToRegex = _interopRequireDefault(require("../pattern-to-regex"));
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
const debug = (0, _debug().default)("babel:config:loading:files:configuration");
|
||||
const BABEL_CONFIG_JS_FILENAME = "babel.config.js";
|
||||
const BABELRC_FILENAME = ".babelrc";
|
||||
const BABELRC_JS_FILENAME = ".babelrc.js";
|
||||
const BABELIGNORE_FILENAME = ".babelignore";
|
||||
|
||||
function findConfigUpwards(rootDir) {
|
||||
let dirname = rootDir;
|
||||
|
||||
while (true) {
|
||||
if (_fs().default.existsSync(_path().default.join(dirname, BABEL_CONFIG_JS_FILENAME))) {
|
||||
return dirname;
|
||||
}
|
||||
|
||||
const nextDir = _path().default.dirname(dirname);
|
||||
|
||||
if (dirname === nextDir) break;
|
||||
dirname = nextDir;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function findRelativeConfig(packageData, envName, caller) {
|
||||
let config = null;
|
||||
let ignore = null;
|
||||
|
||||
const dirname = _path().default.dirname(packageData.filepath);
|
||||
|
||||
for (const loc of packageData.directories) {
|
||||
if (!config) {
|
||||
config = [BABELRC_FILENAME, BABELRC_JS_FILENAME].reduce((previousConfig, name) => {
|
||||
const filepath = _path().default.join(loc, name);
|
||||
|
||||
const config = readConfig(filepath, envName, caller);
|
||||
|
||||
if (config && previousConfig) {
|
||||
throw new Error(`Multiple configuration files found. Please remove one:\n` + ` - ${_path().default.basename(previousConfig.filepath)}\n` + ` - ${name}\n` + `from ${loc}`);
|
||||
}
|
||||
|
||||
return config || previousConfig;
|
||||
}, null);
|
||||
const pkgConfig = packageData.pkg && packageData.pkg.dirname === loc ? packageToBabelConfig(packageData.pkg) : null;
|
||||
|
||||
if (pkgConfig) {
|
||||
if (config) {
|
||||
throw new Error(`Multiple configuration files found. Please remove one:\n` + ` - ${_path().default.basename(pkgConfig.filepath)}#babel\n` + ` - ${_path().default.basename(config.filepath)}\n` + `from ${loc}`);
|
||||
}
|
||||
|
||||
config = pkgConfig;
|
||||
}
|
||||
|
||||
if (config) {
|
||||
debug("Found configuration %o from %o.", config.filepath, dirname);
|
||||
}
|
||||
}
|
||||
|
||||
if (!ignore) {
|
||||
const ignoreLoc = _path().default.join(loc, BABELIGNORE_FILENAME);
|
||||
|
||||
ignore = readIgnoreConfig(ignoreLoc);
|
||||
|
||||
if (ignore) {
|
||||
debug("Found ignore %o from %o.", ignore.filepath, dirname);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
config,
|
||||
ignore
|
||||
};
|
||||
}
|
||||
|
||||
function findRootConfig(dirname, envName, caller) {
|
||||
const filepath = _path().default.resolve(dirname, BABEL_CONFIG_JS_FILENAME);
|
||||
|
||||
const conf = readConfig(filepath, envName, caller);
|
||||
|
||||
if (conf) {
|
||||
debug("Found root config %o in $o.", BABEL_CONFIG_JS_FILENAME, dirname);
|
||||
}
|
||||
|
||||
return conf;
|
||||
}
|
||||
|
||||
function loadConfig(name, dirname, envName, caller) {
|
||||
const filepath = _resolve().default.sync(name, {
|
||||
basedir: dirname
|
||||
});
|
||||
|
||||
const conf = readConfig(filepath, envName, caller);
|
||||
|
||||
if (!conf) {
|
||||
throw new Error(`Config file ${filepath} contains no configuration data`);
|
||||
}
|
||||
|
||||
debug("Loaded config %o from $o.", name, dirname);
|
||||
return conf;
|
||||
}
|
||||
|
||||
function readConfig(filepath, envName, caller) {
|
||||
return _path().default.extname(filepath) === ".js" ? readConfigJS(filepath, {
|
||||
envName,
|
||||
caller
|
||||
}) : readConfigJSON5(filepath);
|
||||
}
|
||||
|
||||
const LOADING_CONFIGS = new Set();
|
||||
const readConfigJS = (0, _caching.makeStrongCache)((filepath, cache) => {
|
||||
if (!_fs().default.existsSync(filepath)) {
|
||||
cache.forever();
|
||||
return null;
|
||||
}
|
||||
|
||||
if (LOADING_CONFIGS.has(filepath)) {
|
||||
cache.never();
|
||||
debug("Auto-ignoring usage of config %o.", filepath);
|
||||
return {
|
||||
filepath,
|
||||
dirname: _path().default.dirname(filepath),
|
||||
options: {}
|
||||
};
|
||||
}
|
||||
|
||||
let options;
|
||||
|
||||
try {
|
||||
LOADING_CONFIGS.add(filepath);
|
||||
|
||||
const configModule = require(filepath);
|
||||
|
||||
options = configModule && configModule.__esModule ? configModule.default || undefined : configModule;
|
||||
} catch (err) {
|
||||
err.message = `${filepath}: Error while loading config - ${err.message}`;
|
||||
throw err;
|
||||
} finally {
|
||||
LOADING_CONFIGS.delete(filepath);
|
||||
}
|
||||
|
||||
if (typeof options === "function") {
|
||||
options = options((0, _configApi.default)(cache));
|
||||
if (!cache.configured()) throwConfigError();
|
||||
}
|
||||
|
||||
if (!options || typeof options !== "object" || Array.isArray(options)) {
|
||||
throw new Error(`${filepath}: Configuration should be an exported JavaScript object.`);
|
||||
}
|
||||
|
||||
if (typeof options.then === "function") {
|
||||
throw new Error(`You appear to be using an async configuration, ` + `which your current version of Babel does not support. ` + `We may add support for this in the future, ` + `but if you're on the most recent version of @babel/core and still ` + `seeing this error, then you'll need to synchronously return your config.`);
|
||||
}
|
||||
|
||||
return {
|
||||
filepath,
|
||||
dirname: _path().default.dirname(filepath),
|
||||
options
|
||||
};
|
||||
});
|
||||
const packageToBabelConfig = (0, _caching.makeWeakCache)(file => {
|
||||
const babel = file.options["babel"];
|
||||
if (typeof babel === "undefined") return null;
|
||||
|
||||
if (typeof babel !== "object" || Array.isArray(babel) || babel === null) {
|
||||
throw new Error(`${file.filepath}: .babel property must be an object`);
|
||||
}
|
||||
|
||||
return {
|
||||
filepath: file.filepath,
|
||||
dirname: file.dirname,
|
||||
options: babel
|
||||
};
|
||||
});
|
||||
const readConfigJSON5 = (0, _utils.makeStaticFileCache)((filepath, content) => {
|
||||
let options;
|
||||
|
||||
try {
|
||||
options = _json().default.parse(content);
|
||||
} catch (err) {
|
||||
err.message = `${filepath}: Error while parsing config - ${err.message}`;
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!options) throw new Error(`${filepath}: No config detected`);
|
||||
|
||||
if (typeof options !== "object") {
|
||||
throw new Error(`${filepath}: Config returned typeof ${typeof options}`);
|
||||
}
|
||||
|
||||
if (Array.isArray(options)) {
|
||||
throw new Error(`${filepath}: Expected config object but found array`);
|
||||
}
|
||||
|
||||
return {
|
||||
filepath,
|
||||
dirname: _path().default.dirname(filepath),
|
||||
options
|
||||
};
|
||||
});
|
||||
const readIgnoreConfig = (0, _utils.makeStaticFileCache)((filepath, content) => {
|
||||
const ignoreDir = _path().default.dirname(filepath);
|
||||
|
||||
const ignorePatterns = content.split("\n").map(line => line.replace(/#(.*?)$/, "").trim()).filter(line => !!line);
|
||||
|
||||
for (const pattern of ignorePatterns) {
|
||||
if (pattern[0] === "!") {
|
||||
throw new Error(`Negation of file paths is not supported.`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
filepath,
|
||||
dirname: _path().default.dirname(filepath),
|
||||
ignore: ignorePatterns.map(pattern => (0, _patternToRegex.default)(pattern, ignoreDir))
|
||||
};
|
||||
});
|
||||
|
||||
function throwConfigError() {
|
||||
throw new Error(`\
|
||||
Caching was left unconfigured. Babel's plugins, presets, and .babelrc.js files can be configured
|
||||
for various types of caching, using the first param of their handler functions:
|
||||
|
||||
module.exports = function(api) {
|
||||
// The API exposes the following:
|
||||
|
||||
// Cache the returned value forever and don't call this function again.
|
||||
api.cache(true);
|
||||
|
||||
// Don't cache at all. Not recommended because it will be very slow.
|
||||
api.cache(false);
|
||||
|
||||
// Cached based on the value of some function. If this function returns a value different from
|
||||
// a previously-encountered value, the plugins will re-evaluate.
|
||||
var env = api.cache(() => process.env.NODE_ENV);
|
||||
|
||||
// If testing for a specific env, we recommend specifics to avoid instantiating a plugin for
|
||||
// any possible NODE_ENV value that might come up during plugin execution.
|
||||
var isProd = api.cache(() => process.env.NODE_ENV === "production");
|
||||
|
||||
// .cache(fn) will perform a linear search though instances to find the matching plugin based
|
||||
// based on previous instantiated plugins. If you want to recreate the plugin and discard the
|
||||
// previous instance whenever something changes, you may use:
|
||||
var isProd = api.cache.invalidate(() => process.env.NODE_ENV === "production");
|
||||
|
||||
// Note, we also expose the following more-verbose versions of the above examples:
|
||||
api.cache.forever(); // api.cache(true)
|
||||
api.cache.never(); // api.cache(false)
|
||||
api.cache.using(fn); // api.cache(fn)
|
||||
|
||||
// Return the value that will be cached.
|
||||
return { };
|
||||
};`);
|
||||
}
|
@ -0,0 +1,59 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.findConfigUpwards = findConfigUpwards;
|
||||
exports.findPackageData = findPackageData;
|
||||
exports.findRelativeConfig = findRelativeConfig;
|
||||
exports.findRootConfig = findRootConfig;
|
||||
exports.loadConfig = loadConfig;
|
||||
exports.resolvePlugin = resolvePlugin;
|
||||
exports.resolvePreset = resolvePreset;
|
||||
exports.loadPlugin = loadPlugin;
|
||||
exports.loadPreset = loadPreset;
|
||||
|
||||
function findConfigUpwards(rootDir) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPackageData(filepath) {
|
||||
return {
|
||||
filepath,
|
||||
directories: [],
|
||||
pkg: null,
|
||||
isPackage: false
|
||||
};
|
||||
}
|
||||
|
||||
function findRelativeConfig(pkgData, envName, caller) {
|
||||
return {
|
||||
pkg: null,
|
||||
config: null,
|
||||
ignore: null
|
||||
};
|
||||
}
|
||||
|
||||
function findRootConfig(dirname, envName, caller) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function loadConfig(name, dirname, envName, caller) {
|
||||
throw new Error(`Cannot load ${name} relative to ${dirname} in a browser`);
|
||||
}
|
||||
|
||||
function resolvePlugin(name, dirname) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolvePreset(name, dirname) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function loadPlugin(name, dirname) {
|
||||
throw new Error(`Cannot load plugin ${name} relative to ${dirname} in a browser`);
|
||||
}
|
||||
|
||||
function loadPreset(name, dirname) {
|
||||
throw new Error(`Cannot load preset ${name} relative to ${dirname} in a browser`);
|
||||
}
|
@ -0,0 +1,67 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
Object.defineProperty(exports, "findPackageData", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _package.findPackageData;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "findConfigUpwards", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _configuration.findConfigUpwards;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "findRelativeConfig", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _configuration.findRelativeConfig;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "findRootConfig", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _configuration.findRootConfig;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "loadConfig", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _configuration.loadConfig;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "resolvePlugin", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _plugins.resolvePlugin;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "resolvePreset", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _plugins.resolvePreset;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "loadPlugin", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _plugins.loadPlugin;
|
||||
}
|
||||
});
|
||||
Object.defineProperty(exports, "loadPreset", {
|
||||
enumerable: true,
|
||||
get: function () {
|
||||
return _plugins.loadPreset;
|
||||
}
|
||||
});
|
||||
|
||||
var _package = require("./package");
|
||||
|
||||
var _configuration = require("./configuration");
|
||||
|
||||
var _plugins = require("./plugins");
|
||||
|
||||
({});
|