爬虫

爬虫

12306 查询余票

Nodejslopo1983 发表了文章 • 0 个评论 • 1060 次浏览 • 2019-01-20 18:02 • 来自相关话题

以下代码仅供学习交流用,切勿用作其他用途
/data/favorite_name
见附件
index.js
const DATAS = require('../12306/data/favorite_name');
const request = require('request-promise-native');
const STATIONS = DATAS.split('|');
const split_array = (arr, len) => {
let a_len = arr.length;
let result = [];
for (let i = 0; i < a_len; i += len) {
result.push(arr.slice(i, i + len));
}
return result;
}
const STATIONSMap = split_array(STATIONS, 5).reduce((a, b) => {
a.push({
code: b[0],
name: b[1],
station: b[2],
station_py: b[3],
station_pyj: b[4]
});
return a
}, []);
const getStation = (name, key = 'name') => STATIONSMap.filter(e => e.station == name)[0][key]
// console.log(STATIONS.filter(e => e.station == 'HJL')[0].name)
const options = {
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' },
uri: 'https://kyfw.12306.cn/otn/leftTicket/queryZ',
qs: {
'leftTicketDTO.train_date': '2019-01-28',
'leftTicketDTO.from_station': 'SRH',
'leftTicketDTO.to_station': 'DYW',
'purpose_codes': 'ADULT'
},
transform: e => JSON.parse(e).data.result.reduce((a, b) => {
const arr = b.split('|');
a.push({
remark: arr[1], //备注
_id: arr[2], //id
type: arr[3].slice(0, 1), //车次类型
number: arr[3], //车号
station_begin: getStation(arr[4]), //起点站
station_end: getStation(arr[5]), //终点站
station_from: getStation(arr[6]), //出发站
station_to: getStation(arr[7]), //到达站
time_go: arr[8], //出发时间
time_arrival: arr[9], //到达时间
time_take: arr[10], //历时
O: arr[32], //二等座
M: arr[31], //一等座
A9: arr[30], //商务特等座
});
return a
}, [])
};
!(async() => {
try {
const datas = await request(options);
console.log(datas)
} catch (error) {
console.log(error)
}
})() 查看全部

以下代码仅供学习交流用,切勿用作其他用途


/data/favorite_name
见附件
index.js
const DATAS = require('../12306/data/favorite_name');
const request = require('request-promise-native');
const STATIONS = DATAS.split('|');
const split_array = (arr, len) => {
let a_len = arr.length;
let result = [];
for (let i = 0; i < a_len; i += len) {
result.push(arr.slice(i, i + len));
}
return result;
}
const STATIONSMap = split_array(STATIONS, 5).reduce((a, b) => {
a.push({
code: b[0],
name: b[1],
station: b[2],
station_py: b[3],
station_pyj: b[4]
});
return a
}, []);
const getStation = (name, key = 'name') => STATIONSMap.filter(e => e.station == name)[0][key]
// console.log(STATIONS.filter(e => e.station == 'HJL')[0].name)
const options = {
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' },
uri: 'https://kyfw.12306.cn/otn/leftTicket/queryZ',
qs: {
'leftTicketDTO.train_date': '2019-01-28',
'leftTicketDTO.from_station': 'SRH',
'leftTicketDTO.to_station': 'DYW',
'purpose_codes': 'ADULT'
},
transform: e => JSON.parse(e).data.result.reduce((a, b) => {
const arr = b.split('|');
a.push({
remark: arr[1], //备注
_id: arr[2], //id
type: arr[3].slice(0, 1), //车次类型
number: arr[3], //车号
station_begin: getStation(arr[4]), //起点站
station_end: getStation(arr[5]), //终点站
station_from: getStation(arr[6]), //出发站
station_to: getStation(arr[7]), //到达站
time_go: arr[8], //出发时间
time_arrival: arr[9], //到达时间
time_take: arr[10], //历时
O: arr[32], //二等座
M: arr[31], //一等座
A9: arr[30], //商务特等座
});
return a
}, [])
};
!(async() => {
try {
const datas = await request(options);
console.log(datas)
} catch (error) {
console.log(error)
}
})()

node爬取ui中国图片

Nodejslopo1983 发表了文章 • 0 个评论 • 956 次浏览 • 2019-01-17 10:46 • 来自相关话题

以下代码仅供学习交流用,切勿做其他用途
/*************************************************************
*
*- Copyright (c), 2018, lopo qq:64832897
*- FileName : default.js
*- Author : 罗波 Version : 1.0 Date:2018-10-14
*- Descripttion : 图片image
*- Other : www.ui.cn
*- JSVersion : ES6
*- Modules :
* 1.request
* 2.cheerio
* 3.fs
*
*- History :
* <Author> <Date> <Desc> <Modi>
*
**************************************************************/
"use strict"
const request = require('request-promise-native');
const cheerio = require('cheerio');
const fs = require('fs');
const timeout = async(ms) => {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
};
module.exports = {
async getImg() {
for (let j = 0; j < 2; j++) {
let DATA = Object.assign({}, BACKJSON);
let typeArr = ['all', 'main', 'edit', 'gener'];
const options = {
uri: 'http://www.ui.cn/list.html',
transform: (body) => cheerio.load(body),
qs: {
'r': 'main',
'p': j
},
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}
};
let setDATA = async($) => {
let arrs = [];
await $('.imgloadinglater').each((i, e) => {
arrs.push({
url: $(e).parent().attr(),
src: $(e).data('original'),
name: $(e).data('original').split('/').pop(),
title: $(e).parents('.cover').next().find('.title').text().trim(),
})
});
for (const e of arrs) {
await request(e.src).pipe(fs.createWriteStream(`d:/ui/uichina/${e.name}`));
await timeout(1500);
}
console.log(arrs)
};
try {
const data = await request(options);
const json = setDATA(data);
} catch (error) {
console.log(error)
}
await timeout(15000);
}
}
} 查看全部

以下代码仅供学习交流用,切勿做其他用途


/*************************************************************
*
*- Copyright (c), 2018, lopo qq:64832897
*- FileName : default.js
*- Author : 罗波 Version : 1.0 Date:2018-10-14
*- Descripttion : 图片image
*- Other : www.ui.cn
*- JSVersion : ES6
*- Modules :
* 1.request
* 2.cheerio
* 3.fs
*
*- History :
* <Author> <Date> <Desc> <Modi>
*
**************************************************************/
"use strict"
const request = require('request-promise-native');
const cheerio = require('cheerio');
const fs = require('fs');
const timeout = async(ms) => {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
};
module.exports = {
async getImg() {
for (let j = 0; j < 2; j++) {
let DATA = Object.assign({}, BACKJSON);
let typeArr = ['all', 'main', 'edit', 'gener'];
const options = {
uri: 'http://www.ui.cn/list.html',
transform: (body) => cheerio.load(body),
qs: {
'r': 'main',
'p': j
},
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}
};
let setDATA = async($) => {
let arrs = [];
await $('.imgloadinglater').each((i, e) => {
arrs.push({
url: $(e).parent().attr(),
src: $(e).data('original'),
name: $(e).data('original').split('/').pop(),
title: $(e).parents('.cover').next().find('.title').text().trim(),
})
});
for (const e of arrs) {
await request(e.src).pipe(fs.createWriteStream(`d:/ui/uichina/${e.name}`));
await timeout(1500);
}
console.log(arrs)
};
try {
const data = await request(options);
const json = setDATA(data);
} catch (error) {
console.log(error)
}
await timeout(15000);
}
}
}

12306 查询余票

Nodejslopo1983 发表了文章 • 0 个评论 • 1060 次浏览 • 2019-01-20 18:02 • 来自相关话题

以下代码仅供学习交流用,切勿用作其他用途
/data/favorite_name
见附件
index.js
const DATAS = require('../12306/data/favorite_name');
const request = require('request-promise-native');
const STATIONS = DATAS.split('|');
const split_array = (arr, len) => {
let a_len = arr.length;
let result = [];
for (let i = 0; i < a_len; i += len) {
result.push(arr.slice(i, i + len));
}
return result;
}
const STATIONSMap = split_array(STATIONS, 5).reduce((a, b) => {
a.push({
code: b[0],
name: b[1],
station: b[2],
station_py: b[3],
station_pyj: b[4]
});
return a
}, []);
const getStation = (name, key = 'name') => STATIONSMap.filter(e => e.station == name)[0][key]
// console.log(STATIONS.filter(e => e.station == 'HJL')[0].name)
const options = {
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' },
uri: 'https://kyfw.12306.cn/otn/leftTicket/queryZ',
qs: {
'leftTicketDTO.train_date': '2019-01-28',
'leftTicketDTO.from_station': 'SRH',
'leftTicketDTO.to_station': 'DYW',
'purpose_codes': 'ADULT'
},
transform: e => JSON.parse(e).data.result.reduce((a, b) => {
const arr = b.split('|');
a.push({
remark: arr[1], //备注
_id: arr[2], //id
type: arr[3].slice(0, 1), //车次类型
number: arr[3], //车号
station_begin: getStation(arr[4]), //起点站
station_end: getStation(arr[5]), //终点站
station_from: getStation(arr[6]), //出发站
station_to: getStation(arr[7]), //到达站
time_go: arr[8], //出发时间
time_arrival: arr[9], //到达时间
time_take: arr[10], //历时
O: arr[32], //二等座
M: arr[31], //一等座
A9: arr[30], //商务特等座
});
return a
}, [])
};
!(async() => {
try {
const datas = await request(options);
console.log(datas)
} catch (error) {
console.log(error)
}
})() 查看全部

以下代码仅供学习交流用,切勿用作其他用途


/data/favorite_name
见附件
index.js
const DATAS = require('../12306/data/favorite_name');
const request = require('request-promise-native');
const STATIONS = DATAS.split('|');
const split_array = (arr, len) => {
let a_len = arr.length;
let result = [];
for (let i = 0; i < a_len; i += len) {
result.push(arr.slice(i, i + len));
}
return result;
}
const STATIONSMap = split_array(STATIONS, 5).reduce((a, b) => {
a.push({
code: b[0],
name: b[1],
station: b[2],
station_py: b[3],
station_pyj: b[4]
});
return a
}, []);
const getStation = (name, key = 'name') => STATIONSMap.filter(e => e.station == name)[0][key]
// console.log(STATIONS.filter(e => e.station == 'HJL')[0].name)
const options = {
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' },
uri: 'https://kyfw.12306.cn/otn/leftTicket/queryZ',
qs: {
'leftTicketDTO.train_date': '2019-01-28',
'leftTicketDTO.from_station': 'SRH',
'leftTicketDTO.to_station': 'DYW',
'purpose_codes': 'ADULT'
},
transform: e => JSON.parse(e).data.result.reduce((a, b) => {
const arr = b.split('|');
a.push({
remark: arr[1], //备注
_id: arr[2], //id
type: arr[3].slice(0, 1), //车次类型
number: arr[3], //车号
station_begin: getStation(arr[4]), //起点站
station_end: getStation(arr[5]), //终点站
station_from: getStation(arr[6]), //出发站
station_to: getStation(arr[7]), //到达站
time_go: arr[8], //出发时间
time_arrival: arr[9], //到达时间
time_take: arr[10], //历时
O: arr[32], //二等座
M: arr[31], //一等座
A9: arr[30], //商务特等座
});
return a
}, [])
};
!(async() => {
try {
const datas = await request(options);
console.log(datas)
} catch (error) {
console.log(error)
}
})()

node爬取ui中国图片

Nodejslopo1983 发表了文章 • 0 个评论 • 956 次浏览 • 2019-01-17 10:46 • 来自相关话题

以下代码仅供学习交流用,切勿做其他用途
/*************************************************************
*
*- Copyright (c), 2018, lopo qq:64832897
*- FileName : default.js
*- Author : 罗波 Version : 1.0 Date:2018-10-14
*- Descripttion : 图片image
*- Other : www.ui.cn
*- JSVersion : ES6
*- Modules :
* 1.request
* 2.cheerio
* 3.fs
*
*- History :
* <Author> <Date> <Desc> <Modi>
*
**************************************************************/
"use strict"
const request = require('request-promise-native');
const cheerio = require('cheerio');
const fs = require('fs');
const timeout = async(ms) => {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
};
module.exports = {
async getImg() {
for (let j = 0; j < 2; j++) {
let DATA = Object.assign({}, BACKJSON);
let typeArr = ['all', 'main', 'edit', 'gener'];
const options = {
uri: 'http://www.ui.cn/list.html',
transform: (body) => cheerio.load(body),
qs: {
'r': 'main',
'p': j
},
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}
};
let setDATA = async($) => {
let arrs = [];
await $('.imgloadinglater').each((i, e) => {
arrs.push({
url: $(e).parent().attr(),
src: $(e).data('original'),
name: $(e).data('original').split('/').pop(),
title: $(e).parents('.cover').next().find('.title').text().trim(),
})
});
for (const e of arrs) {
await request(e.src).pipe(fs.createWriteStream(`d:/ui/uichina/${e.name}`));
await timeout(1500);
}
console.log(arrs)
};
try {
const data = await request(options);
const json = setDATA(data);
} catch (error) {
console.log(error)
}
await timeout(15000);
}
}
} 查看全部

以下代码仅供学习交流用,切勿做其他用途


/*************************************************************
*
*- Copyright (c), 2018, lopo qq:64832897
*- FileName : default.js
*- Author : 罗波 Version : 1.0 Date:2018-10-14
*- Descripttion : 图片image
*- Other : www.ui.cn
*- JSVersion : ES6
*- Modules :
* 1.request
* 2.cheerio
* 3.fs
*
*- History :
* <Author> <Date> <Desc> <Modi>
*
**************************************************************/
"use strict"
const request = require('request-promise-native');
const cheerio = require('cheerio');
const fs = require('fs');
const timeout = async(ms) => {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
};
module.exports = {
async getImg() {
for (let j = 0; j < 2; j++) {
let DATA = Object.assign({}, BACKJSON);
let typeArr = ['all', 'main', 'edit', 'gener'];
const options = {
uri: 'http://www.ui.cn/list.html',
transform: (body) => cheerio.load(body),
qs: {
'r': 'main',
'p': j
},
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}
};
let setDATA = async($) => {
let arrs = [];
await $('.imgloadinglater').each((i, e) => {
arrs.push({
url: $(e).parent().attr(),
src: $(e).data('original'),
name: $(e).data('original').split('/').pop(),
title: $(e).parents('.cover').next().find('.title').text().trim(),
})
});
for (const e of arrs) {
await request(e.src).pipe(fs.createWriteStream(`d:/ui/uichina/${e.name}`));
await timeout(1500);
}
console.log(arrs)
};
try {
const data = await request(options);
const json = setDATA(data);
} catch (error) {
console.log(error)
}
await timeout(15000);
}
}
}