From 3da255f97ecf2afbbd569a634c87f7bedd5b4c4b Mon Sep 17 00:00:00 2001
From: lyg <1543117173@qq.com>
Date: 星期六, 08 六月 2024 16:39:03 +0800
Subject: [PATCH] 修改异常处理
---
src/main.mjs | 75 +++++++++++++++++++++++++++++--------
1 files changed, 58 insertions(+), 17 deletions(-)
diff --git a/src/main.mjs b/src/main.mjs
index 7b3a874..a4f743e 100644
--- a/src/main.mjs
+++ b/src/main.mjs
@@ -6,9 +6,12 @@
import * as fs from "fs";
import { HttpsProxyAgent } from "https-proxy-agent";
+/*-------------璇诲彇閰嶇疆---------------*/
+let config = JSON.parse(fs.readFileSync('./config.json'));
+
/* ------------鏃ュ織-------------- */
const _log = console.log;
-const logFile = fs.createWriteStream('./logs.log');
+const logFile = fs.createWriteStream('./logs.log', { flags: 'a', encoding: 'utf8' });
console.log = function (text) {
text = `${new Date().toLocaleString()} ${text ?? ''}`;
_log(text);
@@ -61,6 +64,12 @@
*/
async function createDriver() {
const opts = new ChromeOptions();
+ if (config.headless) {
+ opts.addArguments("--headless");//寮�鍚棤澶存ā寮�
+ }
+ if (config.disableGpu) {
+ opts.addArguments("--disable-gpu");//绂佹gpu娓叉煋
+ }
opts.addArguments("--ignore-ssl-error"); // 蹇界暐ssl閿欒
opts.addArguments("--no-sandbox"); // 绂佺敤娌欑洅妯″紡
opts.addArguments("blink-settings=imagesEnabled=false"); //绂佺敤鍥剧墖鍔犺浇
@@ -84,7 +93,6 @@
return text.replace(/[^\u4e00-\u9fa5\w \d]/g, "");
}
-const driver = await createDriver();
async function sleep(ms) {
return new Promise((resolve) => {
@@ -223,14 +231,23 @@
async function downloadFile(book, url) {
console.log(`涓嬭浇鏂囦欢: ${url}`);
+ const ext = url.split(".").pop();
+ const filepath = `./downloads/${book.id} ${book.isbn}.${ext}`;
+ if (fs.existsSync(filepath)) {
+ book.state = `涓嬭浇瀹屾垚`;
+ book.format = ext;
+ book.file = filepath;
+ book.url = url;
+ console.log(`涓嬭浇瀹屾垚锛�${filepath}`);
+ return;
+ }
await retry(() => {
return new Promise((resolve, reject) => myAxios
.get(url, { responseType: "stream" })
.then((response) => {
const stream = response.data;
- const ext = url.split(".").pop();
- const filepath = `./downloads/${book.id} ${book.isbn}.${ext}`;
- stream.pipe(fs.createWriteStream(filepath));
+ const out = fs.createWriteStream(filepath);
+ stream.pipe(out);
stream.on("end", () => {
book.state = `涓嬭浇瀹屾垚`;
book.format = ext;
@@ -238,6 +255,19 @@
book.url = url;
console.log(`涓嬭浇瀹屾垚锛�${filepath}`);
resolve(true);
+ });
+ stream.on("error", (err) => {
+ console.error(err);
+ book.state = "涓嬭浇澶辫触";
+ book.url = url;
+ console.log(`涓嬭浇澶辫触: ${book.id} ${book.title}`);
+ reject(false);
+ try {
+ out.close();
+ fs.unlink(filepath,(e)=>console.error(e));
+ } catch (e) {
+ console.error(e);
+ }
});
})
.catch((e) => {
@@ -247,16 +277,19 @@
console.log(`涓嬭浇澶辫触: ${book.id} ${book.title}`);
reject(false);
}));
+ }).catch(e => {
+ return false
});
}
async function downloadBooks(books) {
for (const book of books) {
- if (book.state && (book.state === "娌℃湁鎼滅储缁撴灉" || book.state === "娌℃湁pdf鎴杢ext鏂囦欢")) {
+ bookCount++;
+ if (book.state && (book.state === "娌℃湁鎼滅储缁撴灉" || book.state === "娌℃湁pdf鎴杢ext鏂囦欢" || book.state === "涓嬭浇瀹屾垚")) {
// 璺宠繃娌℃湁鎼滅储缁撴灉鎴栨病鏈塸df鎴杢ext鏂囦欢鐨勪功绫�
+ skipCount++;
continue;
}
- bookCount++;
console.log(`寮�濮嬩笅杞�: ${book.id} ${book.title}`);
// 鎵撳紑鎼滅储椤甸潰骞舵悳绱�
if (!await openSearchPage(book)) {
@@ -286,21 +319,21 @@
// 绛夊緟涓�娈垫椂闂村啀涓嬭浇
await sleep(getRandomNumber(3000, 10000));
// 涓嬭浇鏂囦欢
- await downloadFile(book, url);
- console.log(`涓嬭浇瀹屾垚: ${book.id} ${book.title}`);
+ try {
+ await downloadFile(book, url);
+ console.log(`涓嬭浇瀹屾垚: ${book.id} ${book.title}`);
+ }catch(e){}
successCount++;
// 绛変竴娈垫椂闂村啀涓嬩竴涓�
sleep(getRandomNumber(3000, 10000));
}
- await driver.close();
- await driver.quit();
}
function saveBooks(books) {
console.log("淇濆瓨涓嬭浇鐘舵�佹暟鎹�");
const workSheets = xlsx.parse("銆愮浜屾壒浜屾澶勭悊鍚庛�戜氦浠樻竻鍗�.xlsx");
const sheet = workSheets[0];
- const data = sheet.data.slice(2);
+ const data = sheet.data;
for (const book of books) {
const index = data.findIndex((row) => row[0] === book.id);
if (index > -1) {
@@ -346,17 +379,25 @@
let successCount = 0;
// 鍥句功鏁伴噺
let bookCount = 0;
-
+// 璺宠繃鐨勬暟閲忥紝宸茬粡涓嬭浇杩囨垨娌℃湁鎼滅储鍒扮殑鏁伴噺
+let skipCount = 0;
+const driver = await createDriver();
function main() {
- const range = JSON.parse(fs.readFileSync('./config.json'));
- const books = getBooksFromExcel(range.startRow, range.endRow);
+ const books = getBooksFromExcel(config.startRow, config.endRow);
downloadBooks(books)
.then(() => {
- console.log(`鍏ㄩ儴瀹屾垚锛屽叡涓嬭浇${bookCount}鏈紝鎴愬姛涓嬭浇${successCount}鏈紝澶辫触${bookCount - successCount}鏈紝鑰楁椂锛� ${msFormat(Date.now() - startTime)}銆俙);
+ console.log(`鍏ㄩ儴瀹屾垚锛屽叡涓嬭浇${bookCount}鏈紝鎴愬姛涓嬭浇${successCount}鏈紝璺宠繃${skipCount}鏈紝澶辫触${bookCount - skipCount - successCount}鏈紝鑰楁椂锛� ${msFormat(Date.now() - startTime)}銆俙);
})
- .finally(() => {
+ .catch(e => {
+ console.error(e);
+ })
+ .finally(async () => {
saveBooks(books);
logFile.close();
+ try {
+ await driver.close();
+ await driver.quit();
+ }catch(e){}
});
}
--
Gitblit v1.9.1