From 35a49bbd1b9c131a3a2db734f1351837022930a5 Mon Sep 17 00:00:00 2001
From: lyg <1543117173@qq.com>
Date: 星期二, 11 六月 2024 22:21:11 +0800
Subject: [PATCH] 图书下载修改多线程并发下载分配策略，统一由主线程分配给子线程下载图书信息

---
 src/main.mjs |   46 +++++++++++++++++++++++++++++++++-------------
 1 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/src/main.mjs b/src/main.mjs
index a53edc5..0e0a42f 100644
--- a/src/main.mjs
+++ b/src/main.mjs
@@ -5,7 +5,7 @@
 import axios from "axios";
 import * as fs from "fs";
 import path from "path";
-import { Worker, isMainThread, parentPort, workerData } from 'worker_threads';
+import { Worker, isMainThread, parentPort, workerData, threadId } from 'worker_threads';
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { resolve } from "path";
 
@@ -19,7 +19,7 @@
   if (!fs.existsSync('./logs')) {
     fs.mkdirSync('./logs', { recursive: true });
   }
-  logFile = fs.createWriteStream(`./logs/logs-${config.startRow}-${config.endRow}.log`, { flags: 'a', encoding: 'utf8' });
+  logFile = fs.createWriteStream(`./logs/logs-${config.startRow}-${config.endRow}-thread${threadId}.log`, { flags: 'a', encoding: 'utf8' });
   console.log = function (...text) {
     text = `${new Date().toLocaleString()} ${text.join(' ') ?? ''}`;
     _log(text);
@@ -313,9 +313,29 @@
   return alreadyDownloadedBooks.includes(id);
 }
 
+function nextBook() {
+  return new Promise(resolve => {
+    const cb = (message) => {
+      if (message.type === 'book') {
+        resolve(message.data);
+        parentPort.removeListener('message', cb);
+      }
+    };
+    parentPort.on('message', cb);
+    parentPort.postMessage({ type: 'get-book', threadId });
+
+  });
+}
+
 async function downloadBooks(books) {
   driver = await createDriver();
-  for (const book of books) {
+
+  for (; ;) {
+    const book = await nextBook();
+    if (!book) {
+      break;
+    }
+    books.push(book);
     if (config.endOfTime && Date.now() - startTime > 1000 * 60 * config.endOfTime) {
       // 瀹氭椂閫�鍑�
       break;
@@ -388,7 +408,7 @@
   }
 
   const buffer = xlsx.build([{ name: "Sheet1", data }]);
-  fs.writeFile("./銆愮浜屾壒浜屾澶勭悊鍚庛�戜氦浠樻竻鍗�.xlsx", buffer, (err) => { });
+  fs.writeFileSync("./銆愮浜屾壒浜屾澶勭悊鍚庛�戜氦浠樻竻鍗�.xlsx", buffer, (err) => { });
   console.log("淇濆瓨瀹屾垚: ./銆愮浜屾壒浜屾澶勭悊鍚庛�戜氦浠樻竻鍗�.xlsx");
 }
 
@@ -438,7 +458,7 @@
 
 function main() {
   initLogger();
-  const books = getBooksFromExcel(config.startRow, config.endRow);
+  const books = [];
   downloadBooks(books)
     .then(() => {
       console.log(`鍏ㄩ儴瀹屾垚锛屽叡涓嬭浇${bookCount}鏈紝鎴愬姛涓嬭浇${successCount}鏈紝璺宠繃${skipCount}鏈紝澶辫触${bookCount - skipCount - successCount}鏈紝鑰楁椂锛� ${msFormat(Date.now() - startTime)}銆俙);
@@ -461,13 +481,13 @@
 if (isMainThread) {
   initLogger();
   const alreadyDownloadedBooks = getAlreadyDownloadedBooks();
-  console.log(`绾跨▼鏁帮細${config.threadSize}, 寮�濮嬭锛�${config.startRow}, 缁撴潫琛岋細${config.endRow}`);
-  let startRow = config.startRow;
-  let endRow = config.endRow;
+  const { startRow, endRow, threadSize } = config;
+  console.log(`绾跨▼鏁帮細${threadSize}, 寮�濮嬭锛�${startRow}, 缁撴潫琛岋細${endRow}`);
   let finishCnt = 0;
   const finishBooks = [];
-  const threadSize = config.threadSize;
-  const thBookSize = (endRow - startRow) / threadSize
+  const thBookSize = (endRow - startRow) / threadSize;
+  const books = getBooksFromExcel(startRow, endRow);
+
   for (let sr = startRow; sr < endRow; sr += thBookSize) {
     let er = sr + thBookSize;
     if (er > endRow) {
@@ -478,15 +498,15 @@
       if (message.type === 'books') {
         finishBooks.push(...message.data);
         finishCnt++;
-        if (finishCnt >= config.threadSize) {
+        if (finishCnt >= threadSize) {
           saveBooks(finishBooks);
         }
+      } else if (message.type === 'get-book') {
+        worker.postMessage({ type: "book", data: books.shift() });
       }
     });
   }
 } else {
-  config.startRow = workerData.startRow;
-  config.endRow = workerData.endRow;
   alreadyDownloadedBooks = workerData.alreadyDownloadedBooks;
   main();
 }

--
Gitblit v1.9.1