From 3151fce353a4913daf8182a547fbd6a3f99a34b7 Mon Sep 17 00:00:00 2001 From: datechnoman Date: Thu, 8 Feb 2024 00:39:09 +0000 Subject: [PATCH] Update warc_wat_url_processor.py --- warc_wat_url_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index 7469112..8dd4e4c 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -117,7 +117,7 @@ def main(): urls = [url.strip() for url in urls] - download_concurrency_level = 40 + download_concurrency_level = 15 with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: print("Submitting tasks to the ProcessPoolExecutor...")