diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index 7469112..8dd4e4c 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -117,7 +117,7 @@ def main(): urls = [url.strip() for url in urls] - download_concurrency_level = 40 + download_concurrency_level = 15 with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: print("Submitting tasks to the ProcessPoolExecutor...")