Update warc_wat_url_processor.py

2024-02-08 00:39:09 +00:00 · 2024-02-08 00:39:09 +00:00 · 3151fce353
commit 3151fce353
parent 486a68a796
1 changed files with 1 additions and 1 deletions
--- a/warc_wat_url_processor.py
+++ b/warc_wat_url_processor.py
@ -117,7 +117,7 @@ def main():

    urls = [url.strip() for url in urls]

-    download_concurrency_level = 40
+    download_concurrency_level = 15

    with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor:
        print("Submitting tasks to the ProcessPoolExecutor...")