From 7c157c5a48027023cd6ed198deb94c8ae54da27c Mon Sep 17 00:00:00 2001 From: datechnoman Date: Sun, 31 Mar 2024 11:45:33 +0000 Subject: [PATCH] Update warc_wat_url_processor.py --- warc_wat_url_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index 8dd4e4c..5df9165 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -98,7 +98,7 @@ def process_file(file_path): def download_and_process_file(url): try: - command = f'axel -n 3 {url}' + command = f'axel -q -n 1 {url}' result = subprocess.run(command, shell=True, check=True) if result.returncode == 0: file_path = os.path.join(os.getcwd(), os.path.basename(url)) @@ -117,7 +117,7 @@ def main(): urls = [url.strip() for url in urls] - download_concurrency_level = 15 + download_concurrency_level = 7 with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: print("Submitting tasks to the ProcessPoolExecutor...")