diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index 8dd4e4c..5df9165 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -98,7 +98,7 @@ def process_file(file_path): def download_and_process_file(url): try: - command = f'axel -n 3 {url}' + command = f'axel -q -n 1 {url}' result = subprocess.run(command, shell=True, check=True) if result.returncode == 0: file_path = os.path.join(os.getcwd(), os.path.basename(url)) @@ -117,7 +117,7 @@ def main(): urls = [url.strip() for url in urls] - download_concurrency_level = 15 + download_concurrency_level = 7 with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: print("Submitting tasks to the ProcessPoolExecutor...")