Update warc_wat_url_processor.py

warc_wat_url_processor.py

@@ -98,7 +98,7 @@ def process_file(file_path):
 
 def download_and_process_file(url):
     try:
-        command = f'axel -n 3 {url}'
+        command = f'axel -q -n 1 {url}'
         result = subprocess.run(command, shell=True, check=True)
         if result.returncode == 0:
             file_path = os.path.join(os.getcwd(), os.path.basename(url))
@@ -117,7 +117,7 @@ def main():
 
     urls = [url.strip() for url in urls]
 
-    download_concurrency_level = 15
+    download_concurrency_level = 7
 
     with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor:
         print("Submitting tasks to the ProcessPoolExecutor...")