Update warc_wat_url_processor.py

2024-01-20 03:26:03 +00:00 · 2024-01-20 03:26:03 +00:00 · 17b4ce6077
commit 17b4ce6077
parent 99c2f07498
1 changed files with 1 additions and 1 deletions
--- a/warc_wat_url_processor.py
+++ b/warc_wat_url_processor.py
@ -87,7 +87,7 @@ for batch in batches:

    for url in batch:
        # Create the command to download the URL using axel with 3 connections
-        command = f'axel -n 2 {url}'
+        command = f'axel -n 4 {url}'

        # Start the subprocess in the background
        pool.apply_async(subprocess.run, args=(command,), kwds={'shell': True})