ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update concurrency

Browse Source
This commit is contained in:
datechnoman 2024-01-26 08:01:39 +00:00
parent f76dbb13fd
commit 8244b9241a
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warc_wat_url_processor.py
View File

@ -78,7 +78,7 @@ def process_file(file_path):
def download_and_process_file(url): def download_and_process_file(url):
try: try:
command = f'axel -n 3 {url}' command = f'axel -n 4 {url}'
result = subprocess.run(command, shell=True, check=True) result = subprocess.run(command, shell=True, check=True)
if result.returncode == 0: if result.returncode == 0:
file_path = os.path.join(os.getcwd(), os.path.basename(url)) file_path = os.path.join(os.getcwd(), os.path.basename(url))