ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update concurrency of downloads

Browse Source
This commit is contained in:
datechnoman 2024-01-08 08:27:15 +00:00
parent 2e938624f9
commit 61f5a4ddbf
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediafire_automated_cdx_processor.py
View File

@ -5,7 +5,7 @@ from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
MEDIAFIRE_DIRECTORY = "/opt/MediaFire" MEDIAFIRE_DIRECTORY = "/opt/MediaFire"
CONCURRENCY = 4 # Set the desired concurrency for downloading multiple files CONCURRENCY = 6 # Set the desired concurrency for downloading multiple files
BATCH_SIZE = 6 # Set the batch size for processing BATCH_SIZE = 6 # Set the batch size for processing
def run_cdxsummary(file_path, json_filepath): def run_cdxsummary(file_path, json_filepath):