ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update multithread_script_5.0.py

Browse Source
This commit is contained in:
datechnoman 2023-12-04 10:53:05 +00:00
parent dcf47e4085
commit 6500764baa
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
multithread_script_5.0.py
View File

@ -63,7 +63,7 @@ urls = [url.strip() for url in urls]
batch_size = 100 batch_size = 100
# Define the concurrency level (number of download processes running concurrently) # Define the concurrency level (number of download processes running concurrently)
concurrency_level = 3 concurrency_level = 10
# Split the URLs into batches # Split the URLs into batches
batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)] batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)]