ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented subprocess for running multiple json extractions at once

Browse Source
This commit is contained in:
datechnoman 2023-12-27 08:46:36 +00:00
parent 8496391064
commit 182c58f1ce
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mediafire_cdxprocessor.py
View File

@ -1,8 +1,8 @@
import os
import subprocess
from concurrent.futures import ThreadPoolExecutor
import json
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor
MEDIAFIRE_DIRECTORY = "/opt/MediaFire"
CONCURRENCY = 4 # Set the desired concurrency for downloading multiple files
@ -62,13 +62,14 @@ def process_batch(urls, start_index, end_index):
downloaded_files = download_files(batch_urls)
# Move files and run cdxsummary for each downloaded file
for file_path in downloaded_files:
# Construct file paths
file_path = os.path.join(os.getcwd(), file_path)
json_filepath = os.path.join(MEDIAFIRE_DIRECTORY, file_path.replace(".cdx.gz", ".cdx.json"))
with ThreadPoolExecutor(max_workers=CONCURRENCY) as executor:
for file_path in downloaded_files:
# Construct file paths
file_path = os.path.join(os.getcwd(), file_path)
json_filepath = os.path.join(MEDIAFIRE_DIRECTORY, file_path.replace(".cdx.gz", ".cdx.json"))
# Run cdxsummary and delete .cdx.gz file
run_cdxsummary(file_path, json_filepath)
# Run cdxsummary and delete .cdx.gz file
executor.submit(run_cdxsummary, file_path, json_filepath)
def run_ia_command():
# Get the current date formatted as YYYY-MM-DD
@ -133,4 +134,3 @@ def main():
if __name__ == "__main__":
main()