ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated code to move files to /opt/cdxfiles/mediafire

Browse Source
This commit is contained in:
datechnoman 2024-01-08 11:09:35 +00:00
parent 42be737ee6
commit 85319ba390
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mediafire_automated_cdx_processor.py
View File

@ -7,8 +7,8 @@ from urllib.parse import urlparse
MEDIAFIRE_DIRECTORY = "/opt/cdxfiles/mediafire"
ROOT_DIRECTORY = "/root/mediafire_files"
CONCURRENCY = 6
BATCH_SIZE = 6
CONCURRENCY = 10
BATCH_SIZE = 10
def run_cdxsummary(file_path, json_filepath):
# Construct the cdxsummary command
@ -42,6 +42,11 @@ def run_cdxsummary(file_path, json_filepath):
print(f"Created JSON file for '{file_path}': '{json_filepath}'")
# Move the JSON file to /opt/cdxfiles/mediafire
destination_path = os.path.join(MEDIAFIRE_DIRECTORY, os.path.basename(json_filepath))
os.rename(json_filepath, destination_path)
print(f"Moved '{os.path.basename(json_filepath)}' to '{MEDIAFIRE_DIRECTORY}'.")
# Delete the .cdx.gz file
os.remove(file_path)
print(f"Deleted '{file_path}' after processing.")