ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated cdx file retention to 2 days

Browse Source
This commit is contained in:
datechnoman 2024-01-09 21:14:23 +00:00
parent 8e1912cb9b
commit 1e44917569
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pastebin_automated_cdx_processor.py
View File

@ -179,13 +179,13 @@ def main():
directory_output_file.write(f"{filename}\n") directory_output_file.write(f"{filename}\n")
# Process older files in /opt/cdxfiles/pastebin # Process older files in /opt/cdxfiles/pastebin
older_than_24_hours = datetime.now() - timedelta(days=1) older_than_48_hours = datetime.now() - timedelta(days=2)
for filename in os.listdir(PASTEBIN_DIRECTORY): for filename in os.listdir(PASTEBIN_DIRECTORY):
file_path = os.path.join(PASTEBIN_DIRECTORY, filename) file_path = os.path.join(PASTEBIN_DIRECTORY, filename)
if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_24_hours.timestamp(): if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_48_hours.timestamp():
# Delete the JSON file if it's older than 24 hours # Delete the JSON file if it's older than 48 hours
os.remove(file_path) os.remove(file_path)
print(f"Deleted '{filename}' as it is older than 24 hours.") print(f"Deleted '{filename}' as it is older than 48 hours.")
run_ia_command() # Run IA command after processing older files run_ia_command() # Run IA command after processing older files