ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removing the tar process and now deleting cdx.json files older than 24 hours

Browse Source
This commit is contained in:
datechnoman 2024-01-09 05:49:56 +00:00
parent 51c127601a
commit 08dc0c0bf6
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pastebin_automated_cdx_processor.py
View File

@ -2,7 +2,6 @@
import os import os
import subprocess import subprocess
import json import json
import tarfile # Use tarfile for creating tar archives
from datetime import datetime, timedelta from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse from urllib.parse import urlparse
@ -184,11 +183,9 @@ def main():
for filename in os.listdir(PASTEBIN_DIRECTORY): for filename in os.listdir(PASTEBIN_DIRECTORY):
file_path = os.path.join(PASTEBIN_DIRECTORY, filename) file_path = os.path.join(PASTEBIN_DIRECTORY, filename)
if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_24_hours.timestamp(): if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_24_hours.timestamp():
# Appending to the existing tar file # Delete the JSON file if it's older than 24 hours
with tarfile.open(os.path.join(PASTEBIN_DIRECTORY, 'pastebin_cdxfiles_archive.tar'), 'a') as tar:
tar.add(file_path, arcname=filename)
os.remove(file_path) os.remove(file_path)
print(f"Added '{filename}' to 'pastebin_cdxfiles_archive.tar' and removed the JSON file.") print(f"Deleted '{filename}' as it is older than 24 hours.")
run_ia_command() # Run IA command after processing older files run_ia_command() # Run IA command after processing older files