ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removing the tar process and now deleting cdx.json files older than 24 hours

Browse Source
This commit is contained in:
datechnoman 2024-01-09 05:49:56 +00:00
parent 51c127601a
commit 08dc0c0bf6
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pastebin_automated_cdx_processor.py
View File

@ -2,7 +2,6 @@
import os
import subprocess
import json
import tarfile # Use tarfile for creating tar archives
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse
@ -184,11 +183,9 @@ def main():
for filename in os.listdir(PASTEBIN_DIRECTORY):
file_path = os.path.join(PASTEBIN_DIRECTORY, filename)
if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_24_hours.timestamp():
# Appending to the existing tar file
with tarfile.open(os.path.join(PASTEBIN_DIRECTORY, 'pastebin_cdxfiles_archive.tar'), 'a') as tar:
tar.add(file_path, arcname=filename)
# Delete the JSON file if it's older than 24 hours
os.remove(file_path)
print(f"Added '{filename}' to 'pastebin_cdxfiles_archive.tar' and removed the JSON file.")
print(f"Deleted '{filename}' as it is older than 24 hours.")
run_ia_command() # Run IA command after processing older files
@ -219,4 +216,4 @@ def main():
# Entry point
if __name__ == "__main__":
main()
main()