From 08dc0c0bf66b32c335037beaf6ea601328cfcf45 Mon Sep 17 00:00:00 2001 From: datechnoman Date: Tue, 9 Jan 2024 05:49:56 +0000 Subject: [PATCH] Removing the tar process and now deleting cdx.json files older than 24 hours --- pastebin_automated_cdx_processor.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pastebin_automated_cdx_processor.py b/pastebin_automated_cdx_processor.py index 81293d1..5e94048 100644 --- a/pastebin_automated_cdx_processor.py +++ b/pastebin_automated_cdx_processor.py @@ -2,7 +2,6 @@ import os import subprocess import json -import tarfile # Use tarfile for creating tar archives from datetime import datetime, timedelta from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlparse @@ -184,11 +183,9 @@ def main(): for filename in os.listdir(PASTEBIN_DIRECTORY): file_path = os.path.join(PASTEBIN_DIRECTORY, filename) if filename.endswith(".cdx.json") and os.path.getmtime(file_path) < older_than_24_hours.timestamp(): - # Appending to the existing tar file - with tarfile.open(os.path.join(PASTEBIN_DIRECTORY, 'pastebin_cdxfiles_archive.tar'), 'a') as tar: - tar.add(file_path, arcname=filename) + # Delete the JSON file if it's older than 24 hours os.remove(file_path) - print(f"Added '{filename}' to 'pastebin_cdxfiles_archive.tar' and removed the JSON file.") + print(f"Deleted '{filename}' as it is older than 24 hours.") run_ia_command() # Run IA command after processing older files @@ -219,4 +216,4 @@ def main(): # Entry point if __name__ == "__main__": - main() \ No newline at end of file + main()