ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update due to bug in folder location

Browse Source
This commit is contained in:
datechnoman 2024-03-12 04:19:43 +00:00
parent 7b4651b07e
commit 27a2b56936
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
urls_automated_cdx_processor.py
View File

@ -8,7 +8,7 @@ from urllib.parse import urlparse
# Define constants
URLS_DIRECTORY = "/opt/cdxfiles/urls"
URLS_FILES_DIRECTORY = "/opt/cdxfiles/urls_files"
URLS_FILES_DIRECTORY = "/opt/cdxfiles/urls_tophosts"
ROOT_DIRECTORY = "/root/urls_files"
CONCURRENCY = 10
BATCH_SIZE = 10
@ -209,7 +209,7 @@ def main():
if filename.endswith(".cdx.json"):
directory_output_file.write(f"{filename}\n")
# Process older files in /opt/cdxfiles/urls, URLS_FILES_DIRECTORY, and the ROOT_DIRECTORY
# Process older files in /opt/cdxfiles/urls and urls_tophosts
older_than_48_hours = datetime.now() - timedelta(days=2)
directories_to_check = [URLS_DIRECTORY, URLS_FILES_DIRECTORY]