From eaa8278db2a3264a70ad1e8813d3df07285cace4 Mon Sep 17 00:00:00 2001 From: datechnoman Date: Mon, 11 Mar 2024 23:47:50 +0000 Subject: [PATCH] Update urls_automated_cdx_processor.py --- urls_automated_cdx_processor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/urls_automated_cdx_processor.py b/urls_automated_cdx_processor.py index bb4ed7a..8cb88aa 100644 --- a/urls_automated_cdx_processor.py +++ b/urls_automated_cdx_processor.py @@ -9,13 +9,13 @@ from urllib.parse import urlparse # Define constants URLS_DIRECTORY = "/opt/cdxfiles/urls" ROOT_DIRECTORY = "/root/urls_files" -CONCURRENCY = 5 -BATCH_SIZE = 5 +CONCURRENCY = 2 +BATCH_SIZE = 2 # Function to run cdxsummary command def run_cdxsummary(file_path, json_filepath): # Construct the cdxsummary command - cdxsummary_command = f"/usr/local/bin/cdxsummary -t 30 --json {file_path}" + cdxsummary_command = f"/usr/local/bin/cdxsummary -t 50 --json {file_path}" try: # Run the cdxsummary command and capture the output