Stream to RAM and process in RAM

This commit is contained in:
datechnoman 2024-06-22 13:16:31 +00:00
parent 9e6e5190d3
commit 74d32a7702

View File

@ -19,15 +19,18 @@ def process_file(file, directory_path, output_file_path, keyword, counter_lock,
print(f"Skipping {file_path}. Unsupported file extension.")
return
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, text=True)
# Load the entire file content into memory
try:
file_content = subprocess.check_output(command, shell=True, text=True)
except subprocess.CalledProcessError as e:
print(f"Error processing {file_path}: {e}")
return
# Stream the output line by line and append to the output file
with open(output_file_path, "a") as output_file:
for line in process.stdout:
for line in file_content.splitlines():
if keyword in line:
output_file.write(line)
process.wait() # Wait for the process to finish
output_file.write(line + '\n')
# Update the processed files count
with counter_lock: