ArchiveTeam/Migrated_Keyword_URL_Extractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stream to RAM and process in RAM

Browse Source
This commit is contained in:
datechnoman 2024-06-22 13:16:31 +00:00
parent 9e6e5190d3
commit 74d32a7702
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
url_extractor.py
View File

@ -19,15 +19,18 @@ def process_file(file, directory_path, output_file_path, keyword, counter_lock,
print(f"Skipping {file_path}. Unsupported file extension.") print(f"Skipping {file_path}. Unsupported file extension.")
return return
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, text=True) # Load the entire file content into memory
try:
file_content = subprocess.check_output(command, shell=True, text=True)
except subprocess.CalledProcessError as e:
print(f"Error processing {file_path}: {e}")
return
# Stream the output line by line and append to the output file # Stream the output line by line and append to the output file
with open(output_file_path, "a") as output_file: with open(output_file_path, "a") as output_file:
for line in process.stdout: for line in file_content.splitlines():
if keyword in line: if keyword in line:
output_file.write(line) output_file.write(line + '\n')
process.wait() # Wait for the process to finish
# Update the processed files count # Update the processed files count
with counter_lock: with counter_lock:
@ -81,4 +84,4 @@ with ThreadPoolExecutor(max_workers=num_concurrent_instances) as executor:
for future in futures: for future in futures:
future.result() future.result()
print(f"\nAll files processed. Matching lines appended to {output_file_path}") print(f"\nAll files processed. Matching lines appended to {output_file_path}")