68 lines
2.5 KiB
Python
68 lines
2.5 KiB
Python
|
import os
|
||
|
|
import re
|
||
|
|
from concurrent.futures import ThreadPoolExecutor
|
||
|
|
|
||
|
|
def filter_and_extract_urls(input_file_path, output_file_path):
|
||
|
|
# Check if the input file exists
|
||
|
|
if not os.path.exists(input_file_path):
|
||
|
|
print(f"The file {input_file_path} does not exist.")
|
||
|
|
return
|
||
|
|
|
||
|
|
# Regular expression pattern for URLs
|
||
|
|
url_pattern = re.compile(r'https?://[^\s/$.?#].[^\s]*', re.IGNORECASE)
|
||
|
|
|
||
|
|
unique_urls = set()
|
||
|
|
|
||
|
|
with open(input_file_path, 'r') as input_file:
|
||
|
|
for line in input_file:
|
||
|
|
# Find all URLs in the line
|
||
|
|
urls = url_pattern.findall(line)
|
||
|
|
for url in urls:
|
||
|
|
# Extract the desired format
|
||
|
|
formatted_url = url.split('/')[2]
|
||
|
|
# Add the formatted URL to the set of unique URLs
|
||
|
|
unique_urls.add(formatted_url)
|
||
|
|
|
||
|
|
# Write unique URLs to the output file
|
||
|
|
with open(output_file_path, 'w') as output_file:
|
||
|
|
for unique_url in unique_urls:
|
||
|
|
output_file.write(unique_url + '\n')
|
||
|
|
|
||
|
|
print(f"Extracted and filtered URLs in {input_file_path}. Unique filtered URLs saved to {output_file_path}.")
|
||
|
|
|
||
|
|
def process_directory(directory_path, output_directory, concurrency):
|
||
|
|
# Get a list of all text files in the specified directory
|
||
|
|
text_files = [f for f in os.listdir(directory_path) if f.endswith('.txt')]
|
||
|
|
|
||
|
|
if not text_files:
|
||
|
|
print(f"No text files found in the directory {directory_path}.")
|
||
|
|
return
|
||
|
|
|
||
|
|
# Process text files concurrently with the specified concurrency level
|
||
|
|
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||
|
|
# Submit each text file for processing
|
||
|
|
futures = [executor.submit(
|
||
|
|
filter_and_extract_urls,
|
||
|
|
os.path.join(directory_path, text_file),
|
||
|
|
os.path.join(output_directory, f"filtered_{text_file}")
|
||
|
|
) for text_file in text_files]
|
||
|
|
|
||
|
|
# Wait for all tasks to complete
|
||
|
|
for future in futures:
|
||
|
|
future.result()
|
||
|
|
|
||
|
|
print("Concurrent processing of text files completed.")
|
||
|
|
|
||
|
|
# Get user input for the directory
|
||
|
|
directory_path = input("Enter the path to the directory containing text files: ")
|
||
|
|
|
||
|
|
# Get user input for the output directory
|
||
|
|
output_directory = input("Enter the path to the directory for saving filtered URLs: ")
|
||
|
|
|
||
|
|
# Get user input for the concurrency level
|
||
|
|
concurrency = int(input("Enter the level of concurrency (e.g., 2, 4, 8): "))
|
||
|
|
|
||
|
|
# Call the function to process all text files in the directory concurrently
|
||
|
|
process_directory(directory_path, output_directory, concurrency)
|
||
|
|
|