ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add in checking for new files once list is depleted

Browse Source
This commit is contained in:
datechnoman 2023-12-21 01:58:55 +00:00
parent 0aad853966
commit 50e89b9de2
1 changed files with 54 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
commoncrawl_local_to_share_move.ps1
View File

@ -7,6 +7,16 @@ $assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"
# Specify the path to winscp.exe # Specify the path to winscp.exe
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe" $winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"
# Specify the local directory path
$localPath = "E:\CommonCrawl"
# Specify the remote directory path
$remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-14_March_April_2023"
while ($true) {
$session = $null
try {
# Create a session instance # Create a session instance
$sessionType = [WinSCP.Session].FullName $sessionType = [WinSCP.Session].FullName
$session = New-Object $sessionType $session = New-Object $sessionType
@ -22,20 +32,12 @@ $sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!" $sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY" $sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
try
{
# Set the ExecutablePath property # Set the ExecutablePath property
$session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null) $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
# Connect # Connect
$session.Open($sessionOptions) $session.Open($sessionOptions)
# Specify the local directory path
$localPath = "E:\CommonCrawl"
# Specify the remote directory path
$remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-23_May_June_2023"
# Get a list of local files matching the pattern "_urls.txt.gz" # Get a list of local files matching the pattern "_urls.txt.gz"
$localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse $localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
@ -43,15 +45,12 @@ try
$localFiles = $localFiles | Get-Random -Count $localFiles.Count $localFiles = $localFiles | Get-Random -Count $localFiles.Count
# Check if there are any files to upload # Check if there are any files to upload
if ($localFiles.Count -eq 0) if ($localFiles.Count -eq 0) {
{
Write-Host "No files found to upload." Write-Host "No files found to upload."
} break # Break out of the loop if no files are found
else } else {
{
# Iterate through each file and upload/delete it # Iterate through each file and upload/delete it
foreach ($fileInfo in $localFiles) foreach ($fileInfo in $localFiles) {
{
# Construct the full local file path # Construct the full local file path
$localFilePath = $fileInfo.FullName $localFilePath = $fileInfo.FullName
@ -62,21 +61,20 @@ try
$transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False) $transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False)
# Check if the upload was successful # Check if the upload was successful
if ($transferResult.IsSuccess) if ($transferResult.IsSuccess) {
{
Write-Host "Upload successful: $localFilePath to $remoteFilePath" Write-Host "Upload successful: $localFilePath to $remoteFilePath"
# Attempt to delete the local file # Attempt to delete the local file
Remove-Item -Path $localFilePath -Force Remove-Item -Path $localFilePath -Force
} } else {
else
{
Write-Host "Failed to upload file: $localFilePath" Write-Host "Failed to upload file: $localFilePath"
} }
} }
} }
} } finally {
finally # Dispose of the session if it's not null
{ if ($session -ne $null) {
$session.Dispose() $session.Dispose()
} }
}
}