ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add in checking for new files once list is depleted

Browse Source
This commit is contained in:
datechnoman 2023-12-21 01:58:55 +00:00
parent 0aad853966
commit 50e89b9de2
1 changed files with 54 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
commoncrawl_local_to_share_move.ps1
View File

@ -7,76 +7,74 @@ $assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"
# Specify the path to winscp.exe # Specify the path to winscp.exe
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe" $winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"
# Create a session instance # Specify the local directory path
$sessionType = [WinSCP.Session].FullName $localPath = "E:\CommonCrawl"
$session = New-Object $sessionType
# Set up session options # Specify the remote directory path
$sessionOptionsType = [WinSCP.SessionOptions].FullName $remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-14_March_April_2023"
$sessionOptions = New-Object $sessionOptionsType
# Set properties for session options while ($true) {
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp $session = $null
$sessionOptions.HostName = "192.168.1.248"
$sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
try try {
{ # Create a session instance
# Set the ExecutablePath property $sessionType = [WinSCP.Session].FullName
$session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null) $session = New-Object $sessionType
# Connect # Set up session options
$session.Open($sessionOptions) $sessionOptionsType = [WinSCP.SessionOptions].FullName
$sessionOptions = New-Object $sessionOptionsType
# Specify the local directory path # Set properties for session options
$localPath = "E:\CommonCrawl" $sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
$sessionOptions.HostName = "192.168.1.248"
$sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
# Specify the remote directory path # Set the ExecutablePath property
$remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-23_May_June_2023" $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
# Get a list of local files matching the pattern "_urls.txt.gz" # Connect
$localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse $session.Open($sessionOptions)
# Shuffle the local files array # Get a list of local files matching the pattern "_urls.txt.gz"
$localFiles = $localFiles | Get-Random -Count $localFiles.Count $localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
# Check if there are any files to upload # Shuffle the local files array
if ($localFiles.Count -eq 0) $localFiles = $localFiles | Get-Random -Count $localFiles.Count
{
Write-Host "No files found to upload."
}
else
{
# Iterate through each file and upload/delete it
foreach ($fileInfo in $localFiles)
{
# Construct the full local file path
$localFilePath = $fileInfo.FullName
# Construct the remote file path # Check if there are any files to upload
$remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.Name) if ($localFiles.Count -eq 0) {
Write-Host "No files found to upload."
break # Break out of the loop if no files are found
} else {
# Iterate through each file and upload/delete it
foreach ($fileInfo in $localFiles) {
# Construct the full local file path
$localFilePath = $fileInfo.FullName
# Upload the file # Construct the remote file path
$transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False) $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.Name)
# Check if the upload was successful # Upload the file
if ($transferResult.IsSuccess) $transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False)
{
Write-Host "Upload successful: $localFilePath to $remoteFilePath"
# Attempt to delete the local file # Check if the upload was successful
Remove-Item -Path $localFilePath -Force if ($transferResult.IsSuccess) {
} Write-Host "Upload successful: $localFilePath to $remoteFilePath"
else
{ # Attempt to delete the local file
Write-Host "Failed to upload file: $localFilePath" Remove-Item -Path $localFilePath -Force
} else {
Write-Host "Failed to upload file: $localFilePath"
}
} }
} }
} finally {
# Dispose of the session if it's not null
if ($session -ne $null) {
$session.Dispose()
}
} }
} }
finally
{
$session.Dispose()
}