diff --git a/.gitignore b/.gitignore index ae3dc7d..1f8ddbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ Powershell/Tools/temp.ps1 +Powershell/Tools/temp.json +.vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json index c9a6001..583b9d7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "dotnet.defaultSolution": "disable" + "dotnet.defaultSolution": "disable", + "azureAutomation.directory.basePath": "c:\\Users\\jurjen.ladenius" } \ No newline at end of file diff --git a/Custom roles/Storage Data Reader.json b/Custom roles/Storage Data Reader.json index d0c3b1b..d78b306 100644 --- a/Custom roles/Storage Data Reader.json +++ b/Custom roles/Storage Data Reader.json @@ -60,6 +60,7 @@ "/subscriptions/2021c41e-3582-40a1-85e5-59588741e6d3", "/subscriptions/17a9bacf-5a67-44bf-a532-950a4b3bf25d", "/subscriptions/181146cf-2354-4167-a6db-94ad14329c4d", - "/subscriptions/38a6726a-7c4d-402c-8bc2-36677fc0c3b4" + "/subscriptions/38a6726a-7c4d-402c-8bc2-36677fc0c3b4", + "/subscriptions/e33c30f7-06c7-4765-86a9-7a8979b6d1cc" ] } diff --git a/Powershell/RunBooks/RemoveBlobTags.ps1 b/Powershell/RunBooks/RemoveBlobTags.ps1 new file mode 100644 index 0000000..0f3a842 --- /dev/null +++ b/Powershell/RunBooks/RemoveBlobTags.ps1 @@ -0,0 +1,71 @@ +param ( + [Parameter(mandatory=$true)] + [string] $storageName, + [Parameter(mandatory=$true)] + [string] $subscriptionId, + [Parameter(mandatory=$false)] + [int]$maxIterations=2, + [Parameter(mandatory=$false)] + [int]$maxBlobPerIteration = 1000 +) + +Import-Module Az.Storage + +$stopwatch = [system.diagnostics.stopwatch]::StartNew() +[int] $total = 0 + +try { + # This scripts removes all blob tags from an azure blob storage container based on a tag filter in the script. It performs it in chunks of 1000. + # If there are no files left with the tag criteria the script will terminate + # see: https://github.com/m4m4m4/CleanBlobTags/tree/main + # see: https://www.reddit.com/r/AZURE/comments/1gvmulv/azure_blob_storage_malware_scan_and_blob_index/ + + # Connect to Azure with system-assigned managed identity + Disable-AzContextAutosave -Scope Process + Connect-AzAccount -Identity + + # Set and store context + Set-AzContext -SubscriptionId $subscriptionId + + # Connect to the Azure Storage account + $context = New-AzStorageContext -StorageAccountName $storageName -UseConnectedAccount + + $token = $Null + Do + { + #Retrieve blobs + $blobs = Get-AzStorageBlobByTag -Context $context -TagFilterSqlExpression """Malware Scanning scan time UTC"">'0'" -MaxCount $maxBlobPerIteration -ContinuationToken $token + + $blobCount = 1 + + #Loop through the batch + Foreach ($blob in $blobs) + { + # Remove tags, as there really should 0 tags + Set-AzStorageBlobTag -Context $context -Container $blob.BlobBaseClient.BlobContainerName -Blob $blob.Name -Tag @{} | out-null + + #Display progress bar + $percent = $($blobCount/$maxBlobPerIteration*100) + Write-Progress -Activity "Processing blobs" -Status "$percent% Complete" -PercentComplete $percent + $blobCount++ + } + + #Update $total + $total += $blobs.Count + + #Exit if all blobs processed + If($blobs.Length -le 0) { Break; } + + #Set continuation token to retrieve the next batch + $token = $blobs[$blobs.Count -1].ContinuationToken + $maxIterations-- + } + While ($null -ne $token -and $maxIterations -gt 0) +} +catch +{ + Write-Error $_ +} + +$stopwatch.Stop() +Write-Output "Processed $total blobs in $($stopwatch.Elapsed)." \ No newline at end of file diff --git a/Powershell/Tools/Cleanup/Defaults.json b/Powershell/Tools/Cleanup/Defaults.json new file mode 100644 index 0000000..92ba7c9 --- /dev/null +++ b/Powershell/Tools/Cleanup/Defaults.json @@ -0,0 +1,5 @@ +{ + "TenantId": "e9792fd7-4044-47e7-a40d-3fba46f1cd09", + "AzEnvironment": "AzureCloud", + "AlwaysOnlyMarkForDeletion": true + } \ No newline at end of file