mirror of
https://dev.azure.com/effectory/Survey%20Software/_git/Cloud%20Engineering
synced 2026-02-27 18:52:18 +01:00
added documetation
This commit is contained in:
Jurjen Ladenius
@@ -1,86 +1,215 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Exports Azure Storage blob listings with optional container filtering and blob prefix matching to CSV format.
|
||||
|
||||
.DESCRIPTION
|
||||
This script inventories Azure Storage blobs across containers within a specified storage account.
|
||||
It supports multiple modes of operation including containers-only listing, full blob enumeration,
|
||||
container exclusion filtering, and blob prefix filtering.
|
||||
|
||||
The script uses Azure Storage continuation tokens to handle large datasets efficiently and
|
||||
exports results in batches to prevent memory issues with very large storage accounts.
|
||||
|
||||
Key Features:
|
||||
- Container-only mode for quick container inventory
|
||||
- Full blob enumeration with metadata
|
||||
- Container exclusion filtering for system containers
|
||||
- Blob prefix filtering for targeted inventory
|
||||
- Large dataset handling with continuation tokens
|
||||
- CSV export with timestamped filenames
|
||||
|
||||
.PARAMETER subscriptionId
|
||||
[Required] The Azure subscription ID containing the storage account.
|
||||
|
||||
.PARAMETER resourcegroupName
|
||||
[Required] The resource group name containing the storage account.
|
||||
|
||||
.PARAMETER storageAccountName
|
||||
[Required] The name of the Azure Storage account to inventory.
|
||||
|
||||
.PARAMETER containersOnly
|
||||
[Optional] Switch to export only container information without blob details.
|
||||
Default: $false (full blob enumeration)
|
||||
|
||||
.PARAMETER excludedContainers
|
||||
[Optional] Array of container names to exclude from the inventory.
|
||||
Useful for filtering out system containers like '$logs', '$blobchangefeed', etc.
|
||||
Default: Empty array (no exclusions)
|
||||
|
||||
.PARAMETER blobPrefix
|
||||
[Optional] Blob name prefix filter to limit results to blobs starting with specified string.
|
||||
Useful for targeting specific blob hierarchies or naming patterns.
|
||||
Default: Empty string (no prefix filtering)
|
||||
|
||||
.OUTPUTS
|
||||
CSV file named with timestamp pattern: "yyyy-MM-dd HHmm - [StorageAccountName] - bloblist.csv"
|
||||
Contains columns for subscription, resource group, storage account, container, blob name, and last modified date.
|
||||
|
||||
.EXAMPLE
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "ecestore"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "mailingstore"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "projectcenter"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "effectorycore" -storageAccountName "corerightsaggregator" -ContainersOnly $true
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "14c2354d-45a9-4e0f-98ff-be58cdbcddc7" -resourcegroupName "ec-automation-prod" -storageAccountName "stecautomationprod"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "Results" -storageAccountName "myeffectoryresults"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "3190b0fd-4a66-4636-a204-5b9f18be78a6" -resourcegroupName "authorization" -storageAccountName "authorizationv2"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "effectorycore" -storageAccountName "coremailings"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "results-activity" -storageAccountName "effactivity" -excludedContainers "`$logs","`$blobchangefeed", "activitybackup-applease", "activitybackup-largemessages", "activitybackup-leases", "activitycleanup-applease", "activitycleanup-leases", "activityprojectors-largemessages", "activityprojectors-leases", "activityquestionnaireavailableactivitygenerat-largemessages", "activityquestionnaireavailableactivitygenerat-leases", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "testhubname-applease", "testhubname-largemessages", "testhubname-leases" -blobPrefix "projects"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "86945e42-fa5a-4bbc-948f-3f5407f15d3e" -resourcegroupName "hierarchy" -storageAccountName "hierarchyeff"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "6e2b45e4-5e7b-4628-8827-ec44e23d2f6b" -resourcegroupName "ParticipantIntegration-Settings" -storageAccountName "integrationsettings"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "1ab2120c-947c-40e2-96c7-460d3e9659de" -resourcegroupName "sa-backups" -storageAccountName "archivecommvault"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "1ab2120c-947c-40e2-96c7-460d3e9659de" -resourcegroupName "sa-backups" -storageAccountName "backupcommvault"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-server-weu" -storageAccountName "questionnairestoreweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "f9ab522b-4895-492d-b8a8-ca6e1f60c2a8" -resourcegroupName "participant-exchange" -storageAccountName "participantexchangev2" -excludedContainers "leases","insights-metrics-pt1m","insights-logs-partitionkeystatistics","insights-logs-dataplanerequests","insights-logs-controlplanerequests","event-attachments","command-handlers","aggregates-streaming","aggregates","`$logs","`$blobchangefeed"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "14c2354d-45a9-4e0f-98ff-be58cdbcddc7" -resourcegroupName "ec-measurement" -storageAccountName "stecmeasurementprod"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-server-weu" -storageAccountName "questionnairedataweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-api-weu" -storageAccountName "qmprojectionsweu" -excludedContainers "`$logs","`$blobchangefeed"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "Results" -storageAccountName "myeffectoryresults" -excludedContainers "`$logs","`$blobchangefeed", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "azure-webjobs-dashboard", "azure-webjobs-hosts", "azure-webjobs-secrets", "hierarchydatesettings-leases", "projectcalculations-leases","resultscleanup-applease","resultscleanup-leases","resultsgroupscorecalculator-leases","testhubname-leases"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "results-calculation" -storageAccountName "resultscalculation" -excludedContainers "`$logs","`$blobchangefeed", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "local-leases", "local-applease", "calculations", "calculations-test"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-internaldata_api-weu" -storageAccountName "qmidapiweustore"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-library-weu" -storageAccountName "qmlibraryweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-media-api-weu" -storageAccountName "qmmediaweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-api-weu" -storageAccountName "qmprojectionsweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-data-collector-api-weu" -storageAccountName "quedatacolstoreweu"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "34c83aa8-6a8f-4c5e-9c27-0f1730d233bb" -resourcegroupName "start-a-survey" -storageAccountName "startasurvey" -excludedContainers "active-projects","attachments","attachments-logs","azure-webjobs-hosts","azure-webjobs-secrets","durablefunctionshub-largemessages","durablefunctionshub-leases","event-documents","locales","locales-theme-names","pdf-temp","portal","public","schemas"
|
||||
.\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "rg-yourfeedback-001" -storageAccountName "yourfeedback" -excludedContainers "`$logs","`$blobchangefeed"
|
||||
|
||||
Exports blobs with "projects" prefix while excluding system containers.
|
||||
|
||||
.NOTES
|
||||
Author: Cloud Engineering Team
|
||||
Version: 1.0
|
||||
Created: 2024
|
||||
|
||||
Prerequisites:
|
||||
- Azure PowerShell module (Az.Storage) must be installed
|
||||
- User must be authenticated (Connect-AzAccount)
|
||||
- Requires Storage Blob Data Reader permissions or higher on the target storage account
|
||||
|
||||
Performance Considerations:
|
||||
- Uses continuation tokens to handle large datasets efficiently
|
||||
- Processes results in batches of 100,000 items to manage memory usage
|
||||
- Export operations are performed incrementally to prevent timeouts
|
||||
- Large storage accounts may take considerable time to process
|
||||
|
||||
Security Notes:
|
||||
- Requires appropriate RBAC permissions on storage account
|
||||
- Consider using managed identities for automated scenarios
|
||||
- Output file contains blob metadata and should be handled securely
|
||||
|
||||
Common Use Cases:
|
||||
- Storage account auditing and inventory
|
||||
- Data migration planning and assessment
|
||||
- Cleanup operations and lifecycle management
|
||||
- Compliance reporting and data discovery
|
||||
|
||||
.LINK
|
||||
https://docs.microsoft.com/en-us/azure/storage/blobs/
|
||||
https://docs.microsoft.com/en-us/powershell/module/az.storage/
|
||||
#>
|
||||
|
||||
#Requires -Modules Az.Storage
|
||||
param (
|
||||
[string] $subscriptionId = "",
|
||||
[string] $resourcegroupName = "",
|
||||
[string] $storageAccountName = "",
|
||||
[Parameter(Mandatory = $true, HelpMessage = "Azure subscription ID containing the storage account")]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string] $subscriptionId,
|
||||
|
||||
[Parameter(Mandatory = $true, HelpMessage = "Resource group name containing the storage account")]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string] $resourcegroupName,
|
||||
|
||||
[Parameter(Mandatory = $true, HelpMessage = "Azure Storage account name to inventory")]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string] $storageAccountName,
|
||||
|
||||
[Parameter(Mandatory = $false, HelpMessage = "Export only container information without blob details")]
|
||||
[bool] $containersOnly = $false,
|
||||
|
||||
[Parameter(Mandatory = $false, HelpMessage = "Array of container names to exclude from inventory")]
|
||||
[string[]] $excludedContainers = @(),
|
||||
|
||||
[Parameter(Mandatory = $false, HelpMessage = "Blob name prefix filter for targeted inventory")]
|
||||
[string] $blobPrefix = ""
|
||||
)
|
||||
|
||||
if (("" -eq $subscriptionId) -or ("" -eq $resourcegroupName) -or ("" -eq $storageAccountName)) {
|
||||
throw "Parameter(s) missing."
|
||||
}
|
||||
else {
|
||||
Write-Host "Processing subscription [$subscriptionId], resource group [$resourcegroupName], storage account [$storageAccountName]"
|
||||
}
|
||||
# Parameter validation and initialization
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Write-Host "Starting Azure Storage Blob inventory process"
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "ecestore"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "mailingstore"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "Default-Storage-WestEurope" -storageAccountName "projectcenter"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "effectorycore" -storageAccountName "corerightsaggregator" -ContainersOnly $true
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "14c2354d-45a9-4e0f-98ff-be58cdbcddc7" -resourcegroupName "ec-automation-prod" -storageAccountName "stecautomationprod"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "Results" -storageAccountName "myeffectoryresults"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "3190b0fd-4a66-4636-a204-5b9f18be78a6" -resourcegroupName "authorization" -storageAccountName "authorizationv2"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "a134faf1-7a89-4f2c-8389-06d00bd5e2a7" -resourcegroupName "effectorycore" -storageAccountName "coremailings"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "results-activity" -storageAccountName "effactivity" -excludedContainers "`$logs","`$blobchangefeed", "activitybackup-applease", "activitybackup-largemessages", "activitybackup-leases", "activitycleanup-applease", "activitycleanup-leases", "activityprojectors-largemessages", "activityprojectors-leases", "activityquestionnaireavailableactivitygenerat-largemessages", "activityquestionnaireavailableactivitygenerat-leases", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "testhubname-applease", "testhubname-largemessages", "testhubname-leases" -blobPrefix "projects"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "86945e42-fa5a-4bbc-948f-3f5407f15d3e" -resourcegroupName "hierarchy" -storageAccountName "hierarchyeff"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "6e2b45e4-5e7b-4628-8827-ec44e23d2f6b" -resourcegroupName "ParticipantIntegration-Settings" -storageAccountName "integrationsettings"
|
||||
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "1ab2120c-947c-40e2-96c7-460d3e9659de" -resourcegroupName "sa-backups" -storageAccountName "archivecommvault"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "1ab2120c-947c-40e2-96c7-460d3e9659de" -resourcegroupName "sa-backups" -storageAccountName "backupcommvault"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-server-weu" -storageAccountName "questionnairestoreweu"
|
||||
#
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "f9ab522b-4895-492d-b8a8-ca6e1f60c2a8" -resourcegroupName "participant-exchange" -storageAccountName "participantexchangev2" -excludedContainers "leases","insights-metrics-pt1m","insights-logs-partitionkeystatistics","insights-logs-dataplanerequests","insights-logs-controlplanerequests","event-attachments","command-handlers","aggregates-streaming","aggregates","`$logs","`$blobchangefeed"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "14c2354d-45a9-4e0f-98ff-be58cdbcddc7" -resourcegroupName "ec-measurement" -storageAccountName "stecmeasurementprod"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-server-weu" -storageAccountName "questionnairedataweu"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-api-weu" -storageAccountName "qmprojectionsweu" -excludedContainers "`$logs","`$blobchangefeed"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "Results" -storageAccountName "myeffectoryresults" -excludedContainers "`$logs","`$blobchangefeed", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "azure-webjobs-dashboard", "azure-webjobs-hosts", "azure-webjobs-secrets", "hierarchydatesettings-leases", "projectcalculations-leases","resultscleanup-applease","resultscleanup-leases","resultsgroupscorecalculator-leases","testhubname-leases"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "results-calculation" -storageAccountName "resultscalculation" -excludedContainers "`$logs","`$blobchangefeed", "attachments", "azure-webjobs-hosts", "azure-webjobs-secrets", "local-leases", "local-applease", "calculations", "calculations-test"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-internaldata_api-weu" -storageAccountName "qmidapiweustore"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-library-weu" -storageAccountName "qmlibraryweu"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-media-api-weu" -storageAccountName "qmmediaweu"
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "54794e27-b714-4346-81bc-05eae7ccb5a5" -resourcegroupName "question-management-api-weu" -storageAccountName "qmprojectionsweu"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "2a07dfa7-69ee-4608-b2d5-14124fcccc31" -resourcegroupName "questionnaire-data-collector-api-weu" -storageAccountName "quedatacolstoreweu"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "34c83aa8-6a8f-4c5e-9c27-0f1730d233bb" -resourcegroupName "start-a-survey" -storageAccountName "startasurvey" -excludedContainers "active-projects","attachments","attachments-logs","azure-webjobs-hosts","azure-webjobs-secrets","durablefunctionshub-largemessages","durablefunctionshub-leases","event-documents","locales","locales-theme-names","pdf-temp","portal","public","schemas"
|
||||
|
||||
# .\AzureStoragebloblist.ps1 -subscriptionId "7feeb150-9ee0-4aea-992a-5f3a89d933e6" -resourcegroupName "rg-yourfeedback-001" -storageAccountName "yourfeedback" -excludedContainers "`$logs","`$blobchangefeed"
|
||||
Write-Host "Configuration:"
|
||||
Write-Host " Subscription ID: $subscriptionId"
|
||||
Write-Host " Resource Group: $resourcegroupName"
|
||||
Write-Host " Storage Account: $storageAccountName"
|
||||
Write-Host " Containers Only Mode: $containersOnly"
|
||||
Write-Host " Excluded Containers: $($excludedContainers -join ', ')"
|
||||
Write-Host " Blob Prefix Filter: $($blobPrefix -eq '' ? 'None' : $blobPrefix)"
|
||||
Write-Host ""
|
||||
|
||||
# Class definition for structured blob inventory data
|
||||
class BlobCheck {
|
||||
[string] $SubscriptionId = ""
|
||||
[string] $SubscriptionName = ""
|
||||
[string] $ResourcegroupName = ""
|
||||
[string] $StorageAccountName = ""
|
||||
[string] $ContainerName = ""
|
||||
[string] $BlobName = ""
|
||||
[string] $LastModifiedDate = ""
|
||||
[string] $SubscriptionId = "" # Azure subscription GUID
|
||||
[string] $SubscriptionName = "" # Subscription display name
|
||||
[string] $ResourcegroupName = "" # Resource group containing the storage account
|
||||
[string] $StorageAccountName = "" # Storage account name
|
||||
[string] $ContainerName = "" # Blob container name
|
||||
[string] $BlobName = "" # Individual blob name (empty for container-only mode)
|
||||
[string] $LastModifiedDate = "" # Last modified timestamp for container or blob
|
||||
}
|
||||
|
||||
[int] $maxCount = 100000
|
||||
$containerToken = $null
|
||||
$blobToken = $null
|
||||
# Configuration constants for large dataset handling
|
||||
[int] $maxCount = 100000 # Maximum items per batch to manage memory usage
|
||||
$containerToken = $null # Continuation token for container enumeration
|
||||
$blobToken = $null # Continuation token for blob enumeration
|
||||
|
||||
# Generate timestamped output filename
|
||||
[string] $date = Get-Date -Format "yyyy-MM-dd HHmm"
|
||||
$fileName = ".\$date - $storageAccountName - bloblist.csv"
|
||||
Write-Host "Output file: $fileName"
|
||||
|
||||
$subscription = Set-AzContext -SubscriptionId $subscriptionId
|
||||
$storageAccount = Get-AzStorageAccount -ResourceGroupName $resourcegroupName -Name $storageAccountName
|
||||
try {
|
||||
# Set Azure context and retrieve storage account
|
||||
Write-Host "Setting Azure context and retrieving storage account..."
|
||||
$subscription = Set-AzContext -SubscriptionId $subscriptionId
|
||||
$storageAccount = Get-AzStorageAccount -ResourceGroupName $resourcegroupName -Name $storageAccountName
|
||||
|
||||
Write-Host "Successfully connected to storage account: $($storageAccount.StorageAccountName)"
|
||||
Write-Host "Storage account location: $($storageAccount.Location)"
|
||||
Write-Host "Storage account SKU: $($storageAccount.Sku.Name)"
|
||||
Write-Host ""
|
||||
}
|
||||
catch {
|
||||
Write-Error "Failed to connect to storage account: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Execute inventory based on mode selection
|
||||
if ($containersOnly -eq $true) {
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Write-Host "CONTAINERS ONLY MODE: Inventorying container information without blob details"
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
|
||||
$totalContainers = 0
|
||||
|
||||
# Container-only enumeration loop with continuation token support
|
||||
do {
|
||||
Write-Host "Processing container batch (max $maxCount containers)..."
|
||||
[BlobCheck[]]$Result = @()
|
||||
|
||||
# Retrieve containers with continuation token support
|
||||
$containers = Get-AzStorageContainer -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $containerToken
|
||||
|
||||
# Apply container exclusion filters if specified
|
||||
if ($excludedContainers.Length -gt 0) {
|
||||
$originalCount = $containers.Count
|
||||
$containers = $containers | Where-Object { $excludedContainers -notcontains $_.Name }
|
||||
$filteredCount = $originalCount - $containers.Count
|
||||
if ($filteredCount -gt 0) {
|
||||
Write-Host " Filtered out $filteredCount excluded container(s)"
|
||||
}
|
||||
}
|
||||
|
||||
# Process each container and create inventory records
|
||||
foreach ($container in $containers) {
|
||||
[BlobCheck] $blobCheck = [BlobCheck]::new()
|
||||
$blobCheck.SubscriptionId = $subscription.Subscription.Id
|
||||
@@ -88,44 +217,90 @@ if ($containersOnly -eq $true) {
|
||||
$blobCheck.ResourcegroupName = $resourcegroupName
|
||||
$blobCheck.StorageAccountName = $storageAccountName
|
||||
$blobCheck.ContainerName = $container.Name
|
||||
$blobCheck.BlobName = ""
|
||||
$blobCheck.BlobName = "" # Empty for container-only mode
|
||||
$blobCheck.LastModifiedDate = $container.LastModified
|
||||
$Result += $blobCheck
|
||||
}
|
||||
|
||||
# Export current batch to CSV if results exist
|
||||
if ($Result.Length -gt 0) {
|
||||
$Result | Export-Csv -Path $fileName -NoTypeInformation -Append
|
||||
$totalContainers += $Result.Length
|
||||
Write-Host " Exported $($Result.Length) container(s) to CSV (Total: $totalContainers)"
|
||||
}
|
||||
|
||||
# Check for continuation and prepare next iteration
|
||||
if ($containers.Length -le 0) {
|
||||
Break;
|
||||
Write-Host " No more containers to process"
|
||||
Break
|
||||
}
|
||||
$containerToken = $containers[$containers.Count - 1].ContinuationToken;
|
||||
$containerToken = $containers[$containers.Count - 1].ContinuationToken
|
||||
}
|
||||
while ($null -ne $containerToken)
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Container inventory completed. Total containers processed: $totalContainers"
|
||||
}
|
||||
elseif ($containersOnly -eq $false) {
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Write-Host "FULL BLOB ENUMERATION MODE: Inventorying all blobs across all containers"
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
|
||||
$totalContainers = 0
|
||||
$totalBlobs = 0
|
||||
|
||||
# Full blob enumeration with nested container/blob loops
|
||||
do {
|
||||
|
||||
Write-Host "Processing container batch (max $maxCount containers)..."
|
||||
|
||||
# Retrieve containers with continuation token support
|
||||
$containers = Get-AzStorageContainer -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $containerToken
|
||||
|
||||
# Apply container exclusion filters if specified
|
||||
if ($excludedContainers.Length -gt 0) {
|
||||
$originalCount = $containers.Count
|
||||
$containers = $containers | Where-Object { $excludedContainers -notcontains $_.Name }
|
||||
$filteredCount = $originalCount - $containers.Count
|
||||
if ($filteredCount -gt 0) {
|
||||
Write-Host " Filtered out $filteredCount excluded container(s)"
|
||||
}
|
||||
}
|
||||
|
||||
# Process each container for blob enumeration
|
||||
foreach ($container in $containers) {
|
||||
Write-Host " Processing container: $($container.Name)"
|
||||
$containerBlobCount = 0
|
||||
|
||||
# Reset blob continuation token for each container
|
||||
$blobToken = $null
|
||||
|
||||
# Blob enumeration loop with continuation token support
|
||||
do {
|
||||
[BlobCheck[]]$Result = @()
|
||||
|
||||
if ("" -ne $blobPrefix) {
|
||||
$blobList = Get-AzStorageBlob -Container $Container.Name -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $blobToken -Prefix $blobPrefix
|
||||
# Retrieve blobs with optional prefix filtering
|
||||
try {
|
||||
if ("" -ne $blobPrefix) {
|
||||
Write-Host " Retrieving blobs with prefix '$blobPrefix' (max $maxCount blobs)..."
|
||||
$blobList = Get-AzStorageBlob -Container $Container.Name -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $blobToken -Prefix $blobPrefix
|
||||
}
|
||||
else {
|
||||
Write-Host " Retrieving all blobs (max $maxCount blobs)..."
|
||||
$blobList = Get-AzStorageBlob -Container $Container.Name -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $blobToken
|
||||
}
|
||||
}
|
||||
else {
|
||||
$blobList = Get-AzStorageBlob -Container $Container.Name -Context $storageAccount.Context -MaxCount $maxCount -ContinuationToken $blobToken
|
||||
catch {
|
||||
Write-Warning " Failed to retrieve blobs from container '$($container.Name)': $($_.Exception.Message)"
|
||||
break
|
||||
}
|
||||
|
||||
# Exit loop if no blobs found
|
||||
if ($blobList.Length -le 0) {
|
||||
Break;
|
||||
Write-Host " No more blobs in container"
|
||||
Break
|
||||
}
|
||||
|
||||
# Process each blob and create inventory records
|
||||
foreach ($blob in $blobList) {
|
||||
[BlobCheck] $blobCheck = [BlobCheck]::new()
|
||||
$blobCheck.SubscriptionId = $subscription.Subscription.Id
|
||||
@@ -137,16 +312,66 @@ elseif ($containersOnly -eq $false) {
|
||||
$blobCheck.LastModifiedDate = $blob.LastModified
|
||||
$Result += $blobCheck
|
||||
}
|
||||
|
||||
# Export current batch to CSV
|
||||
$Result | Export-Csv -Path $fileName -NoTypeInformation -Append
|
||||
$blobToken = $blobList[$blobList.Count - 1].ContinuationToken;
|
||||
$containerBlobCount += $Result.Length
|
||||
$totalBlobs += $Result.Length
|
||||
Write-Host " Exported $($Result.Length) blob(s) to CSV (Container total: $containerBlobCount, Overall total: $totalBlobs)"
|
||||
|
||||
# Prepare continuation token for next iteration
|
||||
$blobToken = $blobList[$blobList.Count - 1].ContinuationToken
|
||||
}
|
||||
while ($null -ne $blobToken)
|
||||
|
||||
Write-Host " Container '$($container.Name)' completed. Total blobs: $containerBlobCount"
|
||||
$totalContainers++
|
||||
}
|
||||
|
||||
# Check for continuation and prepare next container batch
|
||||
if ($containers.Length -le 0) {
|
||||
Break;
|
||||
Write-Host "No more containers to process"
|
||||
Break
|
||||
}
|
||||
$containerToken = $containers[$containers.Count - 1].ContinuationToken;
|
||||
$containerToken = $containers[$containers.Count - 1].ContinuationToken
|
||||
}
|
||||
while ($null -ne $containerToken)
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Full blob inventory completed."
|
||||
Write-Host " Total containers processed: $totalContainers"
|
||||
Write-Host " Total blobs exported: $totalBlobs"
|
||||
}
|
||||
|
||||
# Script completion summary
|
||||
Write-Host ""
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Write-Host "Azure Storage blob inventory completed successfully."
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Write-Host "Results exported to: $fileName"
|
||||
Write-Host ""
|
||||
|
||||
# Display final summary based on mode
|
||||
if ($containersOnly) {
|
||||
Write-Host "Summary (Containers Only Mode):"
|
||||
Write-Host " Storage Account: $storageAccountName"
|
||||
Write-Host " Containers inventoried (after exclusions)"
|
||||
if ($excludedContainers.Length -gt 0) {
|
||||
Write-Host " Excluded containers: $($excludedContainers.Length)"
|
||||
}
|
||||
} else {
|
||||
Write-Host "Summary (Full Blob Enumeration Mode):"
|
||||
Write-Host " Storage Account: $storageAccountName"
|
||||
Write-Host " Total containers processed: $totalContainers"
|
||||
Write-Host " Total blobs inventoried: $totalBlobs"
|
||||
if ($excludedContainers.Length -gt 0) {
|
||||
Write-Host " Excluded containers: $($excludedContainers.Length)"
|
||||
}
|
||||
if ($blobPrefix -ne "") {
|
||||
Write-Host " Blob prefix filter applied: $blobPrefix"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "For large storage accounts, consider using container exclusions or blob prefix filters"
|
||||
Write-Host "to optimize processing time and focus on relevant data."
|
||||
Write-Host "======================================================================================================================================================================"
|
||||
Reference in New Issue
Block a user