Created
November 20, 2025 17:34
-
-
Save craig-martin/de9a9b7d02763c008f1459329624ab37 to your computer and use it in GitHub Desktop.
PowerShell script function to split a JSON file into chunks
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| function Split-JsonFile | |
| { | |
| [CmdletBinding()] | |
| Param( | |
| [Parameter(Mandatory = $true, ValueFromPipeline = $true, ValueFromPipelineByPropertyName = $true)] | |
| [ValidateNotNullOrEmpty()] | |
| [string]$Path, | |
| [Parameter()] | |
| [long]$MaxChunkBytes = 8MB, # Keep chunks safely under 10MB for the Kusto ingestion REST API | |
| [switch]$RemoveInputFile = $false | |
| ) | |
| process{ | |
| Write-Verbose "[$(Get-Date)] Processing file: $Path" | |
| if (-not (Test-Path -LiteralPath $Path)) { | |
| throw "File not found: $Path" | |
| } | |
| $file = Get-Item -LiteralPath $Path | |
| if ($file.Length -le $MaxChunkBytes) { | |
| Write-Verbose "[$(Get-Date)] File is <= MaxChunkBytes; no splitting required." | |
| return | |
| } | |
| <# | |
| The Log Analytics Data Collector API has a limit of 30MB for the JSON body | |
| If the JSON input is larger than 25MB then it gets divided into 25MB chunks | |
| Each JSON chunk is then uploaded individually | |
| #> | |
| Write-Verbose "[$(Get-Date)] JSON input is larger than the UploadSize limit of $($MaxChunkBytes / 1MB) MB" | |
| Write-Verbose "[$(Get-Date)] Loading JSON objects..." | |
| $jsonObjects = Get-Content -Path $Path | ConvertFrom-Json | |
| Write-Verbose "[$(Get-Date)] $($jsonObjects.Count) objects loaded." | |
| Write-Verbose "[$(Get-Date)] Dividing the JSON objects into chunks..." | |
| $newJsonBuilder = [System.Text.StringBuilder]::new() | |
| [void]$newJsonBuilder.Append('[') | |
| $chunkCount = 1 | |
| for ($j = 0; $j -lt $jsonObjects.Count; $j++) | |
| { | |
| [void]$newJsonBuilder.Append(($jsonObjects[$j] | ConvertTo-Json)) | |
| if ($newJsonBuilder.Length -gt $MaxChunkBytes -or $j -eq $jsonObjects.Count -1) | |
| { | |
| Write-Verbose "[$(Get-Date)] Created JSON with length of $(($newJsonBuilder.Length / 1MB).ToString("#.##")) MB at object number $($j+1) of $($jsonObjects.Count)" | |
| [void]$newJsonBuilder.Append(']') | |
| $chunkPath = Join-Path $file.Directory ("{0}.chunk{1}.json" -f $file.BaseName, $chunkCount) | |
| Write-Verbose "[$(Get-Date)] Chunk path: $chunkPath" | |
| $newJsonBuilder.ToString() | Out-File -FilePath $chunkPath -Encoding utf8 | |
| $newJsonBuilder = [System.Text.StringBuilder]::new() | |
| [void]$newJsonBuilder.Append('[') | |
| $chunkCount++ | |
| } | |
| else | |
| { | |
| [void]$newJsonBuilder.Append(',') | |
| } | |
| } | |
| if ($RemoveInputFile -eq $true) | |
| { | |
| Write-Verbose "[$(Get-Date)] Removing input file: $Path" | |
| Remove-Item -Path $Path | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment