Skip to content

Instantly share code, notes, and snippets.

@rhnvrm

rhnvrm/iterator_benchmark_test.go

Last active July 8, 2025 08:35
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rhnvrm/401bc298d0f83227212abcac1fb9d46d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rhnvrm/401bc298d0f83227212abcac1fb9d46d to your computer and use it in GitHub Desktop.
Download ZIP
Go Iterators: Complete Performance Guide & Comparison
Raw
iterator_benchmark_test.go
// Go 1.23 Iterator Patterns: Complete Performance Guide
//
// This comprehensive benchmark compares 5 different iteration patterns in Go:
// 1. Go 1.23 Range-over-Function Iterators (NEW) ⭐
// 2. Copy-Everything Pattern (TRADITIONAL)
// 3. Callback Pattern (PRE-ITERATOR)
// 4. Channel Pattern (GOROUTINE-BASED)
// 5. Manual Iterator Struct (EXPLICIT STATE)
//
// 🏆 KEY FINDINGS:
// - Go 1.23 iterators provide significantly better performance than copy-everything
// - Early termination scenarios show massive performance advantages
// - Zero allocations vs substantial memory usage for copy-everything approach
// - Channel-based iteration is consistently much slower
// - Interface{} destroys performance - always use concrete types
//
// 📋 TO RUN:
// go test -bench=. -benchmem
// go test -run=Example -v
// go test -run=GOTCHA -v
// go test -gcflags="-m" # See escape analysis
//
// 📊 RESULTS WILL VARY: Test on your own hardware and environment
// 📄 LICENSE: MIT
// 👨‍💻 AUTHOR: Rohan Verma (rhnvrm) - Generated via LLM Assistance
package main
import (
"context"
"fmt"
"iter"
"math"
"sort"
"sync"
"testing"
"time"
)
// Record represents a realistic data structure you might iterate over
// Size: ~40 bytes (string header + int64 + float64) - typical for database records
type Record struct {
ID string // Unique identifier (e.g., "user_12345", "item_67890")
CreatedAt int64 // Unix timestamp
Score float64 // Priority, rating, weight, or other numeric value
}
// Collection represents a thread-safe data collection (common in many systems)
// This simulates scenarios like: user databases, product catalogs, task queues,
// cache systems, or any large collection that needs safe concurrent access
type Collection struct {
sync.RWMutex
data map[int]Record
}
func NewCollection(n int) *Collection {
c := &Collection{data: make(map[int]Record, n)}
for i := 0; i < n; i++ {
c.data[i] = Record{
ID: fmt.Sprintf("item_%05d", i),
CreatedAt: int64(i * 1000),
Score: float64(i) * 3.14159,
}
}
return c
}
const N = 10000 // Collection size - represents a realistic dataset size
// Helper function for floating point comparison with tolerance
func floatEqual(a, b, tolerance float64) bool {
return math.Abs(a-b) <= tolerance
}
// =============================================================================
// ⚠️ CRITICAL PERFORMANCE WARNING: NEVER USE interface{} IN ITERATORS
// =============================================================================
// Using interface{} in iterator signatures causes boxing - every value becomes
// an interface{} allocation. This destroys performance and causes excessive
// garbage collection. Always use concrete types like Record, not interface{}.
// =============================================================================
// PATTERN 1: Go 1.23 Range-over-Function Iterator (THE NEW WAY) ⭐
// =============================================================================
// ✅ Pros: Memory efficient, familiar syntax, composable, excellent performance
// ❌ Cons: Requires Go 1.23+, consumers can make performance mistakes
//
// This is the CORRECT way to implement iterators in Go 1.23+
// Key insight: Use concrete types (Record) not interface{} for performance
func (c *Collection) IterRecords() iter.Seq[Record] {
return func(yield func(Record) bool) {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
if !yield(v) {
return
}
}
}
}
// Advanced: Ordered iteration (when deterministic order matters)
func (c *Collection) IterRecordsOrdered() iter.Seq[Record] {
return func(yield func(Record) bool) {
c.RLock()
defer c.RUnlock()
// Sort keys for deterministic iteration
keys := make([]int, 0, len(c.data))
for k := range c.data {
keys = append(keys, k)
}
sort.Ints(keys)
for _, k := range keys {
if v, exists := c.data[k]; exists {
if !yield(v) {
return
}
}
}
}
}
// Advanced: Iterator with filtering built-in
func (c *Collection) IterRecordsWhere(predicate func(Record) bool) iter.Seq[Record] {
return func(yield func(Record) bool) {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
if predicate(v) {
if !yield(v) {
return
}
}
}
}
}
// Advanced: Iterator with cancellation support
func (c *Collection) IterRecordsWithContext(ctx context.Context) iter.Seq[Record] {
return func(yield func(Record) bool) {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
select {
case <-ctx.Done():
return // Respect context cancellation
default:
if !yield(v) {
return
}
}
}
}
}
// Advanced: Key-value iterator (iter.Seq2)
func (c *Collection) IterRecordsWithKeys() iter.Seq2[int, Record] {
return func(yield func(int, Record) bool) {
c.RLock()
defer c.RUnlock()
for k, v := range c.data {
if !yield(k, v) {
return
}
}
}
}
// ❌ ANTI-PATTERN: interface{} causes boxing - DON'T DO THIS
func (c *Collection) IterRecordsAny() iter.Seq[any] {
return func(yield func(any) bool) {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
if !yield(v) { // ← Each yield(v) boxes Record to interface{}
return
}
}
}
}
// =============================================================================
// PATTERN 2: Copy Everything (THE TRADITIONAL WAY)
// =============================================================================
// ✅ Pros: Simple, familiar, works with any Go version
// ❌ Cons: High memory usage, expensive allocation, long lock time, poor early termination
func (c *Collection) GetAllRecords() map[int]Record {
c.RLock()
defer c.RUnlock()
// Creates a complete copy of the entire map - expensive!
result := make(map[int]Record, len(c.data))
for k, v := range c.data {
result[k] = v // Each assignment copies the entire Record struct
}
return result
}
func (c *Collection) GetAllRecordsSlice() []Record {
c.RLock()
defer c.RUnlock()
result := make([]Record, 0, len(c.data))
for _, v := range c.data {
result = append(result, v) // Still copying each Record
}
return result
}
// =============================================================================
// PATTERN 3: Callback Pattern (THE PRE-ITERATOR WAY)
// =============================================================================
// ✅ Pros: Memory efficient, good performance, works with any Go version
// ❌ Cons: Unfamiliar syntax, can't use break/continue, callback hell for complex logic
func (c *Collection) ForEachRecord(fn func(Record) bool) {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
if !fn(v) { // Return false to break iteration
return
}
}
}
func (c *Collection) ForEachRecordWithError(fn func(Record) error) error {
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
if err := fn(v); err != nil {
return err
}
}
return nil
}
// =============================================================================
// PATTERN 4: Channel Pattern (THE GOROUTINE + CHANNEL WAY)
// =============================================================================
// ✅ Pros: Familiar range syntax, works with existing Go patterns, composable
// ❌ Cons: Expensive goroutine overhead, forces heap escapes, complex error handling
// Note: Basic channel pattern removed due to goroutine leak potential
// Always use context cancellation with channels to avoid resource leaks
func (c *Collection) RecordsChanWithContext(ctx context.Context) <-chan Record {
ch := make(chan Record, 100)
go func() {
defer close(ch)
c.RLock()
defer c.RUnlock()
for _, v := range c.data {
select {
case ch <- v:
case <-ctx.Done():
return // Proper cancellation support
}
}
}()
return ch
}
// =============================================================================
// PATTERN 5: Manual Iterator Struct (THE EXPLICIT STATE WAY)
// =============================================================================
// ✅ Pros: Full control, explicit state, can pause/resume, familiar to C++/Java devs
// ❌ Cons: Very verbose, complex state management, still slower due to key copying overhead
type RecordIterator struct {
collection *Collection
keys []int
index int
finished bool
}
func (c *Collection) NewRecordIterator() *RecordIterator {
c.RLock()
// Copy all keys and release lock immediately to avoid deadlock
keys := make([]int, 0, len(c.data))
for k := range c.data {
keys = append(keys, k)
}
c.RUnlock() // ← CRITICAL: Release lock immediately!
// Sort keys for deterministic iteration in benchmarks
sort.Ints(keys)
return &RecordIterator{
collection: c,
keys: keys,
index: 0,
finished: false,
}
}
func (it *RecordIterator) Next() (Record, bool) {
if it.finished || it.index >= len(it.keys) {
return Record{}, false
}
key := it.keys[it.index]
it.index++
// Re-acquire lock briefly for each value access
it.collection.RLock()
record, exists := it.collection.data[key]
it.collection.RUnlock()
if !exists {
// Use loop instead of recursion to avoid stack overflow
return it.Next()
}
return record, true
}
func (it *RecordIterator) Close() {
// No lock cleanup needed since we don't hold locks across calls
it.finished = true
}
// =============================================================================
// ITERATOR COMPOSITION & CHAINING EXAMPLES
// =============================================================================
// Transform iterator: map function over records
func Transform[T, U any](seq iter.Seq[T], fn func(T) U) iter.Seq[U] {
return func(yield func(U) bool) {
for v := range seq {
if !yield(fn(v)) {
return
}
}
}
}
// Filter iterator: only yield items matching predicate
func Filter[T any](seq iter.Seq[T], predicate func(T) bool) iter.Seq[T] {
return func(yield func(T) bool) {
for v := range seq {
if predicate(v) {
if !yield(v) {
return
}
}
}
}
}
// Take iterator: yield at most n items
func Take[T any](seq iter.Seq[T], n int) iter.Seq[T] {
return func(yield func(T) bool) {
count := 0
for v := range seq {
if count >= n {
return
}
count++
if !yield(v) {
return
}
}
}
}
// Collect helper: gather iterator results into slice
func Collect[T any](seq iter.Seq[T]) []T {
var result []T
for v := range seq {
result = append(result, v)
}
return result
}
// =============================================================================
// PULL ITERATORS: Converting Push to Pull Style
// =============================================================================
// Pull iterators convert range-over-function to explicit Next() calls
func Example_pullIterator() {
collection := NewCollection(5)
// Convert push-style iterator to pull-style (using ordered iteration for deterministic output)
next, stop := iter.Pull(collection.IterRecordsOrdered())
defer stop() // Important: always call stop to cleanup
fmt.Println("Pull-style iteration:")
for i := 0; i < 3; i++ { // Process only first 3 items
record, ok := next()
if !ok {
break
}
fmt.Printf("Item %d: %s\n", i+1, record.ID)
}
// Remaining items are automatically cleaned up by defer stop()
// Output:
// Pull-style iteration:
// Item 1: item_00000
// Item 2: item_00001
// Item 3: item_00002
}
// =============================================================================
// BENCHMARKS: Realistic Processing Scenarios
// =============================================================================
// Scenario 1: Simple processing - count items and sum values
func BenchmarkIterator_SimpleProcessing(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
sum := 0.0
for record := range collection.IterRecords() {
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
}
_ = count
_ = sum
}
}
func BenchmarkCopyAll_SimpleProcessing(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
records := collection.GetAllRecordsSlice()
count := 0
sum := 0.0
for _, record := range records {
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
}
_ = count
_ = sum
}
}
func BenchmarkCallback_SimpleProcessing(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
sum := 0.0
collection.ForEachRecord(func(record Record) bool {
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
return true
})
_ = count
_ = sum
}
}
func BenchmarkChannel_SimpleProcessing(b *testing.B) {
collection := NewCollection(N)
ctx := context.Background()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
sum := 0.0
for record := range collection.RecordsChanWithContext(ctx) {
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
}
_ = count
_ = sum
}
}
func BenchmarkManualIterator_SimpleProcessing(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
sum := 0.0
iter := collection.NewRecordIterator()
for {
record, ok := iter.Next()
if !ok {
break
}
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
}
iter.Close()
_ = count
_ = sum
}
}
// Scenario 2: Early termination - find first item matching condition
// NOTE: Performance advantage depends on when condition is met in your data
func BenchmarkIterator_EarlyTermination(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
for record := range collection.IterRecords() {
if record.Score > 5000.0 {
break
}
}
}
}
func BenchmarkCopyAll_EarlyTermination(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
records := collection.GetAllRecordsSlice()
for _, record := range records {
if record.Score > 5000.0 {
break
}
}
}
}
func BenchmarkCallback_EarlyTermination(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
collection.ForEachRecord(func(record Record) bool {
if record.Score > 5000.0 {
return false
}
return true
})
}
}
func BenchmarkChannel_EarlyTermination(b *testing.B) {
collection := NewCollection(N)
ctx := context.Background()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
for record := range collection.RecordsChanWithContext(ctx) {
if record.Score > 5000.0 {
break
}
}
}
}
func BenchmarkManualIterator_EarlyTermination(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
iter := collection.NewRecordIterator()
for {
record, ok := iter.Next()
if !ok {
break
}
if record.Score > 5000.0 {
break
}
}
iter.Close()
}
}
// Scenario 3: Building collections - the dangerous pattern for iterators
func BenchmarkIterator_BuildCollection(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var ids []string
for record := range collection.IterRecords() {
ids = append(ids, record.ID) // This causes string escapes!
}
_ = ids
}
}
func BenchmarkIteratorAny_BuildCollection(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var items []any
for record := range collection.IterRecordsAny() {
items = append(items, record) // Interface boxing disaster!
}
_ = items
}
}
// Scenario 4: Iterator composition performance
func BenchmarkIterator_Composition(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Chain: filter -> take -> collect
highScores := Take(
Filter(collection.IterRecords(), func(r Record) bool {
return r.Score > 15000.0
}),
10,
)
results := Collect(highScores)
_ = results
}
}
func BenchmarkCallback_Composition(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var results []Record
count := 0
collection.ForEachRecord(func(record Record) bool {
if record.Score > 15000.0 {
results = append(results, record)
count++
if count >= 10 {
return false
}
}
return true
})
_ = results
}
}
// Scenario 5: Parallel iteration - tests concurrent safety
func BenchmarkIterator_Parallel(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
count := 0
for record := range collection.IterRecords() {
count++
_ = record.Score // Simulate processing
if count > 100 { // Process subset to reduce contention
break
}
}
}
})
}
func BenchmarkCallback_Parallel(b *testing.B) {
collection := NewCollection(N)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
count := 0
collection.ForEachRecord(func(record Record) bool {
count++
_ = record.Score
if count > 100 {
return false
}
return true
})
}
})
}
// =============================================================================
// EXAMPLES: Correct Usage Patterns
// =============================================================================
func ExampleCollection_IterRecords() {
collection := NewCollection(5)
// ✅ CORRECT: Simple iteration with early termination (using ordered iteration for deterministic output)
fmt.Println("Finding first record with score > 10:")
for record := range collection.IterRecordsOrdered() {
if record.Score > 10.0 {
fmt.Printf("Found: %s (score: %.2f)\n", record.ID, record.Score)
break
}
}
// ✅ CORRECT: Streaming processing
fmt.Println("\nProcessing all records:")
count := 0
sum := 0.0
for record := range collection.IterRecordsOrdered() {
count++
sum += record.Score
}
fmt.Printf("Processed %d records, average score: %.2f\n", count, sum/float64(count))
// Output:
// Finding first record with score > 10:
// Found: item_00004 (score: 12.57)
//
// Processing all records:
// Processed 5 records, average score: 6.28
}
func Example_iteratorComposition() {
collection := NewCollection(100)
// ✅ CORRECT: Iterator composition (using ordered iteration for deterministic output)
fmt.Println("Top 3 high-scoring records:")
highScores := Take(
Filter(collection.IterRecordsOrdered(), func(r Record) bool {
return r.Score > 100.0
}),
3,
)
for record := range highScores {
fmt.Printf("- %s: %.2f\n", record.ID, record.Score)
}
// Output:
// Top 3 high-scoring records:
// - item_00032: 100.53
// - item_00033: 103.67
// - item_00034: 106.81
}
func Example_contextCancellation() {
collection := NewCollection(1000)
// Create a context that cancels after 1ms
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
defer cancel()
fmt.Println("Iterator with context cancellation:")
count := 0
for record := range collection.IterRecordsWithContext(ctx) {
count++
_ = record.Score // Use the record to prevent "unused variable" error
// Simulate some processing time
time.Sleep(100 * time.Microsecond)
if count >= 1000 { // This probably won't be reached due to timeout
break
}
}
fmt.Printf("Processed %d records before context cancellation\n", count)
// Output will vary based on timing, showing fewer than 1000 records due to context timeout
}
// =============================================================================
// GOTCHA TESTS: Common Mistakes and How to Avoid Them
// =============================================================================
func TestRangeVariableCapture_GOTCHA(t *testing.T) {
collection := NewCollection(3)
// NOTE: Go 1.22+ fixed the range variable capture issue automatically!
// This test demonstrates the OLD behavior and shows it's now fixed
t.Run("FixedInModernGo", func(t *testing.T) {
var wg sync.WaitGroup
var results []string
var mu sync.Mutex
for record := range collection.IterRecords() {
wg.Add(1)
go func() { // In Go 1.22+, this now works correctly!
defer wg.Done()
mu.Lock()
results = append(results, record.ID)
mu.Unlock()
}()
}
wg.Wait()
t.Logf("Results in Go 1.22+: %v", results)
// In modern Go, these should be different (the fix worked!)
unique := make(map[string]bool)
for _, id := range results {
unique[id] = true
}
if len(unique) >= 2 {
t.Logf("✅ Range variable capture is FIXED in Go 1.22+: got %d unique IDs", len(unique))
} else {
t.Logf("⚠️ Only got %d unique IDs - might be running on older Go version", len(unique))
}
})
// ✅ BEST PRACTICE: Always pass parameters explicitly for clarity
t.Run("ExplicitParameterPassing", func(t *testing.T) {
var wg sync.WaitGroup
var results []string
var mu sync.Mutex
for record := range collection.IterRecords() {
wg.Add(1)
go func(r Record) { // ← EXPLICIT: Always pass as parameter
defer wg.Done()
mu.Lock()
results = append(results, r.ID)
mu.Unlock()
}(record) // ← Pass the current value explicitly
}
wg.Wait()
t.Logf("EXPLICIT results: %v", results)
unique := make(map[string]bool)
for _, id := range results {
unique[id] = true
}
if len(unique) < 2 {
t.Errorf("Expected different record IDs, but got: %v", results)
}
t.Logf("✅ Success: Got %d unique record IDs with explicit parameters", len(unique))
})
}
func TestInterfaceBoxing_GOTCHA(t *testing.T) {
collection := NewCollection(100)
// ✅ GOOD: Typed iterator
t.Run("TypedIterator_Fast", func(t *testing.T) {
start := time.Now()
count := 0
for record := range collection.IterRecords() {
count++
_ = record.ID
}
typedDuration := time.Since(start)
t.Logf("✅ Typed iterator: %v for %d records", typedDuration, count)
})
// ❌ SLOW: Interface{} iterator causes boxing
t.Run("InterfaceIterator_Slow", func(t *testing.T) {
start := time.Now()
count := 0
for record := range collection.IterRecordsAny() {
count++
r := record.(Record) // Type assertion required
_ = r.ID
}
interfaceDuration := time.Since(start)
t.Logf("❌ Interface iterator: %v for %d records", interfaceDuration, count)
t.Logf("⚠️ Interface version forces boxing - every value becomes interface{}")
})
}
func TestCollectionBuilding_GOTCHA(t *testing.T) {
collection := NewCollection(1000)
// ❌ ANTI-PATTERN: Building collections defeats streaming purpose
t.Run("AntiPattern_CollectThenProcess", func(t *testing.T) {
var allIDs []string
for record := range collection.IterRecords() {
allIDs = append(allIDs, record.ID)
}
count := 0
for _, id := range allIDs {
if len(id) > 0 {
count++
}
}
t.Logf("❌ Anti-pattern: Collected %d IDs then processed them", len(allIDs))
t.Logf("⚠️ This uses extra memory and defeats streaming benefits")
})
// ✅ GOOD: Stream processing
t.Run("GoodPattern_StreamProcessing", func(t *testing.T) {
count := 0
for record := range collection.IterRecords() {
if len(record.ID) > 0 {
count++
}
}
t.Logf("✅ Good pattern: Streamed and processed %d records", count)
t.Logf("✅ Zero extra allocations, constant memory usage")
})
}
func TestIteratorBreakBehavior_GOTCHA(t *testing.T) {
collection := NewCollection(1000)
t.Run("IteratorStopsOnBreak", func(t *testing.T) {
count := 0
start := time.Now()
for record := range collection.IterRecords() {
count++
if record.Score > 50.0 { // Should find this quickly
break
}
}
duration := time.Since(start)
t.Logf("✅ Iterator stopped after %d iterations in %v", count, duration)
if count > 100 {
t.Errorf("Expected early termination, but processed %d records", count)
}
})
t.Run("ChannelMayLeakGoroutine", func(t *testing.T) {
count := 0
start := time.Now()
ctx := context.Background()
for record := range collection.RecordsChanWithContext(ctx) {
count++
if record.Score > 50.0 {
break // With context, goroutine will eventually stop
}
}
duration := time.Since(start)
t.Logf("✅ Channel with context: processed %d records in %v", count, duration)
t.Logf("✅ Context cancellation prevents resource leaks")
})
}
// =============================================================================
// UNIT TESTS: Verify Correctness of All Patterns
// =============================================================================
func TestAllIterationPatterns_Correctness(t *testing.T) {
const testSize = 100
collection := NewCollection(testSize)
allRecords := collection.GetAllRecordsSlice()
expectedCount := len(allRecords)
expectedSum := 0.0
for _, r := range allRecords {
expectedSum += r.Score
}
patterns := []struct {
name string
test func() (int, float64)
}{
{
"Iterator_Pattern",
func() (int, float64) {
count := 0
sum := 0.0
for record := range collection.IterRecords() {
count++
sum += record.Score
}
return count, sum
},
},
{
"CopyAll_Pattern",
func() (int, float64) {
count := 0
sum := 0.0
records := collection.GetAllRecordsSlice()
for _, record := range records {
count++
sum += record.Score
}
return count, sum
},
},
{
"Callback_Pattern",
func() (int, float64) {
count := 0
sum := 0.0
collection.ForEachRecord(func(record Record) bool {
count++
sum += record.Score
return true
})
return count, sum
},
},
{
"Channel_Pattern",
func() (int, float64) {
count := 0
sum := 0.0
ctx := context.Background()
for record := range collection.RecordsChanWithContext(ctx) {
count++
sum += record.Score
}
return count, sum
},
},
{
"ManualIterator_Pattern",
func() (int, float64) {
count := 0
sum := 0.0
iter := collection.NewRecordIterator()
defer iter.Close()
for {
record, ok := iter.Next()
if !ok {
break
}
count++
sum += record.Score
}
return count, sum
},
},
}
for _, pattern := range patterns {
t.Run(pattern.name, func(t *testing.T) {
count, sum := pattern.test()
if count != expectedCount {
t.Errorf("Expected %d records, got %d", expectedCount, count)
}
if !floatEqual(sum, expectedSum, 0.01) { // Use tolerance for float comparison
t.Errorf("Expected sum %.2f, got %.2f (diff: %.6f)", expectedSum, sum, math.Abs(sum-expectedSum))
}
t.Logf("✅ %s: %d records, sum %.2f", pattern.name, count, sum)
})
}
}
func TestIteratorComposition_Correctness(t *testing.T) {
collection := NewCollection(100)
// Test Filter + Take composition
t.Run("FilterAndTake", func(t *testing.T) {
results := Collect(Take(
Filter(collection.IterRecords(), func(r Record) bool {
return r.Score > 50.0
}),
5,
))
if len(results) > 5 {
t.Errorf("Expected at most 5 results, got %d", len(results))
}
for _, r := range results {
if r.Score <= 50.0 {
t.Errorf("Expected all scores > 50.0, got %.2f", r.Score)
}
}
t.Logf("✅ Composition: got %d filtered results", len(results))
})
}
func TestZeroAllocations_Verification(t *testing.T) {
collection := NewCollection(100)
t.Run("Iterator_ZeroAllocs", func(t *testing.T) {
allocs := testing.AllocsPerRun(10, func() {
count := 0
sum := 0.0
for record := range collection.IterRecords() {
count++
sum += record.Score
if len(record.ID) == 0 {
count--
}
}
_ = count
_ = sum
})
t.Logf("Iterator allocations per run: %.2f", allocs)
if allocs > 0 {
t.Errorf("Expected zero allocations, but got %.2f allocs per run", allocs)
} else {
t.Logf("✅ Confirmed: Zero allocations achieved!")
}
})
t.Run("CollectionBuilding_ManyAllocs", func(t *testing.T) {
allocs := testing.AllocsPerRun(5, func() {
var ids []string
for record := range collection.IterRecords() {
ids = append(ids, record.ID)
}
_ = ids
})
t.Logf("Collection building allocations per run: %.2f", allocs)
if allocs == 0 {
t.Errorf("Expected multiple allocations for collection building")
} else {
t.Logf("⚠️ Confirmed: Collection building causes %.2f allocations per run", allocs)
}
})
}
// =============================================================================
// PERFORMANCE SUMMARY & RECOMMENDATIONS
// =============================================================================
/*
🏆 PERFORMANCE ANALYSIS:
⚠️ IMPORTANT DISCLAIMERS:
• Results vary significantly based on CPU, memory, OS, and Go version
• These patterns are for RELATIVE comparison only
• Always benchmark in YOUR target environment
• Focus on performance RATIOS, not absolute numbers
🔍 TYPICAL PERFORMANCE CHARACTERISTICS:
SIMPLE PROCESSING:
• Iterator: Excellent performance, zero allocations
• Copy-All: ~2x slower, significant memory overhead
• Callback: Similar to iterator performance
• Channel: Much slower due to goroutine overhead
• Manual: Slower due to key copying and re-locking
EARLY TERMINATION:
• Iterator: Excellent - stops immediately when condition met
• Copy-All: Poor - still copies entire collection first
• Callback: Excellent - stops immediately
• Channel: Poor - goroutine may continue running
• Manual: Poor - pays key copying cost upfront
MEMORY EFFICIENCY:
• Iterator: Zero allocations for streaming
• Copy-All: High memory usage (full collection copy)
• Callback: Zero allocations
• Channel: Goroutine and channel overhead
• Manual: Key copying allocation overhead
=========================================================================================================
🎯 WHEN TO USE EACH PATTERN:
🟢 USE GO 1.23 ITERATORS WHEN:
• You need streaming access to large collections
• Early termination scenarios are common
• Memory efficiency is important
• You want familiar for-range syntax with good performance
• You're building libraries that others will consume
• Context cancellation support is needed
• You want composable, chainable operations
🟡 USE CALLBACK PATTERNS WHEN:
• You're stuck on Go < 1.23 and need maximum performance
• You don't mind the unfamiliar syntax
🔴 AVOID THESE PATTERNS:
• Copy-everything for large collections (poor performance, high memory)
• Channel-based iteration (much slower, resource management complexity)
• Manual iterator structs (verbose, error-prone, slower)
• interface{} in iterator signatures (boxing overhead, excessive allocations)
⚠️ COMMON PITFALLS TO AVOID:
1. DON'T use interface{} types in iterators
2. DON'T build collections from iterators unless necessary
3. DO pass loop variables explicitly to goroutines for clarity
4. DO use escape analysis (-gcflags="-m") to verify zero allocations
5. DO prefer streaming processing over collect-then-process
🏆 THE VERDICT: Go 1.23 Range-over-Function Iterators
Excellent balance of:
✅ Performance (better than alternatives)
✅ Memory efficiency (zero allocations)
✅ Developer experience (familiar syntax)
✅ Composability (works with break, continue, defer, etc.)
✅ Type safety (when used correctly)
The Go team delivered a feature that makes the right thing the performant thing.
========================================================================================================
🚀 REPRODUCTION INSTRUCTIONS:
1. Save this code as iterator_benchmark_test.go
2. Requires Go 1.23+ for iter package support
3. Run: go test -bench=. -benchmem
4. Run examples: go test -run=Example -v
5. Run gotcha tests: go test -run=GOTCHA -v
6. See escape analysis: go test -bench=. -benchmem -gcflags="-m"
7. Verify zero allocations: go test -run=ZeroAllocations -v
Adjust the N constant to test with different collection sizes.
Results will vary by hardware - focus on relative performance ratios.
This benchmark represents realistic usage patterns found in production Go applications.
Happy iterating! 🚀
*/
@rhnvrm
Copy link
Author

rhnvrm commented Jul 8, 2025

Results:

➜  go test -bench=. -benchmem
goos: linux
goarch: amd64
pkg: foo/gistv4
cpu: 13th Gen Intel(R) Core(TM) i7-1355U
BenchmarkIterator_SimpleProcessing-12              16669             69160 ns/op               0 B/op          0 allocs/op
BenchmarkCopyAll_SimpleProcessing-12                7426            144076 ns/op          327682 B/op          1 allocs/op
BenchmarkCallback_SimpleProcessing-12              17739             67212 ns/op               0 B/op          0 allocs/op
BenchmarkChannel_SimpleProcessing-12                1866            536351 ns/op            3618 B/op          3 allocs/op
BenchmarkManualIterator_SimpleProcessing-12         1884            704271 ns/op           81968 B/op          2 allocs/op
BenchmarkIterator_EarlyTermination-12           26608862                39.78 ns/op            0 B/op          0 allocs/op
BenchmarkCopyAll_EarlyTermination-12                8391            142869 ns/op          327680 B/op          1 allocs/op
BenchmarkCallback_EarlyTermination-12           31593428                38.39 ns/op            0 B/op          0 allocs/op
BenchmarkChannel_EarlyTermination-12              142581              7255 ns/op            4208 B/op          5 allocs/op
BenchmarkManualIterator_EarlyTermination-12         2336            507635 ns/op           81968 B/op          2 allocs/op
BenchmarkIterator_BuildCollection-12                8144            249802 ns/op          665968 B/op         18 allocs/op
BenchmarkIteratorAny_BuildCollection-12             2847            461670 ns/op          985968 B/op      10018 allocs/op
BenchmarkIterator_Composition-12                 1726719               671.3 ns/op           992 B/op          5 allocs/op
BenchmarkCallback_Composition-12                 1847635               586.9 ns/op           992 B/op          5 allocs/op
BenchmarkIterator_Parallel-12                    8033890               153.3 ns/op             0 B/op          0 allocs/op
BenchmarkCallback_Parallel-12                    6859370               178.4 ns/op             0 B/op          0 allocs/op
PASS
ok      foo/gistv4      26.741s

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment