Last active
November 7, 2025 10:17
-
-
Save wjkoh/26b95df96c161603d8b297977034d58b to your computer and use it in GitHub Desktop.
Go: BufferedIterator uses a goroutine and a buffered channel to pre-fetch the next page of data while the current page is being processed, optimizing for slow network calls.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // BufferedIterator returns an iterator over data pages. | |
| // | |
| // This implementation uses a goroutine and a buffered channel | |
| // to pre-fetch the next page of data while the current page | |
| // is being processed, optimizing for slow network calls. | |
| // | |
| // This version starts the producer goroutine *lazily* (only when | |
| // iteration begins) to prevent resource leaks if the | |
| // iterator is created but never used. | |
| func BufferedIterator(ctx context.Context, releaseID int) iter.Seq2[*Data, error] { | |
| // A private struct to pass data+error over the channel | |
| type Result struct { | |
| Data *Data | |
| Err error | |
| } | |
| // The returned function *is* the iterator (the consumer) | |
| // We move all setup logic inside this function. | |
| return func(yield func(*Data, error) bool) { | |
| // 1. Setup resources lazily | |
| // Create a new context that we can cancel when the | |
| // consumer (iterator) stops. | |
| ctx, cancel := context.WithCancel(ctx) | |
| // When the consumer is done (e.g., 'break' or 'return' | |
| // from the loop), cancel the context to stop the | |
| // producer goroutine. This is crucial for cleanup. | |
| defer cancel() | |
| // Use a buffered channel of size 1. | |
| resultCh := make(chan Result, 1) | |
| // 2. Start the producer goroutine lazily | |
| go func() { | |
| // Ensure the channel is closed when the goroutine exits, | |
| // signaling the end of iteration to the consumer. | |
| defer close(resultCh) | |
| var nextCursor string | |
| for { | |
| // 1. Fetch the next page of data. | |
| o, err := getDataOverHTTP(ctx, releaseID, nextCursor) | |
| // 2. Send the result (or error) to the consumer. | |
| // This will block if the channel is full (i.e., consumer | |
| // is still processing the previous page). | |
| select { | |
| case resultCh <- Result{Data: o, Err: err}: | |
| // Result sent successfully. | |
| case <-ctx.Done(): | |
| // Context was canceled (e.g., consumer stopped), | |
| // so exit the goroutine. | |
| return | |
| } | |
| // 3. Stop the producer if an error occurred. | |
| if err != nil { | |
| return | |
| } | |
| // 4. Stop the producer if there are no more pages. | |
| if !o.HasMore { | |
| return | |
| } | |
| // 5. Prepare for the next iteration. | |
| nextCursor = o.NextCursor | |
| } | |
| }() | |
| // 3. Run the consumer logic | |
| // Read from the channel until it's closed by the producer. | |
| for result := range resultCh { | |
| // Yield the page and its error to the consumer. | |
| // If yield returns false, the consumer is done, | |
| // so we stop iterating and the 'defer cancel()' | |
| // will clean up the goroutine. | |
| if !yield(result.Data, result.Err) { | |
| return | |
| } | |
| // If an error was yielded, we don't need to explicitly | |
| // return. The producer goroutine will have already | |
| // returned (and closed the channel), so this | |
| // 'for range' loop will terminate on its next iteration. | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment