Created
September 5, 2025 22:16
-
-
Save cowboyd/128fe29143c1c1cf891c7e6eb4735b2d to your computer and use it in GitHub Desktop.
Lazy JSONL reader in Effection
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import { | |
| call, | |
| each, | |
| main, | |
| resource, | |
| sleep, | |
| type Stream, | |
| until, | |
| } from "@effection/effection"; | |
| import { TextLineStream } from "@std/streams"; | |
| import { JsonParseStream, type JsonValue } from "@std/json"; | |
| export function useJSONLFile<T extends JsonValue>( | |
| path: string, | |
| ): Stream<T, void> { | |
| return resource(function* (provide) { | |
| const file = yield* call(() => Deno.open(path, { read: true })); | |
| const lines = file | |
| .readable | |
| .pipeThrough(new TextDecoderStream()) | |
| .pipeThrough(new TextLineStream()) | |
| .pipeThrough(new JsonParseStream()); | |
| const reader = lines.getReader(); | |
| try { | |
| yield* provide({ | |
| *next() { | |
| console.log('reading next line of json'); | |
| let next = yield* until(reader.read()); | |
| if (next.done) { | |
| return { done: true, value: void (0) }; | |
| } else { | |
| return { done: false, value: next.value as T}; | |
| } | |
| }, | |
| }); | |
| } finally { | |
| console.log('cleaning up...'); | |
| reader.releaseLock(); | |
| //file.close(); | |
| } | |
| }); | |
| } | |
| await main(function*() { | |
| for (const json of yield* each(useJSONLFile("./hack.jsonl"))) { | |
| console.log({ json }); | |
| yield* sleep(5000); | |
| yield* each.next(); | |
| } | |
| }) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment