Created
May 19, 2025 14:25
-
-
Save dodbrian/651d6b78477a0b4cecdf50c6da7c2dbb to your computer and use it in GitHub Desktop.
Splits a long file created by git2gpt into smaller chunks
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env node | |
| import { createWriteStream, createReadStream } from 'fs'; | |
| import { basename, extname } from 'path'; | |
| import { createInterface } from 'readline'; | |
| const args = process.argv.slice(2); | |
| if (args.length !== 2) { | |
| console.error('Usage: slicer <input_file> <target_size_mb>'); | |
| process.exit(1); | |
| } | |
| const inputFile = args[0]; | |
| const targetSizeMB = parseFloat(args[1]); | |
| const targetSizeBytes = targetSizeMB * 1024 * 1024; | |
| if (isNaN(targetSizeMB) || targetSizeMB <= 0) { | |
| console.error('Error: Target size must be a positive number.'); | |
| process.exit(1); | |
| } | |
| const baseFileName = basename(inputFile, extname(inputFile)); | |
| let fileCounter = 1; | |
| let currentOutputSize = 0; | |
| let currentOutputStream = null; | |
| const openNewOutputFile = () => { | |
| if (currentOutputStream) { | |
| currentOutputStream.end(); | |
| } | |
| const outputFileName = `${baseFileName}.${fileCounter}${extname(inputFile)}`; | |
| currentOutputStream = createWriteStream(outputFileName); | |
| currentOutputSize = 0; | |
| console.log(`Created ${outputFileName}`); | |
| fileCounter++; | |
| }; | |
| const rl = createInterface({ | |
| input: createReadStream(inputFile), | |
| crlfDelay: Infinity | |
| }); | |
| openNewOutputFile(); | |
| let previousLineWasDelimiterStart = false; | |
| rl.on('line', (line) => { | |
| const lineSize = Buffer.from(line + '\n', 'utf8').length; | |
| const isDelimiterStart = line.trim() === '----'; | |
| const isDelimiterEnd = previousLineWasDelimiterStart; | |
| previousLineWasDelimiterStart = isDelimiterStart; | |
| // If the current line is the second line of a delimiter, | |
| // and there's content in the current file, | |
| // and the current file size is greater than or equal to the target size, | |
| // open a new file. | |
| if (isDelimiterEnd && currentOutputSize > 0 && currentOutputSize >= targetSizeBytes) { | |
| openNewOutputFile(); | |
| } | |
| currentOutputStream.write(line + '\n'); | |
| currentOutputSize += lineSize; | |
| }); | |
| rl.on('close', () => { | |
| if (currentOutputStream) { | |
| currentOutputStream.end(); | |
| } | |
| console.log('File slicing complete.'); | |
| }); | |
| rl.on('error', (err) => { | |
| console.error(`Error reading file line by line: ${err.message}`); | |
| if (currentOutputStream) { | |
| currentOutputStream.end(); | |
| } | |
| process.exit(1); | |
| }); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment