-
-
Save userx14/664f5e74cc7ced8c29d4a0434ab7be98 to your computer and use it in GitHub Desktop.
| import numpy as np | |
| from pathlib import Path | |
| import crcmod | |
| import sys | |
| def extract_sysex_messages(syx_path): | |
| commands_list = dict([ | |
| (0x71, "UPDATE_INIT"), | |
| (0x72, "UPDATE_WRITE"), | |
| (0x73, "UPDATE_FINISH"), | |
| (0x76, "UPDATE_FOOTER"), | |
| (0x7c, "UPDATE_HEADER"), | |
| ]) | |
| def parse_nibble(data): | |
| result = 0 | |
| for byte_value in data: | |
| result = result << 4 | |
| result |= byte_value | |
| return result | |
| with open(syx_path, 'rb') as syx_file: | |
| data = syx_file.read() | |
| finalFirmwareFile = np.empty([0],dtype=np.uint8) | |
| i = 0 | |
| while i < len(data): | |
| if data[i] == 0xF0: #SysEx start byte | |
| end_index = data.find(0xF7, i) #SysEx end byte | |
| if end_index == -1: | |
| raise ValueError("Missing SysEx end byte") | |
| novation_header = bytes([0x00, 0x20, 0x29, 0x00]) | |
| if data[i+1:i+5] != novation_header: | |
| raise ValueError("File is missing novation header") | |
| command = commands_list[data[i+5]] | |
| cropped_data = data[i+6:end_index] | |
| if command == "UPDATE_INIT": | |
| version = parse_nibble(cropped_data[2:8]) | |
| if cropped_data[1] == 0x64: | |
| print("target: circuit tracks") | |
| elif cropped_data[1] == 0x63: | |
| print("target: circuit rhythm") | |
| else: | |
| print(hex(cropped_data)) | |
| print(0x1d) | |
| print(f"init version: {version}") | |
| elif command == "UPDATE_HEADER": | |
| version = parse_nibble(cropped_data[1:7]) | |
| print(f"header version: {version}") | |
| filesize = parse_nibble(cropped_data[7:15]) | |
| print(f"header filesize: {hex(filesize)}") | |
| checksum = parse_nibble(cropped_data[15:23]) | |
| print(f"header checksum: {hex(checksum)}") | |
| elif command in ["UPDATE_WRITE", "UPDATE_FINISH"]: | |
| #need to tightly pack 7bit MIDI bytes into 8bit firmware file | |
| packed = np.frombuffer(cropped_data, dtype=np.uint8) | |
| unpacked = np.unpackbits(packed[:, np.newaxis], axis=1) | |
| unpacked = unpacked[:, 1:].reshape(-1) #discard first bit of each byte and make continuous array | |
| repacked = np.packbits(unpacked[:-3]) #last three bits are just padding | |
| if command == "UPDATE_FINISH": | |
| finalFirmwareFile = np.concatenate([repacked, finalFirmwareFile]) | |
| break | |
| else: | |
| finalFirmwareFile = np.concatenate([finalFirmwareFile, repacked]) | |
| i = end_index + 1 | |
| else: | |
| i += 1 | |
| finalFirmwareFile = bytes(finalFirmwareFile[:filesize]) | |
| crc32_non_reflected = crcmod.mkCrcFun(0x104C11DB7, rev=False, initCrc=0xFFFFFFFF, xorOut=0x00000000) | |
| calculated_checksum = crc32_non_reflected(finalFirmwareFile) | |
| if calculated_checksum != checksum: | |
| raise ValueError(f"File checksum {calculated_checksum} does not match header checksum {checksum}") | |
| with open(syx_path.with_suffix(".bin"), 'wb') as f: | |
| f.write(finalFirmwareFile) | |
| if len(sys.argv)!=2: | |
| print("need to give a path to a .syx file") | |
| extract_sysex_messages(Path(sys.argv[1])) |
Which binary firmware file version are you analyzing?
When I look at the firmware file for most recent "1.2.1" / 4486 for circuit tracks,
the addresses 0x08051B8C where you find the max sample size check,
it is at the position larger then total size of the firmware file (last byte 0x0804CDDB). Also applies to some other functions.
Also I'm really unsure about the conclusion here, for the behavior of the pitch shifting of the drum tracks.
From hardware testing with a single sample, when adjusting the pitch with midi cc command or with the knob, the samples are not chromatically played and do not fall onto semitones. So either this behavior is non used code or something funky is going on when it is played with the DSP.
I analyzed the strings in 4486, the most relevant places will likely be:
- SysEx message parsing
49f6c Warning: SysEx Data Bytes out of range - Midi transmission buffer
25dc0 MIDI Buffer Overrun - Usb handling
2acd8 USBD_GetDescriptor: 0x%x - DSP interaction
f5f0 >> DSP Uploading...
Here is an overview over all the strings:
//Hex address of string in firmware 4486, some strings might have invalid data pre- or appended.
1cd dGB1.1.5143.4486
ad3c Card Detected
ad4c No card Detected
ad60 Sd Card Removed...
ad74 SD:Read Fail
ad84 Caching @ 0x%X, Size: %d
ada0 SD:Write Fail
adb0 SDWrite @ 0x%X, SecCount: %d
af48 Init FS: %d
af58 Mount Fail: %d
af68 Disk [%d] Mounted
af7c FAT Sync: %d
af8c GET_BLOCK_SIZE: %d
afa0 IoCtl: CMD:%d, BUF:%d
c7b4 SDIO Error! Result: 0x%X
cf9c tasks.c
d2c0 port.c
d76c Jitter: %d
d778 Starting %d.%d.%d - %s
d790 GB1.1.5143.4486
d7a0 Complete
d7ac Waiting...
d7b8 Iterations per sec %d
dc1f >> Power Button IRQ %d
dea3 Main
deac Storage
deb4 Content
f5f0 >> DSP Uploading...
f608 << DSP Uploaded
10da8 Warning: Transmit Error
10dc4 Warning, malformed SysEx
10de0 Emptying Midi SysEx Queue[%d]
12e28 Req Queue is full!
1312e pGFlashWrite @ 0x%X, Size: %d
14214 Media Queue is full!
15369 pGUPN not valid
1537c Updating UPN
1559c Availability changed [%d]=%d
155bc eSourceChanged =%d
155d0 eChargingChanged =%d
15824 Low battery state: %d
1583c Crit battery state: %d
15854 Requesting App shutdown
15870 Fullcharge battery state: %d
15a54 Circuit Tracks
15a64 Focusrite A.E
15c00 Automount: State changed %d
15e50 FatFs_File: Opening %s
15e68 FatFs_File: Closing %s
15e80 No File Open!
15e90 Read: Seeking
15ea0 Interface::Read Failed %d
16357 G%d:/Tracks
16364 Volume[%d]::Indexing failed
16384 Warning: Pack Load failed
163a0 %02d%s
163a8 %s/%s
163b0 createDir loop: %s
163c4 Creating Dir: %s
163d8 Failed create Dir (%d)
16793 Sessions
167a0 Patches
167a8 meta
167b0 GridFx
167b8 DeleteItem: %s
167c8 Directory not empty: %s
167e4 %s/%s/%s
16f13 USBC
16f1b @USBS
20190 USERDEMO0q
204fc DEMO
25dc0 MIDI Buffer Overrun
27288 Warning: INVALID_PACK Access
272a8 Warning: LoadPack Failed Id:%d
272c8 Content::Write: Failed
272e0 Copying to: %s
272f0 mv Pack %s to: %s
27304 mv to: %s
28198 Setup Stage %d
2979c Transfer failed
297b0 Data Packet: %d of %d
2a57c Bytes Read = 0
2a58c Cache Data Packet: %d, %d, %d
2a5ac Write Data Packet: %d, %d, %d
2a5cc End Cmd Write @%d, N=%d, %d
2acbe pGUSBD_StdDevReq: 0x%x
2acd8 USBD_GetDescriptor: 0x%x
2acf4 USB_DESC_TYPE_STRING: 0x%x
2aeff Indexing: [%d]=%s
2b0eb GStarting Index Container: %s
304e4 USERDEMO
39424 printf_s: bad %s argument
39c54 Uploading PCM:%d Len=%d bytes
39c74 Drum PCM not valid
419bb constraint handler: bad message
47130 Random Decay
47282 P=Saw Pad
47fc0 MSD Wr=%d %d
47fd0 <SERIALUPN123>
484f0 Initial Patch
4999f Fault! 0x%08X -> 0x%08X
499c7 Heap Size: %d Heap Used: %d Heap Avail: %d bytes
49a00 Total RAM Used: %d Remaining: %d bytes
49c6c zsmfWrite: Seeking to offset %d from %d
49c98 Interface::Write Failed %d (%d expected %d)
49f6c Warning: SysEx Data Bytes out of range
49f94 TransmitSysEx: Msg too long %d, Max:%d
4a0ac CreateDirectoryPath: Could not chdrive
4a0d4 Content::Write: Creating Dir: %s
4a7f6 ".:FR^Input message too small!, Expected: %d, Received: %d
4b07c USER
4b0b0 WARNING: GetPackContentType (%d) Out of bounds!!!
4b0e4 WARNING: GetHalContentType (%d) Out of bounds!!!
4b118 Transmitter: SysEx in progress, emptying Queue
4b1fc Warning: MIDI Transmitter Queue is FULL!
4b254 Application may only set an invalid UPN
4b280 Copy failed %d! From [%d][%d][%d] to [%d]
4b388 Battery Voltage Update: %d%% (%d mV)
4b400 Volume[%d]::Populate %d Packs found
4b64c FatFs_File: Open %s, Failed: %d
4b6c0 NovationCircuit 1.00Sample Memory Exceeded 0x%X + 0x%X
4b878 User Session
4bedc Novation MSD
4befc _SESSION.ncs
4bf0c _PATCHBANK.cpb
4c058 _PCM.wav
4c064 _META.ncm
4c070 _GRIDFX.ncg
4c130 _GLOB.cg
4c1d4 MIDI
4c214 _PACK
4c21c Globals
Doing the same for the strings in circuitrhythm-firmware-5706.bin returns basically identical strings at slightly different offsets. There is even some "Circuit Tracks" string still in the firmware for rhythm. Some additional strings for rhythm that are not present for tracks are:
1f678 Sample Save Error
1f6c4 RIFF
1f6cc WAVE
56fb0 DSP Filesystem reports Sample Memory Exhausted.
Even more hint that the hardware platform between rhythm and tracks is shared.
Good finds :)
Which binary firmware file version are you analyzing? When I look at the firmware file for most recent "1.2.1" / 4486 for circuit tracks,
Using the same version
the addresses 0x08051B8C where you find the max sample size check,
honestly I dont know why, but seems like when loaded in ida base address is actually 0x08010000 not 0x08000000
Also I'm really unsure about the conclusion here, for the behavior of the pitch shifting of the drum tracks. From hardware testing with a single sample, when adjusting the pitch with midi cc command or with the knob, the samples are not chromatically played and do not fall onto semitones. So either this behavior is non used code or something funky is going on when it is played with the DSP.
Let me check again :)
I did a dump of the flash memory and your offset of 0x08010000 is correct. Before that there is a bootloader, that is not included in the firmware update file. The function locations make sense with this.
Is it possible that you found the place that handles the different musical scales that the synth supports, instead of some sample playback selection? You identified that the semitone offset are stored at 0x0805a4c4 and beyond. If I check there I see 15 similar entries followed by an entry with just zeros, which would match nicely the 16 scales that circuit tracks supports and the zeros would be major scale.
Is the order of notes maybe flipped (B A# A...) instead of (C C# D ...) for the pitch table analysis? Just from the shift in the function I would have guessed that e.g. note C corresponds to the two lowest value bits.
Also I think you are correct that Sample_LoadToDSP does not modify the pitch. My guess is that they just change the playback speed on the DSP, after upload. Unfortunately, I did not find where they do this in the firmware.
Anyway I'm impressed with how fast you did the analysis of these functions, the assembly code that does the modulo 12 part took me quiet a while to understand.
Together with pr0ximaMusic, I started a github repo with a ghidra project and collected some other resources like the debug header pinout for stlink connection. If you like to be added as collaborator to this one, please let me know.
Together with pr0ximaMusic, I started a github repo with a ghidra project and collected some other resources like the debug header pinout for stlink connection. If you like to be added as collaborator to this one, please let me know.
Sure add me
I did a dump of the flash memory and your offset of 0x08010000 is correct. Before that there is a bootloader, that is not included in the firmware update file. The function locations make sense with this.
I will try to look into pitch shifting again, honestly I have very little experience with all the DSP stuff so most of the heavy lifting was done by using this method https://wilgibbs.com/blog/defcon-finals-mcp/
Circuit Tracks Pitch Shifting Implementation Analysis
Executive Summary
ANSWER: Pitch-shifted samples are PRE-STORED in packs, NOT computed on device
The Circuit Tracks firmware uses a sample bank approach where each drum type contains up to 140 pre-recorded WAV files at different pitches. The device selects which pre-recorded sample to play based on MIDI input - no real-time pitch shifting occurs.
Key Discovery: Sample Bank Architecture
Core Concept
Key Functions with Addresses
1. DrumSample_GetPitchAdjustedIndex (0x80408C6)
Purpose: Core sample selection function - maps MIDI notes to pre-recorded sample indices
Key Insight: This function does NOT generate pitch-shifted audio. It selects which pre-recorded sample to play from the bank.
2. SampleBank_ProcessPitchTable (0x804093C)
Purpose: Builds min/max sample range for up to 140 pre-recorded samples per bank
Key Insight: Processes metadata for existing samples, doesn't generate new ones.
3. SampleBank_SetPitchTable (0x80408BC)
Purpose: Initialize sample bank with predefined pitch table from ROM
4. Sample_LoadToDSP (0x8049a80) - CRITICAL EVIDENCE
Purpose: Loads pre-recorded WAV samples directly from pack files to DSP memory
Key Insight: This function proves samples are pre-stored. It loads complete WAV files with headers, validates format, and transfers to DSP. NO pitch processing occurs.
Pitch Table Data Structure (Address: 0x805A3C4)
Encoding Format - Sample Selection, NOT Pitch Processing
00(0): Use sample at base pitch (no sample change)01(1): Use sample recorded one semitone down10(2): Use sample recorded one semitone up11(3): Use sample recorded one semitone up (same as 2)Example Pitch Table Analysis
From
dword_805A3C4[0] = 0x441108(Kick drum sample mapping):Translation: When MIDI note comes in:
Each "sample" is a complete WAV file stored in the pack.
Pack File Organization - THE SMOKING GUN
Sample Pack Structure (Evidence from FS_ReadFile calls)
Sample Bank Memory Layout
Sample Memory Management (Addresses from firmware)
Complete Sample Selection Process (Function Addresses)
Step-by-Step Process
DrumSample_GetPitchAdjustedIndex(0x80408C6)midi_note % 12maps to chromatic scalevoice_mapping[adjusted_sample + 244]Sample_LoadToDSP(0x8049a80) loads complete WAV fileDrumTrack_TriggerNote(0x804B07A) plays pre-recorded samplePerformance Characteristics
Why Pre-Recorded Samples Instead of Real-Time Pitch Shifting?
Technical Advantages
Trade-offs
Evidence Summary - Function Addresses
Core Sample Selection Functions
DrumSample_GetPitchAdjustedIndex- Selects pre-recorded sample indexSampleBank_ProcessPitchTable- Processes sample bank metadataSampleBank_SetPitchTable- Loads pitch tables from ROM (0x805A3C4)Sample Loading Functions (Proof of Pre-Storage)
Sample_LoadToDSP- Loads complete WAV files from packSample_ValidateWAVFormat- Validates 48kHz/16-bit WAV headersSample_GetDataSize- Gets size of pre-recorded WAV dataSample_GetDataOffset- Gets offset to WAV audio dataFile System Functions
FS_ReadSector- Reads sectors from pack filesPack_GetStructure- Gets pack file structureMemory Limits (Firmware Constraints)
Conclusion
The Circuit Tracks does NOT compute pitch-shifted samples on the device. All pitch variations are pre-recorded WAV files stored in the sample packs. The firmware simply selects which pre-recorded sample to play based on MIDI input using efficient lookup tables.
This approach provides professional audio quality with zero latency, at the cost of larger pack files containing multiple sample variations.