-
-
Save userx14/664f5e74cc7ced8c29d4a0434ab7be98 to your computer and use it in GitHub Desktop.
| import numpy as np | |
| from pathlib import Path | |
| import crcmod | |
| import sys | |
| def extract_sysex_messages(syx_path): | |
| commands_list = dict([ | |
| (0x71, "UPDATE_INIT"), | |
| (0x72, "UPDATE_WRITE"), | |
| (0x73, "UPDATE_FINISH"), | |
| (0x76, "UPDATE_FOOTER"), | |
| (0x7c, "UPDATE_HEADER"), | |
| ]) | |
| def parse_nibble(data): | |
| result = 0 | |
| for byte_value in data: | |
| result = result << 4 | |
| result |= byte_value | |
| return result | |
| with open(syx_path, 'rb') as syx_file: | |
| data = syx_file.read() | |
| finalFirmwareFile = np.empty([0],dtype=np.uint8) | |
| i = 0 | |
| while i < len(data): | |
| if data[i] == 0xF0: #SysEx start byte | |
| end_index = data.find(0xF7, i) #SysEx end byte | |
| if end_index == -1: | |
| raise ValueError("Missing SysEx end byte") | |
| novation_header = bytes([0x00, 0x20, 0x29, 0x00]) | |
| if data[i+1:i+5] != novation_header: | |
| raise ValueError("File is missing novation header") | |
| command = commands_list[data[i+5]] | |
| cropped_data = data[i+6:end_index] | |
| if command == "UPDATE_INIT": | |
| version = parse_nibble(cropped_data[2:8]) | |
| if cropped_data[1] == 0x64: | |
| print("target: circuit tracks") | |
| elif cropped_data[1] == 0x63: | |
| print("target: circuit rhythm") | |
| else: | |
| print(hex(cropped_data)) | |
| print(0x1d) | |
| print(f"init version: {version}") | |
| elif command == "UPDATE_HEADER": | |
| version = parse_nibble(cropped_data[1:7]) | |
| print(f"header version: {version}") | |
| filesize = parse_nibble(cropped_data[7:15]) | |
| print(f"header filesize: {hex(filesize)}") | |
| checksum = parse_nibble(cropped_data[15:23]) | |
| print(f"header checksum: {hex(checksum)}") | |
| elif command in ["UPDATE_WRITE", "UPDATE_FINISH"]: | |
| #need to tightly pack 7bit MIDI bytes into 8bit firmware file | |
| packed = np.frombuffer(cropped_data, dtype=np.uint8) | |
| unpacked = np.unpackbits(packed[:, np.newaxis], axis=1) | |
| unpacked = unpacked[:, 1:].reshape(-1) #discard first bit of each byte and make continuous array | |
| repacked = np.packbits(unpacked[:-3]) #last three bits are just padding | |
| if command == "UPDATE_FINISH": | |
| finalFirmwareFile = np.concatenate([repacked, finalFirmwareFile]) | |
| break | |
| else: | |
| finalFirmwareFile = np.concatenate([finalFirmwareFile, repacked]) | |
| i = end_index + 1 | |
| else: | |
| i += 1 | |
| finalFirmwareFile = bytes(finalFirmwareFile[:filesize]) | |
| crc32_non_reflected = crcmod.mkCrcFun(0x104C11DB7, rev=False, initCrc=0xFFFFFFFF, xorOut=0x00000000) | |
| calculated_checksum = crc32_non_reflected(finalFirmwareFile) | |
| if calculated_checksum != checksum: | |
| raise ValueError(f"File checksum {calculated_checksum} does not match header checksum {checksum}") | |
| with open(syx_path.with_suffix(".bin"), 'wb') as f: | |
| f.write(finalFirmwareFile) | |
| if len(sys.argv)!=2: | |
| print("need to give a path to a .syx file") | |
| extract_sysex_messages(Path(sys.argv[1])) |
I analyzed the strings in 4486, the most relevant places will likely be:
- SysEx message parsing
49f6c Warning: SysEx Data Bytes out of range - Midi transmission buffer
25dc0 MIDI Buffer Overrun - Usb handling
2acd8 USBD_GetDescriptor: 0x%x - DSP interaction
f5f0 >> DSP Uploading...
Here is an overview over all the strings:
//Hex address of string in firmware 4486, some strings might have invalid data pre- or appended.
1cd dGB1.1.5143.4486
ad3c Card Detected
ad4c No card Detected
ad60 Sd Card Removed...
ad74 SD:Read Fail
ad84 Caching @ 0x%X, Size: %d
ada0 SD:Write Fail
adb0 SDWrite @ 0x%X, SecCount: %d
af48 Init FS: %d
af58 Mount Fail: %d
af68 Disk [%d] Mounted
af7c FAT Sync: %d
af8c GET_BLOCK_SIZE: %d
afa0 IoCtl: CMD:%d, BUF:%d
c7b4 SDIO Error! Result: 0x%X
cf9c tasks.c
d2c0 port.c
d76c Jitter: %d
d778 Starting %d.%d.%d - %s
d790 GB1.1.5143.4486
d7a0 Complete
d7ac Waiting...
d7b8 Iterations per sec %d
dc1f >> Power Button IRQ %d
dea3 Main
deac Storage
deb4 Content
f5f0 >> DSP Uploading...
f608 << DSP Uploaded
10da8 Warning: Transmit Error
10dc4 Warning, malformed SysEx
10de0 Emptying Midi SysEx Queue[%d]
12e28 Req Queue is full!
1312e pGFlashWrite @ 0x%X, Size: %d
14214 Media Queue is full!
15369 pGUPN not valid
1537c Updating UPN
1559c Availability changed [%d]=%d
155bc eSourceChanged =%d
155d0 eChargingChanged =%d
15824 Low battery state: %d
1583c Crit battery state: %d
15854 Requesting App shutdown
15870 Fullcharge battery state: %d
15a54 Circuit Tracks
15a64 Focusrite A.E
15c00 Automount: State changed %d
15e50 FatFs_File: Opening %s
15e68 FatFs_File: Closing %s
15e80 No File Open!
15e90 Read: Seeking
15ea0 Interface::Read Failed %d
16357 G%d:/Tracks
16364 Volume[%d]::Indexing failed
16384 Warning: Pack Load failed
163a0 %02d%s
163a8 %s/%s
163b0 createDir loop: %s
163c4 Creating Dir: %s
163d8 Failed create Dir (%d)
16793 Sessions
167a0 Patches
167a8 meta
167b0 GridFx
167b8 DeleteItem: %s
167c8 Directory not empty: %s
167e4 %s/%s/%s
16f13 USBC
16f1b @USBS
20190 USERDEMO0q
204fc DEMO
25dc0 MIDI Buffer Overrun
27288 Warning: INVALID_PACK Access
272a8 Warning: LoadPack Failed Id:%d
272c8 Content::Write: Failed
272e0 Copying to: %s
272f0 mv Pack %s to: %s
27304 mv to: %s
28198 Setup Stage %d
2979c Transfer failed
297b0 Data Packet: %d of %d
2a57c Bytes Read = 0
2a58c Cache Data Packet: %d, %d, %d
2a5ac Write Data Packet: %d, %d, %d
2a5cc End Cmd Write @%d, N=%d, %d
2acbe pGUSBD_StdDevReq: 0x%x
2acd8 USBD_GetDescriptor: 0x%x
2acf4 USB_DESC_TYPE_STRING: 0x%x
2aeff Indexing: [%d]=%s
2b0eb GStarting Index Container: %s
304e4 USERDEMO
39424 printf_s: bad %s argument
39c54 Uploading PCM:%d Len=%d bytes
39c74 Drum PCM not valid
419bb constraint handler: bad message
47130 Random Decay
47282 P=Saw Pad
47fc0 MSD Wr=%d %d
47fd0 <SERIALUPN123>
484f0 Initial Patch
4999f Fault! 0x%08X -> 0x%08X
499c7 Heap Size: %d Heap Used: %d Heap Avail: %d bytes
49a00 Total RAM Used: %d Remaining: %d bytes
49c6c zsmfWrite: Seeking to offset %d from %d
49c98 Interface::Write Failed %d (%d expected %d)
49f6c Warning: SysEx Data Bytes out of range
49f94 TransmitSysEx: Msg too long %d, Max:%d
4a0ac CreateDirectoryPath: Could not chdrive
4a0d4 Content::Write: Creating Dir: %s
4a7f6 ".:FR^Input message too small!, Expected: %d, Received: %d
4b07c USER
4b0b0 WARNING: GetPackContentType (%d) Out of bounds!!!
4b0e4 WARNING: GetHalContentType (%d) Out of bounds!!!
4b118 Transmitter: SysEx in progress, emptying Queue
4b1fc Warning: MIDI Transmitter Queue is FULL!
4b254 Application may only set an invalid UPN
4b280 Copy failed %d! From [%d][%d][%d] to [%d]
4b388 Battery Voltage Update: %d%% (%d mV)
4b400 Volume[%d]::Populate %d Packs found
4b64c FatFs_File: Open %s, Failed: %d
4b6c0 NovationCircuit 1.00Sample Memory Exceeded 0x%X + 0x%X
4b878 User Session
4bedc Novation MSD
4befc _SESSION.ncs
4bf0c _PATCHBANK.cpb
4c058 _PCM.wav
4c064 _META.ncm
4c070 _GRIDFX.ncg
4c130 _GLOB.cg
4c1d4 MIDI
4c214 _PACK
4c21c Globals
Doing the same for the strings in circuitrhythm-firmware-5706.bin returns basically identical strings at slightly different offsets. There is even some "Circuit Tracks" string still in the firmware for rhythm. Some additional strings for rhythm that are not present for tracks are:
1f678 Sample Save Error
1f6c4 RIFF
1f6cc WAVE
56fb0 DSP Filesystem reports Sample Memory Exhausted.
Even more hint that the hardware platform between rhythm and tracks is shared.
Good finds :)
Which binary firmware file version are you analyzing? When I look at the firmware file for most recent "1.2.1" / 4486 for circuit tracks,
Using the same version
the addresses 0x08051B8C where you find the max sample size check,
honestly I dont know why, but seems like when loaded in ida base address is actually 0x08010000 not 0x08000000
Also I'm really unsure about the conclusion here, for the behavior of the pitch shifting of the drum tracks. From hardware testing with a single sample, when adjusting the pitch with midi cc command or with the knob, the samples are not chromatically played and do not fall onto semitones. So either this behavior is non used code or something funky is going on when it is played with the DSP.
Let me check again :)
I did a dump of the flash memory and your offset of 0x08010000 is correct. Before that there is a bootloader, that is not included in the firmware update file. The function locations make sense with this.
Is it possible that you found the place that handles the different musical scales that the synth supports, instead of some sample playback selection? You identified that the semitone offset are stored at 0x0805a4c4 and beyond. If I check there I see 15 similar entries followed by an entry with just zeros, which would match nicely the 16 scales that circuit tracks supports and the zeros would be major scale.
Is the order of notes maybe flipped (B A# A...) instead of (C C# D ...) for the pitch table analysis? Just from the shift in the function I would have guessed that e.g. note C corresponds to the two lowest value bits.
Also I think you are correct that Sample_LoadToDSP does not modify the pitch. My guess is that they just change the playback speed on the DSP, after upload. Unfortunately, I did not find where they do this in the firmware.
Anyway I'm impressed with how fast you did the analysis of these functions, the assembly code that does the modulo 12 part took me quiet a while to understand.
Together with pr0ximaMusic, I started a github repo with a ghidra project and collected some other resources like the debug header pinout for stlink connection. If you like to be added as collaborator to this one, please let me know.
Together with pr0ximaMusic, I started a github repo with a ghidra project and collected some other resources like the debug header pinout for stlink connection. If you like to be added as collaborator to this one, please let me know.
Sure add me
I did a dump of the flash memory and your offset of 0x08010000 is correct. Before that there is a bootloader, that is not included in the firmware update file. The function locations make sense with this.
I will try to look into pitch shifting again, honestly I have very little experience with all the DSP stuff so most of the heavy lifting was done by using this method https://wilgibbs.com/blog/defcon-finals-mcp/
Which binary firmware file version are you analyzing?
When I look at the firmware file for most recent "1.2.1" / 4486 for circuit tracks,
the addresses
0x08051B8Cwhere you find the max sample size check,it is at the position larger then total size of the firmware file (last byte 0x0804CDDB). Also applies to some other functions.
Also I'm really unsure about the conclusion here, for the behavior of the pitch shifting of the drum tracks.
From hardware testing with a single sample, when adjusting the pitch with midi cc command or with the knob, the samples are not chromatically played and do not fall onto semitones. So either this behavior is non used code or something funky is going on when it is played with the DSP.