Skip to content

Instantly share code, notes, and snippets.

@Michael-J-Ward

Michael-J-Ward/checksum.py

Created March 2, 2019 21:07
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Michael-J-Ward/fdaad16ffe56e94fcaa7e90910ddf1d2 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Michael-J-Ward/fdaad16ffe56e94fcaa7e90910ddf1d2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
checksum.py
import zlib
import itertools
def get_signed_value(unsigned_value):
bitsize = 32
return unsigned_value if unsigned_value < (1 << bitsize-1) else unsigned_value - (1 << bitsize)
def create_checksum_string(bids, asks):
bids = (f"{price}:{qty}" for price, qty, _, _ in bids)
asks = (f"{price}:{qty}" for price, qty, _, _ in asks)
merged = [item
for pair in itertools.zip_longest(bids, asks)
for item in pair
if item is not None]
return ":".join(merged)
def check_checksum(item):
bids = item['bids'][:25]
asks = item['asks'][:25]
string = create_checksum_string(bids, asks)
checksum = item['checksum']
unsigned = zlib.crc32(string.encode('utf8'))
signed = get_signed_value(unsigned)
return signed == checksum
def update_side(old_levels, new_levels, reverse):
level_map = {k[0]: k for k in old_levels}
for update in new_levels:
price, qty, _, _ = update
if qty == 0:
level_map.pop(price, None)
else:
level_map[price] = update
return [level_map[k] for k in sorted(level_map.keys(), reverse=reverse)]
# These examples taken from https://www.okex.com/docs/en/#futures_ws-checksum
even_example = {
"instrument_id": "BTC-USD-170310",
"asks": [[3366.8, 9, 10, 3],[ 3368,8,3,4 ]],
"bids": [
[3366.1, 7, 0, 3],[ 3366,6,3,4 ]
],
"timestamp": "2018-12-04T09:38:36.300Z",
"checksum": -1881014294
}
assert check_checksum(even_example)
odd_example = {
"instrument_id": "BTC-USD-170310",
"asks": [[3366.8, 9, 10, 3],[ 3368,8,3,4 ],[ 3372,8,3,4 ]],
"bids": [[3366.1, 7, 0, 3]],
"timestamp": "2018-12-04T09:38:36.300Z",
"checksum": 831078360
}
assert check_checksum(odd_example)
increment_example = {
"table": "futures/depth",
"action": "update",
"data": [{
"instrument_id": "BTC-USD-SWAP",
"asks": [],
"bids": [
[3983, 789, 0, 3]
],
"timestamp": "2018-12-04T09:38:36.300Z",
"checksum": -1200119424
}]
}
# this fails
assert check_checksum(increment_example)
# actual first book update, this fails
message_200_levels = {'instrument_id': 'BTC-USD-190329',
'asks': [[3782.85, 95, 0, 3],
[3782.88, 53, 0, 1],
[3782.9, 39, 0, 1],
[3783.06, 92, 0, 4],
[3783.09, 40, 0, 1],
[3783.5, 17, 0, 1],
[3783.89, 100, 0, 1],
[3783.91, 3, 0, 1],
[3783.95, 5, 0, 2],
[3783.96, 131, 0, 3],
[3783.97, 77, 0, 2],
[3783.99, 28, 0, 2],
[3784, 45, 0, 4],
[3784.01, 100, 0, 1],
[3784.02, 22, 0, 1],
[3784.06, 2, 0, 1],
[3784.1, 9, 0, 2],
[3784.13, 33, 0, 1],
[3784.18, 15, 0, 1],
[3784.19, 7, 0, 1],
[3784.22, 5, 0, 1],
[3784.28, 11, 0, 1],
[3784.37, 8, 0, 1],
[3784.38, 8, 0, 1],
[3784.4, 8, 0, 1],
[3784.41, 11, 0, 1],
[3784.46, 100, 0, 1],
[3784.54, 17, 0, 2],
[3784.55, 7, 0, 0],
[3784.64, 3, 0, 2],
[3784.69, 25, 0, 1],
[3784.73, 8, 0, 1],
[3784.74, 16, 0, 2],
[3784.81, 8, 0, 1],
[3784.87, 10, 0, 1],
[3784.91, 7, 0, 1],
[3784.96, 16, 0, 2],
[3785, 1, 0, 1],
[3785.06, 40, 0, 2],
[3785.17, 100, 0, 1],
[3785.21, 8, 0, 1],
[3785.24, 8, 0, 1],
[3785.32, 20, 0, 2],
[3785.37, 12, 0, 1],
[3785.38, 2, 0, 1],
[3785.44, 7, 0, 1],
[3785.45, 16, 0, 3],
[3785.49, 15, 0, 2],
[3785.52, 1, 0, 1],
[3785.55, 3, 0, 1],
[3785.56, 5, 0, 2],
[3785.6, 8, 0, 0],
[3785.61, 42, 0, 2],
[3785.68, 3, 0, 1],
[3785.75, 100, 0, 2],
[3785.79, 5, 0, 1],
[3785.86, 8, 0, 1],
[3785.89, 8, 0, 1],
[3785.93, 7, 0, 1],
[3786, 1, 0, 1],
[3786.01, 2, 0, 1],
[3786.04, 9, 0, 1],
[3786.08, 6, 0, 1],
[3786.24, 8, 0, 1],
[3786.28, 50, 0, 2],
[3786.37, 10, 0, 1],
[3786.5, 2, 0, 1],
[3786.54, 43, 0, 1],
[3786.56, 10, 0, 1],
[3786.62, 40, 0, 1],
[3786.79, 2, 0, 1],
[3786.82, 13, 0, 1],
[3786.83, 16, 0, 1],
[3786.88, 12, 0, 1],
[3786.89, 2, 0, 1],
[3786.94, 16, 0, 2],
[3786.95, 2, 0, 1],
[3786.99, 30, 0, 1],
[3787, 18, 0, 4],
[3787.02, 1, 0, 1],
[3787.09, 22, 0, 1],
[3787.14, 30, 0, 2],
[3787.16, 2, 0, 1],
[3787.26, 17, 0, 1],
[3787.29, 5, 0, 1],
[3787.34, 2, 0, 1],
[3787.37, 40, 0, 1],
[3787.39, 1, 0, 1],
[3787.46, 33, 0, 2],
[3787.5, 2, 0, 1],
[3787.51, 2, 0, 1],
[3787.54, 16, 0, 1],
[3787.58, 6, 0, 1],
[3787.63, 74, 0, 1],
[3787.68, 4, 0, 0],
[3787.82, 9, 0, 1],
[3787.83, 2, 0, 1],
[3787.88, 89, 0, 1],
[3787.92, 2, 0, 1],
[3788, 4, 0, 3],
[3788.02, 8, 0, 1],
[3788.15, 11, 0, 2],
[3788.24, 30, 0, 1],
[3788.34, 1, 0, 1],
[3788.39, 30, 0, 1],
[3788.5, 2, 0, 1],
[3788.54, 10, 0, 1],
[3788.6, 13, 0, 1],
[3788.65, 22, 0, 2],
[3788.66, 2, 0, 1],
[3788.69, 2, 0, 1],
[3788.71, 1, 0, 1],
[3788.73, 2, 0, 1],
[3788.78, 35, 0, 3],
[3788.8, 1, 0, 1],
[3789, 2, 0, 1],
[3789.07, 5, 0, 1],
[3789.09, 1, 0, 1],
[3789.16, 1, 0, 1],
[3789.35, 8, 0, 1],
[3789.36, 6, 0, 1],
[3789.5, 2, 0, 1],
[3789.84, 2, 0, 1],
[3790, 39, 0, 2],
[3790.08, 10, 0, 1],
[3790.48, 1, 0, 1],
[3790.5, 2, 0, 1],
[3790.58, 27, 0, 1],
[3790.95, 60, 0, 2],
[3790.98, 50, 0, 1],
[3791, 402, 0, 2],
[3791.1, 50, 0, 1],
[3791.16, 40, 0, 1],
[3791.18, 30, 0, 1],
[3791.42, 30, 0, 1],
[3791.45, 12, 0, 1],
[3791.47, 1, 0, 1],
[3791.5, 2, 0, 1],
[3791.76, 15, 0, 1],
[3791.77, 40, 0, 1],
[3791.8, 20, 0, 1],
[3791.83, 20, 0, 1],
[3792, 2, 0, 1],
[3792.07, 24, 0, 1],
[3792.5, 2, 0, 1],
[3792.64, 24, 0, 1],
[3792.69, 30, 0, 1],
[3792.72, 1, 0, 1],
[3792.84, 40, 0, 1],
[3792.9, 37, 0, 1],
[3792.95, 7, 0, 1],
[3793, 2, 0, 1],
[3793.06, 40, 0, 1],
[3793.17, 1, 0, 1],
[3793.21, 33, 0, 2],
[3793.33, 40, 0, 1],
[3793.5, 12, 0, 2],
[3793.51, 40, 0, 1],
[3793.53, 260, 0, 1],
[3793.76, 50, 0, 1],
[3793.82, 80, 0, 1],
[3794, 26, 0, 2],
[3794.1, 20, 0, 1],
[3794.23, 20, 0, 1],
[3794.5, 2, 0, 1],
[3794.57, 24, 0, 1],
[3794.74, 80, 0, 1],
[3795, 123, 0, 6],
[3795.24, 49, 0, 1],
[3795.3, 44, 0, 1],
[3795.5, 2, 0, 1],
[3795.75, 100, 0, 1],
[3796, 12, 0, 2],
[3796.49, 13, 0, 1],
[3796.5, 2, 0, 1],
[3796.58, 2, 0, 1],
[3796.7, 20, 0, 1],
[3796.92, 150, 0, 1],
[3797, 109, 0, 4],
[3797.26, 100, 0, 1],
[3797.3, 270, 0, 1],
[3797.31, 100, 0, 1],
[3797.33, 18, 0, 1],
[3797.35, 1, 0, 1],
[3797.5, 2, 0, 1],
[3798, 30, 0, 3],
[3798.1, 11, 0, 1],
[3798.5, 2, 0, 1],
[3798.83, 100, 0, 1],
[3799, 108, 0, 2],
[3799.1, 37, 0, 1],
[3799.12, 30, 0, 2],
[3799.5, 2, 0, 1],
[3799.54, 74, 0, 1],
[3800, 1353, 0, 18],
[3800.1, 50, 0, 1],
[3800.34, 1, 0, 1],
[3800.5, 2, 0, 1],
[3800.76, 15, 0, 1],
[3800.97, 8, 0, 2]],
'bids': [[3782.52, 2, 0, 1],
[3782.45, 1, 0, 1],
[3782.2, 41, 0, 2],
[3782.18, 5, 0, 1],
[3782.12, 3, 0, 1],
[3782.11, 1, 0, 1],
[3782, 1, 0, 1],
[3781.97, 1, 0, 1],
[3781.83, 1, 0, 1],
[3781.66, 1, 0, 1],
[3781.64, 7, 0, 1],
[3781.63, 7, 0, 1],
[3781.36, 14, 0, 1],
[3781.35, 2, 0, 1],
[3781.29, 50, 0, 1],
[3781.27, 8, 0, 1],
[3781.19, 15, 0, 1],
[3781.16, 5, 0, 1],
[3781.12, 1, 0, 1],
[3781.05, 10, 0, 1],
[3781.03, 1, 0, 1],
[3781.02, 2, 0, 1],
[3781.01, 13, 0, 1],
[3781, 11, 0, 3],
[3780.95, 1, 0, 1],
[3780.94, 35, 0, 1],
[3780.86, 9, 0, 1],
[3780.73, 3, 0, 2],
[3780.7, 1, 0, 1],
[3780.68, 5, 0, 1],
[3780.67, 100, 0, 2],
[3780.55, 100, 0, 1],
[3780.53, 10, 0, 1],
[3780.5, 2, 0, 1],
[3780.29, 27, 0, 1],
[3780.18, 29, 0, 1],
[3780.15, 1, 0, 1],
[3780.05, 23, 0, 1],
[3780, 3, 0, 2],
[3779.85, 2, 0, 1],
[3779.8, 7, 0, 1],
[3779.55, 10, 0, 1],
[3779.5, 2, 0, 1],
[3779.49, 11, 0, 1],
[3779.48, 33, 0, 1],
[3779.36, 20, 0, 2],
[3779.24, 2, 0, 1],
[3779.22, 2, 0, 1],
[3779.18, 20, 0, 1],
[3779.17, 1, 0, 1],
[3779.16, 31, 0, 1],
[3779.14, 24, 0, 1],
[3779.01, 2, 0, 1],
[3779, 15, 0, 3],
[3778.9, 5, 0, 1],
[3778.79, 10, 0, 1],
[3778.66, 2, 0, 1],
[3778.6, 8, 0, 1],
[3778.58, 28, 0, 2],
[3778.52, 43, 0, 1],
[3778.5, 2, 0, 1],
[3778.42, 30, 0, 2],
[3778.31, 24, 0, 1],
[3778.27, 8, 0, 2],
[3778.25, 18, 0, 1],
[3778.22, 9, 0, 1],
[3778.16, 90, 0, 4],
[3778.14, 40, 0, 1],
[3778, 3, 0, 2],
[3777.81, 20, 0, 1],
[3777.8, 34, 0, 1],
[3777.55, 1, 0, 1],
[3777.53, 22, 0, 1],
[3777.5, 2, 0, 1],
[3777.45, 2, 0, 1],
[3777.35, 1, 0, 1],
[3777.06, 30, 0, 1],
[3777.03, 40, 0, 1],
[3777, 23, 0, 3],
[3776.95, 1, 0, 1],
[3776.93, 30, 0, 1],
[3776.82, 89, 0, 1],
[3776.66, 20, 0, 1],
[3776.58, 74, 0, 1],
[3776.5, 2, 0, 1],
[3776.45, 1, 0, 1],
[3776.39, 50, 0, 1],
[3776.36, 6, 0, 1],
[3776.09, 37, 0, 2],
[3776, 303, 0, 3],
[3775.94, 4, 0, 1],
[3775.84, 8, 0, 1],
[3775.69, 30, 0, 1],
[3775.64, 1, 0, 1],
[3775.6, 30, 0, 1],
[3775.56, 40, 0, 1],
[3775.5, 2, 0, 1],
[3775.42, 30, 0, 1],
[3775.4, 56, 0, 1],
[3775.23, 15, 0, 1],
[3775.16, 1, 0, 1],
[3775.14, 40, 0, 1],
[3775, 60, 0, 4],
[3774.96, 16, 0, 1],
[3774.7, 16, 0, 1],
[3774.5, 2, 0, 1],
[3774.44, 31, 0, 2],
[3774.34, 1, 0, 1],
[3774.09, 30, 0, 1],
[3774, 2, 0, 1],
[3773.9, 30, 0, 1],
[3773.88, 20, 0, 1],
[3773.72, 38, 0, 1],
[3773.69, 2, 0, 1],
[3773.57, 17, 0, 2],
[3773.52, 40, 0, 1],
[3773.5, 2, 0, 1],
[3773.31, 7, 0, 1],
[3773.02, 30, 0, 1],
[3773, 208, 0, 4],
[3772.69, 758, 0, 1],
[3772.54, 24, 0, 1],
[3772.5, 2, 0, 1],
[3772.38, 30, 0, 1],
[3772.16, 1, 0, 1],
[3772, 58, 0, 4],
[3771.91, 20, 0, 1],
[3771.7, 277, 0, 1],
[3771.61, 27, 0, 1],
[3771.59, 24, 0, 1],
[3771.5, 2, 0, 1],
[3771.35, 40, 0, 1],
[3771.34, 40, 0, 1],
[3771.3, 11, 0, 1],
[3771.27, 2, 0, 1],
[3771.21, 24, 0, 1],
[3771.05, 30, 0, 1],
[3771, 24, 0, 5],
[3770.95, 17, 0, 1],
[3770.86, 7, 0, 1],
[3770.85, 20, 0, 1],
[3770.83, 40, 0, 2],
[3770.81, 1, 0, 1],
[3770.72, 150, 0, 1],
[3770.67, 80, 0, 1],
[3770.5, 2, 0, 1],
[3770.37, 10, 0, 1],
[3770.31, 20, 0, 1],
[3770.13, 6, 0, 1],
[3770.1, 10, 0, 1],
[3770.03, 11, 0, 1],
[3770, 1724, 0, 19],
[3769.91, 24, 0, 1],
[3769.86, 20, 0, 1],
[3769.85, 50, 0, 1],
[3769.5, 2, 0, 1],
[3769.43, 30, 0, 1],
[3769.21, 80, 0, 1],
[3769.02, 49, 0, 1],
[3769, 78, 0, 4],
[3768.8, 6, 0, 1],
[3768.54, 24, 0, 1],
[3768.5, 2, 0, 1],
[3768.38, 3, 0, 1],
[3768.34, 100, 0, 1],
[3768.23, 100, 0, 1],
[3768.02, 230, 0, 1],
[3768, 10, 0, 3],
[3767.94, 11, 0, 1],
[3767.5, 2, 0, 1],
[3767.47, 50, 0, 1],
[3767.37, 1, 0, 1],
[3767.31, 50, 0, 1],
[3767, 265, 0, 2],
[3766.82, 100, 0, 1],
[3766.73, 100, 0, 1],
[3766.5, 2, 0, 1],
[3766.24, 74, 0, 1],
[3766.01, 5, 0, 1],
[3766, 204, 0, 6],
[3765.5, 2, 0, 1],
[3765.47, 50, 0, 1],
[3765.24, 15, 0, 1],
[3765.21, 100, 0, 1],
[3765, 252, 0, 12],
[3764.8, 75, 0, 1],
[3764.59, 10, 0, 1],
[3764.5, 2, 0, 1],
[3764.34, 1, 0, 1],
[3764, 202, 0, 2],
[3763.85, 1500, 0, 1],
[3763.76, 11, 0, 2],
[3763.53, 222, 0, 1],
[3763.5, 2, 0, 1],
[3763, 2, 0, 1],
[3762.5, 2, 0, 1],
[3762, 5, 0, 2],
[3761.5, 2, 0, 1],
[3761, 106, 0, 5],
[3760.9, 11, 0, 1]],
'timestamp': '2019-03-02T17:51:54.418Z',
'checksum': 1276686467}
assert check_checksum(message_200_levels)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment