Skip to content

Instantly share code, notes, and snippets.

@geowurster

geowurster/parser.py

Last active October 24, 2015 04:04
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save geowurster/c3d566955fd442cf1638 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save geowurster/c3d566955fd442cf1638 to your computer and use it in GitHub Desktop.
Download ZIP
libais parsing
Raw
parser.py
import ais.tag_block
import ais.vdm
import ais
import json
import lru # pip install lru-dict
import collections
FIELD_MAP = {
'positional_accuracy': {'tag': 'body', 'key': 'accuracy'},
: 'addressed',
: 'aid_type',
'ais_version': {'tag': 'body', 'key': 'ais_version'},
: 'alt',
: 'app_id',
: 'assigned',
: 'band',
: 'band_a',
: 'band_b',
'callsign': {'tag': 'body', 'key': 'callsign'},
: 'channel_a',
: 'channel_b',
'cog': {'tag': 'body', 'key': 'course'},
: 'cs',
'ack_dac': {'tag': 'body', 'key': 'dac'},
: 'data',
'day': {'tag': 'body', 'key': 'day'},
: 'dest1',
: 'dest2',
'mmsi_dst': {'tag': 'body', 'key': 'dest_mmsi'},
'destination': {'tag': 'body', 'key': 'destination'},
: 'display',
'draught': {'tag': 'body', 'key': 'draught'},
: 'dsc',
'dte': {'tag': 'body', 'key': 'dte'},
: 'epfd',
'req_fi': {'tag': 'body', 'key': 'fid'},
: 'gnss',
'true_heading': {'tag': 'body', 'key': 'heading'},
'hour': {'tag': 'body', 'key': 'hour'},
'imo_num': {'tag': 'body', 'key': 'imo'},
: 'increment1',
: 'increment2',
: 'increment3',
: 'increment4',
: 'interval',
'y': {'tag': 'body', 'key': 'lat'},
'x' : {'tag': 'body', 'key': 'lon'},
'special_manoeuvre': {'tag': 'body', 'key': 'maneuver'},
'minute': {'tag': 'body', 'key': 'minute'},
'mmsi': {'tag': 'body', 'key': 'mmsi'},
'spare': {'tag': 'spare', 'key': 'spare'},
: 'mmsi1',
: 'mmsi2',
: 'mmsi3',
: 'mmsi4',
: 'mmsiseq1',
: 'mmsiseq2',
: 'mmsiseq3',
: 'mmsiseq4',
: 'model',
'month': {'tag': 'body', 'key': 'month'},
: 'mothership_mmsi',
: 'msg22',
: 'name',
: 'ne_lat',
: 'ne_lon',
: 'number1',
: 'number2',
: 'number3',
: 'number4',
: 'off_position',
: 'offset1',
: 'offset1_1',
: 'offset1_2',
: 'offset2',
: 'offset2_1',
: 'offset3',
: 'offset4',
: 'partno',
: 'power',
: 'quiet',
'radio': {'tag': 'body', 'key': 'radio'},
'raim': {'tag': 'body', 'key': 'raim'},
: 'regional',
'repeat': {'tag': 'body', 'key': 'repeat'},
: 'reserved',
'retransmit': {'tag': 'body', 'key': 'retransmit'},
'timestamp': {'tag': 'body', 'key': 'second'},
'msg_seq': {'tag': 'body', 'key': 'seqno'},
: 'serial',
: 'ship_type',
'name': {'tag': 'body', 'key': 'shipname'},
'type_and_cargo': {'tag': 'body', 'key': 'shiptype'},
'sog': {'tag': 'body', 'key': 'speed'},
: 'station_type',
'nav_status': {'tag': 'body', 'key': 'status'},
: 'structured',
: 'sw_lat',
: 'sw_lon',
: 'text',
: 'timeout1',
: 'timeout2',
: 'timeout3',
: 'timeout4',
'dim_a': {'tag': 'body', 'key': 'to_bow'},
'to_port': {'tag': 'body', 'key': 'to_port'},
'dim_d': {'tag': 'body', 'key': 'to_starboard'},
'dim_b': {'tag': 'body', 'key': 'to_stern'},
'rot': {'tag': 'body', 'key': 'turn'},
: 'txrx',
'id': {'tag': 'body', 'key': 'type'},
: 'type1_1',
: 'type1_2',
: 'type2_1',
: 'vendorid',
: 'virtual_aid',
'year': {'tag': 'body', 'key': 'year'},
: 'zonesize:
}
# TODO: Are dim_a|b|c|d mapped correctly?
def d(t):
return json.dumps(t, indent=4, sort_keys=True)
CACHE = lru.LRU(1000)
def in_cache(val):
try:
CACHE[val]
return True
except KeyError:
return False
with open('100k.nmea') as f:
for line in f:
# We can always parse these
tagblock = ais.tag_block.Parse(line)
payload = ais.vdm.Parse(tagblock['payload'])
sen_tot = int(payload['sen_tot'])
# Single line message. Parse!
if sen_tot is 1:
try:
body = ais.decode(payload['body'], int(payload['fill_bits']))
msg = {
'tagblock': tagblock,
'payload': payload,
'body': body
}
except ais.DecodeError:
msg = None
print("======= ERRROR SINGLE =======")
else:
sen_num = int(payload['sen_num'])
seq_id = int(payload['seq_id'])
# Sequence not in CACHE
if not in_cache(seq_id):
CACHE[seq_id] = {sen_num: {'tagblock': tagblock, 'payload': payload}}
else:
CACHE[seq_id][sen_num] = {'tagblock': tagblock, 'payload': payload}
# Last message in a sequence
if sen_num is sen_tot:
sen_data = CACHE[seq_id]
# Manual dump is cheaper than LRU
del CACHE[seq_id]
# Build the NMEA sentence
nmea = ''
for k in sorted(sen_data.keys()):
nmea += sen_data[k]['payload']['body']
# Parse NMEA
try:
body = ais.decode(nmea, payload['fill_bits'])
msg = {
'tagblock': tagblock,
'payload': payload,
'body': body
}
except ais.DecodeError:
msg = None
print("======= ERRROR MULTI =======")
# Only processed part of a multiline message - keep going
continue
print(d(msg))
gpsd = {
'type': ['id'],
'course': ['cog'],
'mmsi': ['mmsi'],
'status': ['nav_status'],
'turn': ['rot'],
'lon': ['x'],
'lat': ['y'],
'speed': ['sog'],
'repeat': ['repeat'],
'heading': ['true_heading'],
'raim': ['raim'],
'maneuver': ['special_manoeuvre'],
'spare': ['spare'],
'second': ['timestamp'],
'accuracy': ['position_accuracy'],
'body': {
'rot_over_range': false,
'slot_timeout': 1,
'sync_state': 0,
'utc_hour': 23,
'utc_min': 59,
'utc_spare': 0,
},
'payload': {
'body': '15SimR001Q6jmk9pwgp<Rr1n06sd',
'chan': 'A',
'checksum': '66',
'fill_bits': 0,
'sen_num': 1,
'sen_tot': 1,
'seq_id': null,
'talker': 'AI',
'vdm': '!AIVDM,1,1,,A,15SimR001Q6jmk9pwgp<Rr1n06sd,0*66',
'vdm_type': 'VDM"
},
'tagblock': {
'dest': null,
'group': null,
'group_id': null,
'line_num': null,
'metadata': 's:rORBCOMM109,q:u,c:1420070400,T:2015-01-01 00.00.00*57',
'payload': '!AIVDM,1,1,,A,15SimR001Q6jmk9pwgp<Rr1n06sd,0*66',
'quality': 'u',
'rcvr': 'rORBCOMM109',
'rel_time': null,
'sentence_num': null,
'sentence_tot': null,
'tag_checksum': '57',
'text': null,
'text_date': '2015-01-01 00.00.00',
'time': 1420070400
}
}
break
FIELDS_BY_TYPE = {
'1': [
'accuracy', 'course', 'heading', 'lat', 'lon', 'maneuver', 'mmsi',
'radio', 'raim', 'repeat', 'second', 'speed', 'status', 'turn', 'type'
],
'2': [
'accuracy', 'course', 'heading', 'lat', 'lon', 'maneuver', 'mmsi',
'radio', 'raim', 'repeat', 'second', 'speed', 'status', 'turn', 'type'
],
'3': [
'accuracy', 'course', 'heading', 'lat', 'lon', 'maneuver', 'mmsi',
'radio', 'raim', 'repeat', 'second', 'speed', 'status', 'turn', 'type'
],
'4': [
'accuracy', 'day', 'epfd', 'hour', 'lat', 'lon', 'minute', 'mmsi',
'month', 'radio', 'raim', 'repeat', 'second', 'type', 'year'
],
'5': [
'ais_version', 'callsign', 'day', 'destination', 'draught', 'dte',
'epfd', 'hour', 'imo', 'minute', 'mmsi', 'month', 'repeat', 'shipname',
'shiptype', 'to_bow', 'to_port', 'to_starboard', 'to_stern', 'type'
],
'6': [
'dac', 'data', 'day', 'dest_mmsi', 'destination', 'draught', 'dte',
'epfd', 'fid', 'hour', 'minute', 'mmsi', 'month', 'repeat',
'retransmit', 'seqno', 'to_port', 'to_starboard', 'type'
],
'7': [
'day', 'destination', 'draught', 'dte', 'epfd', 'hour', 'minute',
'mmsi', 'mmsi1', 'mmsi2', 'mmsi3', 'mmsi4', 'mmsiseq1', 'mmsiseq2',
'mmsiseq3', 'mmsiseq4', 'month', 'repeat', 'type'
],
'8': [
'assigned', 'course', 'dac', 'data', 'destination', 'draught', 'dte',
'dte', 'fid', 'lat', 'minute', 'mmsi', 'radio', 'raim', 'regional',
'repeat', 'second', 'type'
],
'9': [
'accuracy', 'alt', 'assigned', 'course', 'destination', 'draught',
'dte', 'dte', 'lat', 'lon', 'minute', 'mmsi', 'radio', 'raim',
'regional', 'repeat', 'second', 'speed', 'type'
],
'10': [
'assigned', 'course', 'dest_mmsi', 'destination', 'draught', 'dte',
'dte', 'lat', 'lon', 'minute', 'mmsi', 'radio', 'raim', 'regional',
'repeat', 'second', 'type'
],
'11': [
'accuracy', 'day', 'epfd', 'hour', 'lat', 'lon', 'minute', 'mmsi',
'month', 'radio', 'raim', 'repeat', 'second', 'type', 'year'
],
'12': [
'assigned', 'course', 'dest_mmsi', 'destination', 'draught', 'dte',
'dte', 'minute', 'mmsi', 'radio', 'raim', 'regional', 'repeat',
'retransmit', 'second', 'seqno', 'text', 'type'
],
'13': [
'day', 'destination', 'draught', 'dte', 'epfd', 'hour', 'minute',
'mmsi', 'mmsi1', 'mmsi2', 'mmsi3', 'mmsi4', 'mmsiseq1', 'mmsiseq2',
'mmsiseq3', 'mmsiseq4', 'month', 'repeat', 'type'
],
'14': [
'assigned', 'course', 'destination', 'draught', 'dte', 'dte', 'minute',
'mmsi', 'radio', 'raim', 'regional', 'repeat', 'retransmit', 'second',
'text', 'text', 'type'
],
'15': [
'destination', 'draught', 'dte', 'minute', 'mmsi', 'mmsi1', 'mmsi2',
'offset1_1', 'offset1_2', 'offset2_1', 'radio', 'repeat', 'type',
'type1_1', 'type1_2', 'type2_1'
],
'16': [
'destination', 'draught', 'dte', 'increment1', 'minute', 'mmsi',
'mmsi1', 'mmsi2', 'mmsi2', 'offset1', 'offset2', 'offset2_1', 'radio',
'repeat', 'type', 'type2_1'
],
'17': [
'data', 'lat', 'lon', 'mmsi', 'repeat', 'type'
],
'18': [
'accuracy', 'assigned', 'band', 'course', 'cs', 'display', 'dsc',
'heading', 'lat', 'lon', 'mmsi', 'msg22', 'radio', 'raim', 'regional',
'repeat', 'reserved', 'second', 'speed', 'type'
],
'19': [
'accuracy', 'assigned', 'course', 'dte', 'epfd', 'heading', 'lat',
'lon', 'mmsi', 'raim', 'regional', 'repeat', 'reserved', 'second',
'shipname', 'shiptype', 'speed', 'to_bow', 'to_port', 'to_starboard',
'to_stern', 'type'
],
'20': [
'assigned', 'dte', 'increment1', 'increment2', 'increment3',
'increment4', 'mmsi', 'number1', 'number2', 'number3', 'number4',
'offset1', 'offset2', 'offset3', 'offset4', 'repeat', 'timeout1',
'timeout2', 'timeout3', 'timeout4', 'type'
],
'21': [
'accuracy', 'aid_type', 'assigned', 'assigned', 'epfd', 'lat', 'lon',
'mmsi', 'name', 'off_position', 'raim', 'regional', 'repeat', 'second',
'to_bow', 'to_port', 'to_starboard', 'to_stern', 'type', 'virtual_aid'
],
'22': [
'addressed', 'assigned', 'band_a', 'band_b', 'channel_a', 'channel_b',
'dest1', 'dest2', 'mmsi', 'ne_lat', 'ne_lon', 'power', 'repeat',
'sw_lat', 'sw_lon', 'txrx', 'type', 'zonesize'
],
'23': [
'assigned', 'band_a', 'band_b', 'interval', 'mmsi', 'ne_lat', 'ne_lon',
'quiet', 'repeat', 'ship_type', 'station_type', 'sw_lat', 'sw_lon', 'txrx',
'type', 'zonesize'
],
'24': [
'assigned', 'callsign', 'mmsi', 'model', 'mothership_mmsi', 'partno',
'repeat', 'serial', 'shipname', 'shiptype', 'to_bow', 'to_port',
'to_starboard', 'to_stern', 'type', 'vendorid', 'zonesize'
],
'25': [
'addressed', 'app_id', 'assigned', 'callsign', 'data', 'dest_mmsi',
'mmsi', 'model', 'mothership_mmsi', 'repeat', 'serial', 'structured',
'to_bow', 'to_port', 'to_starboard', 'to_stern', 'type', 'zonesize'
],
'26': [
'addressed', 'app_id', 'assigned', 'callsign', 'data', 'dest_mmsi',
'mmsi', 'mothership_mmsi', 'radio', 'repeat', 'serial', 'structured',
'to_bow', 'to_port', 'to_starboard', 'to_stern', 'type', 'zonesize'
],
'27': [
'accuracy', 'assigned', 'course', 'gnss', 'lat', 'lon', 'mmsi',
'mothership_mmsi', 'raim', 'repeat', 'speed', 'status', 'to_port',
'to_starboard', 'to_stern', 'type', 'zonesize'
]
}
TYPE_BY_FIELD = collections.defaultdict(set)
for t, f in FIELDS_BY_TYPE.items():
TYPE_BY_FIELD[f].add(f)
TYPE_BY_FIELD = {k: tuple(sorted(v)) for k, v in TYPE_BY_FIELD.items()}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment