pcap_json_converter.py
Go to the documentation of this file.
1 """
2  pcap_json_converter converts TiM781S pcap files to json
3 
4  Usage:
5 
6  pip install scapy
7  pip install pypcapfile
8  pip install python-pcapng
9 
10  python pcap_json_converter.py --pcap_filename=<filepath.pcapng>
11 
12 """
13 
14 import argparse
15 import socket
16 import time
17 import os
18 import sys
19 
20 from pcapng import FileScanner
21 from pcapng.blocks import EnhancedPacket #, InterfaceDescription, SectionHeader
22 
23 import scapy.all
24 import scapy.packet
25 from scapy.layers.l2 import Ether
26 
27 def isJsonPrintable(b):
28  return (b == 0x20) or (b >= 48 and b <= 57) or (b >= 65 and b <= 90) or (b >= 97 and b <= 122)
29 
30 def appendToJsonfile(json_filename, relative_timestamp, payload, pcap_filename):
31  if os.path.isfile(json_filename):
32  json_file = open(json_filename, "a")
33  json_file.write(",\n")
34  else:
35  json_file = open(json_filename, "a")
36  json_file.write("[\n")
37  json_file.write(" {\n")
38  json_file.write(" \"_source\": {\n")
39  json_file.write(" \"layers\": {\n")
40  json_file.write(" \"tcp\": {\n")
41  json_file.write(" \"tcp.analysis\": {\n")
42  json_file.write(" \"tcp.analysis.push_bytes_sent\": \"{}\"\n".format(len(payload)))
43  json_file.write(" },\n")
44  json_file.write(" \"Timestamps\": {\n")
45  json_file.write(" \"tcp.time_relative\": \"{}\"\n".format(relative_timestamp))
46  json_file.write(" },\n")
47  json_file.write(" \"tcp.comment\": \"All entries created by pcap_json_converter, not by Wireshark.\",\n")
48  json_file.write(" \"tcp.producer\": \"{} --pcap_filename={}\",\n".format(os.path.basename(__file__), os.path.basename(pcap_filename)))
49  readable_values = [chr(b) if isJsonPrintable(b) else '.' for b in payload]
50  description = "".join(readable_values)
51  hex_values = [format(b,"02x") for b in payload]
52  hex_payload = ":".join(hex_values)
53  json_file.write(" \"tcp.description\": \"{}\",\n".format(description))
54  json_file.write(" \"tcp.payload\": \"{}\"\n".format(hex_payload))
55  json_file.write(" }\n")
56  json_file.write(" }\n")
57  json_file.write(" }\n")
58  json_file.write(" }")
59  json_file.close()
60 
61 def closeJsonfile(json_filename):
62  if os.path.isfile(json_filename):
63  json_file = open(json_filename, "a")
64  json_file.write("\n]\n")
65  json_file.close()
66 
67 def appendToCppfile(cpp_filename, payload, is_cola_ascii):
68  if os.path.isfile(cpp_filename):
69  cpp_file = open(cpp_filename, "a")
70  else:
71  cpp_file = open(cpp_filename, "a")
72  cpp_file.write(" std::map<std::string, sick_scan::SickLocColaTelegramMsg> emulator_responses = { // emulator responses to driver requests\n")
73  payload_all = [b for b in payload]
74  if is_cola_ascii: # cola ascii: remove 1 leading byte <STX> = 0x02 and remove trailing <ETX> = 0x03
75  payload_unpacked = payload_all[1:-1]
76  else: # cola binary: remove 8 byte 0x02020202 + { 4 byte length } and remove trailing CRC byte
77  payload_unpacked = payload_all[8:-1]
78  # Split command into <type> <name> <parameter>
79  sep_idx1 = 0
80  sep_idx2 = 0
81  for n, b in enumerate(payload_unpacked):
82  if b == 0x20:
83  if sep_idx1 == 0:
84  sep_idx1 = n # first space separates command type
85  else:
86  sep_idx2 = n # second space separates parameter
87  break
88  if sep_idx2 == 0: # no parameter if second space is missing
89  sep_idx2 = len(payload_unpacked)
90  cola_type = "".join([chr(b) for b in payload_unpacked[0:sep_idx1]])
91  cola_name = "".join([chr(b) for b in payload_unpacked[sep_idx1+1:sep_idx2]])
92  if sep_idx2+1 < len(payload_unpacked):
93  cola_args = "".join([format(b,"02x") for b in payload_unpacked[sep_idx2+1:]])
94  else:
95  cola_args = ""
96  # Write C++ emulator call
97  cpp_file.write(" {{\"{}\", sick_scan::ColaParser::createColaTelegram(sick_scan::ColaParser::convertSopasCommand(\"{}\"), \"{}\", {{\"{}\"}})}},\n".format(cola_name, cola_type, cola_name, cola_args));
98  cpp_file.close()
99 
100 def closeCppfile(cpp_filename):
101  if os.path.isfile(cpp_filename):
102  cpp_file = open(cpp_filename, "a")
103  cpp_file.write(" };\n")
104  cpp_file.close()
105 
106 # Decodes and returns the payload length of a cola message, i.e. returns message_payload_length in a message := { 4 byte STX 0x02020202 } + { 4 byte message_payload_length } + { message_payload } + { 1 byte CRC }
107 def parseColaPayloadLength(payload):
108  length = 0
109  if len(payload) > 9 and payload.startswith(b'\x02\x02\x02\x02'):
110  length = (payload[4] << 24) + (payload[5] << 16) + (payload[6] << 8) + (payload[7] << 0)
111  return length
112 
113 if __name__ == "__main__":
114 
115  pcap_filename = "example.pcapng"
116  arg_parser = argparse.ArgumentParser()
117  arg_parser.add_argument("--pcap_filename", help="pcapng filepath", default=pcap_filename, type=str)
118  cli_args = arg_parser.parse_args()
119  pcap_filename = cli_args.pcap_filename
120  print("pcap_json_converter {} started.".format(pcap_filename))
121 
122  # Read and parse pcap file, extract tcp raw data
123  cpp_filename = pcap_filename + ".cpp"
124  json_filename = pcap_filename + ".json"
125  if os.path.isfile(cpp_filename):
126  os.remove(cpp_filename)
127  if os.path.isfile(json_filename):
128  os.remove(json_filename)
129  start_timestamp = -1.0
130  payload = b''
131  payload_completed = False
132  cola_ascii = False # default: cola binary
133  with open(pcap_filename, 'rb') as pcap_file:
134  pcap_scanner = FileScanner(pcap_file)
135  for block_cnt, block in enumerate(pcap_scanner):
136  if isinstance(block, EnhancedPacket):
137 
138  # Decode a single pcap block
139  if block.captured_len != block.packet_len:
140  print("## pcap_json_converter block {}: {} byte block truncated to {} bytes".format(block_cnt, block.packet_len, block.captured_len))
141  block_data = Ether(block.packet_data)
142  block_decoded = block_data
143  for n in range(0,10):
144  if isinstance(block_decoded.payload, scapy.packet.Raw):
145  break
146  elif isinstance(block_decoded.payload, scapy.packet.Packet):
147  block_decoded = block_decoded.payload
148  else:
149  break
150 
151  # Write json file, if payload starts with 0x02020202 (i.e. payload is a lidar message)
152  if start_timestamp < 0:
153  start_timestamp = block.timestamp
154  if isinstance(block_decoded.payload, scapy.packet.Raw) and len(block_decoded.payload) > 0:
155  payload_chunk = bytes(block_decoded.payload)
156  # Check start resp. continuation of binary messages
157  if payload_chunk.startswith(b'\x02\x02\x02\x02'): # start of a new binary message
158  relative_timestamp = block.timestamp - start_timestamp
159  payload = payload_chunk
160  payload_completed = (len(payload) >= (parseColaPayloadLength(payload) + 9)) # binary message := { 4 byte STX 0x02020202 } + { 4 byte message_payload_length } + { message_payload } + { 1 byte CRC }
161  elif payload_completed == False and payload.startswith(b'\x02\x02\x02\x02'): # binary message continued
162  payload = payload + payload_chunk
163  payload_completed = (len(payload) >= (parseColaPayloadLength(payload) + 9)) # message := { 4 byte STX 0x02020202 } + { 4 byte message_payload_length } + { message_payload } + { 1 byte CRC }
164  # Check start resp. continuation of ascii messages
165  elif payload_chunk.startswith(b'\x02\x73'): # start of a new ascii message <STX>s...<ETX> with <STX>=\x02 and <ETX>=\x03
166  relative_timestamp = block.timestamp - start_timestamp
167  payload = payload_chunk
168  payload_completed = (payload.find(b'\x03') > 0) # ascii message ends with <ETX>=\x03
169  elif payload_completed == False and payload.startswith(b'\x02\x73'): # ascii message continued
170  payload = payload + payload_chunk
171  payload_completed = (payload.find(b'\x03') > 0) # ascii message ends with <ETX>=\x03
172  # Write payload (binary or ascii messages)
173  is_cola_ascii = payload.startswith(b'\x02\x73')
174  is_cola_binary = payload.startswith(b'\x02\x02\x02\x02')
175  if payload_completed == True and (is_cola_binary or is_cola_ascii):
176  appendToCppfile(cpp_filename, payload, is_cola_ascii)
177  appendToJsonfile(json_filename, relative_timestamp, payload, pcap_filename)
178  print("block {}: rel_timestamp = {}, payload = {}".format(block_cnt, relative_timestamp, payload))
179  payload = b''
180  payload_completed = False
181 
182  closeCppfile(cpp_filename)
183  closeJsonfile(json_filename)
184  print("pcap_json_converter finished.")
pcap_json_converter.closeCppfile
def closeCppfile(cpp_filename)
Definition: pcap_json_converter.py:100
pcap_json_converter.isJsonPrintable
def isJsonPrintable(b)
Definition: pcap_json_converter.py:27
pcap_json_converter.parseColaPayloadLength
def parseColaPayloadLength(payload)
Definition: pcap_json_converter.py:107
pcap_json_converter.appendToJsonfile
def appendToJsonfile(json_filename, relative_timestamp, payload, pcap_filename)
Definition: pcap_json_converter.py:30
pcap_json_converter.appendToCppfile
def appendToCppfile(cpp_filename, payload, is_cola_ascii)
Definition: pcap_json_converter.py:67
pcap_json_converter.closeJsonfile
def closeJsonfile(json_filename)
Definition: pcap_json_converter.py:61

sick_scan
Author(s): Michael Lehning , Jochen Sprickerhof , Martin Günther
autogenerated on Wed May 5 2021 03:05:48