Receiving data¶
The following examples were taken from test script and it shows comparison of two different approaches to receive and process UniRec records.
Reading in loop¶
The first example is a classical one with main loop and processing UniRec messages one by one:
>>> import pytrap
>>> import time
>>> # Start Receiver
>>> c2 = pytrap.TrapCtx()
>>> c2.init(["-i", "f:/tmp/pytrap_test3"], 1)
>>> c2.setRequiredFmt(0, pytrap.FMT_UNIREC, urtempl)
>>> startt = time.process_time()
>>> data = list()
>>> while True:
... d = c2.recv()
... if not d:
... break
... t.setData(d)
... data.append(t.getDict())
>>> elapsed_time = time.process_time() - startt
>>> print(f"recv() Elapsed time for {messages} messages is: {elapsed_time}")
recv() Elapsed time for 10000000 messages is: 9.396470939999999
>>> c2.finalize()
Bulk reading¶
The second example seems to be more efficient and faster, however, it loads all data into memory (so beware of time and count parameters):
>>> import pytrap
>>> import time
>>> # Start Receiver
>>> c2 = pytrap.TrapCtx()
>>> c2.init(["-i", "f:/tmp/pytrap_test3"], 1)
>>> c2.setRequiredFmt(0, pytrap.FMT_UNIREC, urtempl)
>>> startt = time.process_time()
>>> data = c2.recvBulk(t, time=10, count=messages)
>>> elapsed_time = time.process_time() - startt
>>> print(f"recvBulk() Elapsed time for {messages} messages is: {elapsed_time}")
recvBulk() Elapsed time for 10000000 messages is: 5.647379766
>>> c2.finalize()
Load for pandas DataFrame¶
To load data (5 records) into pandas.DataFrame, it is very simple like this:
>>> import pytrap
>>> pytrap.read_nemea("f:~/pstats.trapcap", nrows=5)
DST_IP SRC_IP BYTES BYTES_REV ... D_PHISTS_SIZES S_PHISTS_IPT S_PHISTS_SIZES PPI_PKT_TIMES
0 185.199.111.133 192.168.1.248 143 143 ... [] [] [] [1636152115.816, 1636152115.827]
1 185.199.111.133 192.168.1.248 143 143 ... [0, 0, 1, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 1, 0, 0, 0, 0, 0] [1636152115.816, 1636152115.827]
2 3.68.63.139 192.168.1.248 159 161 ... [] [] [] [1636152118.645, 1636152118.668]
3 3.68.63.139 192.168.1.248 159 161 ... [0, 0, 1, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 1, 0, 0, 0, 0, 0] [1636152118.645, 1636152118.668]
4 2a00:1450:4014:80c::200e 2001:470:5828:100::82e 1216 1547 ... [] [] [] [1636152112.855, 1636152112.856, 1636152112.95...
[5 rows x 25 columns] >>>
Load into array (list)¶
It can be also switched to return array:
>>> pytrap.read_nemea("f:~/pstats.trapcap", nrows=2, array=True)
[{'DST_IP': UnirecIPAddr('185.199.111.133'), 'SRC_IP': UnirecIPAddr('192.168.1.248'), 'BYTES': 143, 'BYTES_REV': 143, 'LINK_BIT_FIELD': 1, 'TIME_FIRST': UnirecTime(1636152115, 816), 'TIME_LAST': UnirecTime(1636152115, 827), 'DST_MAC': UnirecMACAddr('ac:84:c6:52:dd:15'), 'SRC_MAC': UnirecMACAddr('d4:3b:04:6d:31:2f'), 'PACKETS': 2, 'PACKETS_REV': 2, 'DST_PORT': 443, 'SRC_PORT': 51922, 'DIR_BIT_FIELD': 0, 'PROTOCOL': 6, 'TCP_FLAGS': 24, 'TCP_FLAGS_REV': 24, 'PPI_PKT_DIRECTIONS': [1, -1], 'PPI_PKT_FLAGS': [24, 24], 'PPI_PKT_LENGTHS': [39, 39], 'D_PHISTS_IPT': [], 'D_PHISTS_SIZES': [], 'S_PHISTS_IPT': [], 'S_PHISTS_SIZES': [], 'PPI_PKT_TIMES': [UnirecTime(1636152115, 816), UnirecTime(1636152115, 827)]}, {'DST_IP': UnirecIPAddr('185.199.111.133'), 'SRC_IP': UnirecIPAddr('192.168.1.248'), 'BYTES': 143, 'BYTES_REV': 143, 'LINK_BIT_FIELD': 1, 'TIME_FIRST': UnirecTime(1636152115, 816), 'TIME_LAST': UnirecTime(1636152115, 827), 'DST_MAC': UnirecMACAddr('ac:84:c6:52:dd:15'), 'SRC_MAC': UnirecMACAddr('d4:3b:04:6d:31:2f'), 'PACKETS': 2, 'PACKETS_REV': 2, 'DST_PORT': 443, 'SRC_PORT': 51922, 'DIR_BIT_FIELD': 0, 'PROTOCOL': 6, 'TCP_FLAGS': 24, 'TCP_FLAGS_REV': 24, 'PPI_PKT_DIRECTIONS': [1, -1], 'PPI_PKT_FLAGS': [24, 24], 'PPI_PKT_LENGTHS': [39, 39], 'D_PHISTS_IPT': [0, 0, 0, 0, 0, 0, 0, 0], 'D_PHISTS_SIZES': [0, 0, 1, 0, 0, 0, 0, 0], 'S_PHISTS_IPT': [0, 0, 0, 0, 0, 0, 0, 0], 'S_PHISTS_SIZES': [0, 0, 1, 0, 0, 0, 0, 0], 'PPI_PKT_TIMES': [UnirecTime(1636152115, 816), UnirecTime(1636152115, 827)]}]
>>>