This isn't intended to be a full tutorial on how to do so, you still have a bit of work to do to understand how it works and what is required. However, i have written some python code which should greatly assist in understanding how the data works.
I was going to document it in the wiki, but it's broken at the moment.
The below is not a script you can run, it is pieces of code, definitions, functions, etc. that might help you write your own script or application.
I have these all stored in a Jupyter notebook which i use to selectively run bits and pieces as required.
Code: Select all
# Libraries
import datetime
# Globals
timedef = {}
timedef[80] = "MINUTE"
timedef[81] = "HOUR"
timedef[82] = "DAY"
timedef[83] = "WEEK"
timedef[84] = "MONTH"
timedef[85] = "NEVER"
timedef["MINUTE"] = 80
timedef["HOUR"] = 81
timedef["DAY"] = 82
timedef["WEEK"] = 83
timedef["MONTH"] = 84
timedef["NEVER"] = 85
familydef = {}
familydef[2] = "IPv4"
familydef[10] = "IPv6"
familydef["IPv4"] = 2
familydef["IPv6"] = 10
secondsdef = {}
secondsdef['MINUTE'] = 60
secondsdef['HOUR'] = 3600
secondsdef['DAY'] = 86400
secondsdef['WEEK'] = 604800
secondsdef['MONTH'] = 16934400 # 28 day month
secondsdef['NEVER'] = 0
# Functions
def getBigEndian(indata, offset, numbytes):
data=[]
for beidx in range(offset,(offset+numbytes)):
data.append(indata[beidx])
return data
def getLittleEndian(indata, offset, numbytes):
data=[]
for leidx in reversed(range(offset,(offset+numbytes))):
data.append(indata[leidx])
return data
def getBigEndianInteger(indata, offset, numbytes):
data = getBigEndian(indata, offset, numbytes)
retval = 0
for beintidx in range(0,len(data)):
retval = retval + data[beintidx]*(256**(len(data)-1-beintidx))
return retval
def getLittleEndianInteger(indata, offset, numbytes):
data = getLittleEndian(indata, offset, numbytes)
retval = 0
for beintidx in reversed(range(0,len(data))):
retval = retval + data[beintidx]*(256**(len(data)-1-beintidx))
return retval
def writeLittleEndianInteger(outdata, numbytes):
return (outdata).to_bytes(numbytes, byteorder='little')
def getIP(family, indata):
retval = ""
if family=="IPv4":
retval = "{}.{}.{}.{}".format(indata[0],indata[1],indata[2],indata[3])
if family == "IPv6":
for ipidx in range(0,len(indata)):
indata[ipidx] = format(indata[ipidx], '02x')
retval = "{}{}:{}{}:{}{}:{}{}:{}{}:{}{}:{}{}:{}{}".format(indata[0],indata[1],indata[2],indata[3],indata[4],indata[5],indata[6],indata[7],indata[8],indata[9],indata[10],indata[11],indata[12],indata[13],indata[14],indata[15])
return retval
def writeIP(family, outdata):
retval = bytes()
if family=="IPv4":
retval = bytes(map(int, outdata.split('.'))) + bytes(12)
if family == "IPv6":
retval = bytes.fromhex(outdata.replace(":",""))
return retval
def readUsage(indata):
ipdata = {"data":{}}
# Data format
#
# timestamp
# IP Family IP Address Data (Bytes)
#
dptr = 0
tstamp = indata[dptr].strip()
tstamp = datetime.datetime.fromtimestamp(int(tstamp))
ipdata["time"] = tstamp
dptr = dptr + 1
for x in range(1,len(indata)-1):
line = indata[dptr].split("\t")
family = line[0]
family = familydef[int(family)]
ipaddr = line[1].strip()
if ipaddr == '0.0.0.0':
ipaddr = 'COMBINED'
databytes = line[2].strip()
dptr = dptr + 1
ipdata["data"][str(ipaddr)] = {"family":family, "data":databytes}
return ipdata
def writeUsage(outdata, outfile):
numout = len(outdata["data"])
if numout == 0:
return 'No Data';
fptr = open(outfile, "w")
# Data format
#
# timestamp
# IP Family IP Address Data (Bytes)
#
fptr.write(str(int(outdata['time'].timestamp())) + '\n')
for idx,ip in enumerate(outdata["data"]):
ipstr = ip
if ip == 'COMBINED':
ipstr = '0.0.0.0'
outstr = '{}\t{}\t{}\n'.format(familydef[outdata["data"][ip]['family']],ipstr,outdata["data"][ip]['data'])
fptr.write(outstr)
fptr.close()
return 'OK';
def printUsage(indata):
print("==============================================================")
print("Timestamp: {}".format(indata["time"]))
for key,value in indata["data"].items():
print("--------------------------------------------------------------")
print("IP: {}".format(key))
print("{} Bytes".format(value['data']))
print("--------------------------------------------------------------")
print("==============================================================")
def readHistory(indata):
ipdata = {"data":{}}
# Data format
#
# 0-3 - Num IPs in file
# 4-11 - Time Interval
# 12-19 - Timestamp
# 20 - Is constant time interval
#
# Then for each IP (for num of IPs)
#
# 0-3 - IP Family
# 4-19 - IP Address (IPv4 25-28 (reminder zero), IPv6 25-40)
# 20-23 - Number of history nodes for IP
# 24-31 - First Start Timestamp
# 32-39 - First End Timestamp
# 40-47 - Last End Timestamp
# 48 - Is data 32 or 64 bit?
#
# Then for each History (for num of histories)
#
# 0-3 (or 0-7) - Data
#
# Interval
# MINUTE - 80
# HOUR - 81
# DAY - 82
# WEEK - 83
# MONTH - 84
# NEVER - 85
dataoffset = 0
numips = getLittleEndianInteger(fdata, dataoffset, 4)
dataoffset = dataoffset + 4
tinterval = getLittleEndianInteger(fdata, dataoffset, 8)
dataoffset = dataoffset + 8
tinterval = timedef[tinterval]
ipdata["interval"] = tinterval
tstamp = getLittleEndianInteger(fdata, dataoffset, 8)
dataoffset = dataoffset + 8
constantinterval = getLittleEndianInteger(fdata, dataoffset, 1)
dataoffset = dataoffset + 1
for x in range(0,numips):
family = getLittleEndianInteger(fdata, dataoffset, 4)
family = familydef[family]
dataoffset = dataoffset + 4
iplen = 4
if family == "IPv6":
iplen = 16
ipaddr = getIP(family,getBigEndian(fdata, dataoffset, iplen))
dataoffset = dataoffset + 16
if ipaddr == '0.0.0.0':
ipaddr = 'COMBINED'
family = 'BOTH'
ipdata["data"][ipaddr] = {"family":family, "history":[]}
historylen = getLittleEndianInteger(fdata, dataoffset, 4)
dataoffset = dataoffset + 4
firststarttstamp = getLittleEndianInteger(fdata, dataoffset, 8)
firststarttstamp = datetime.datetime.fromtimestamp(firststarttstamp)
dataoffset = dataoffset + 8
firstendtstamp = getLittleEndianInteger(fdata, dataoffset, 8)
firstendtstamp = datetime.datetime.fromtimestamp(firstendtstamp)
dataoffset = dataoffset + 8
lastendtstamp = getLittleEndianInteger(fdata, dataoffset, 8)
lastendtstamp = datetime.datetime.fromtimestamp(lastendtstamp)
dataoffset = dataoffset + 8
datatype = getLittleEndianInteger(fdata, dataoffset, 1)
dataoffset = dataoffset + 1
datalen = datatype
ipdata["data"][ipaddr]["datalen"] = datalen
for y in range(0,historylen):
data = getLittleEndianInteger(fdata, dataoffset, int(datalen/8))
dataoffset = dataoffset + 4
if datalen == 64:
dataoffset = dataoffset + 4
if y == 0:
ipdata["data"][ipaddr]['history'].append({"start":firststarttstamp, "end":firstendtstamp, "data":data})
elif y == (historylen-1):
ipdata["data"][ipaddr]['history'].append({"start":firstendtstamp + datetime.timedelta(seconds=(y-1)*secondsdef[tinterval]), "end":lastendtstamp, "data":data})
else:
ipdata["data"][ipaddr]['history'].append({"start":firstendtstamp + datetime.timedelta(seconds=(y-1)*secondsdef[tinterval]), "end":firstendtstamp + datetime.timedelta(seconds=y*secondsdef[tinterval]), "data":data})
return ipdata
def writeHistory(outdata, outfile):
numout = len(outdata["data"])
if numout == 0:
return 'No Data';
fptr = open(outfile, "wb")
# Data format
#
# 0-3 - Num IPs in file
# 4-11 - Time Interval
# 12-19 - Timestamp
# 20 - Is constant time interval
#
# Then for each IP (for num of IPs)
#
# 0-3 - IP Family
# 4-19 - IP Address (IPv4 25-28 (reminder zero), IPv6 25-40)
# 20-23 - Number of history nodes for IP
# 24-31 - First Start Timestamp
# 32-39 - First End Timestamp
# 40-47 - Last End Timestamp
# 48 - Is data 32 or 64 bit?
#
# Then for each History (for num of histories)
#
# 0-3 (or 0-7) - Data
#
# Interval
# MINUTE - 80
# HOUR - 81
# DAY - 82
# WEEK - 83
# MONTH - 84
# NEVER - 85
#
fptr.write(writeLittleEndianInteger(numout,4))
fptr.write(writeLittleEndianInteger(timedef[outdata["interval"]],8))
fptr.write(writeLittleEndianInteger(0,8))
fptr.write(writeLittleEndianInteger(False,1))
for idx,ip in enumerate(outdata["data"]):
ipstr = ip
if ip == 'COMBINED':
ipstr = '0.0.0.0'
familystr = outdata["data"][ip]["family"]
if familystr == "BOTH":
familystr = "IPv4"
family = familydef[familystr]
fptr.write(writeLittleEndianInteger(family,4))
fptr.write(writeIP(familystr,ipstr))
fptr.write(writeLittleEndianInteger(len(outdata["data"][ip]["history"]),4))
fptr.write(writeLittleEndianInteger(int(outdata["data"][ip]["history"][0]["start"].timestamp()),8))
fptr.write(writeLittleEndianInteger(int(outdata["data"][ip]["history"][0]["end"].timestamp()),8))
fptr.write(writeLittleEndianInteger(int(outdata["data"][ip]["history"][-1]["end"].timestamp()),8))
fptr.write(writeLittleEndianInteger(outdata["data"][ip]["datalen"],1))
datalen = 4
if outdata["data"][ip]["datalen"] == 64:
datalen = 8
for history in outdata["data"][ip]["history"]:
fptr.write(writeLittleEndianInteger(history["data"],datalen))
fptr.close()
return 'OK';
def printHistory(indata):
print("==============================================================")
print("Interval: {}".format(indata["interval"]))
for key,value in indata["data"].items():
print("--------------------------------------------------------------")
histories = value['history']
print("IP: {}".format(key))
for history in histories:
print("{} Bytes from {} to {}".format(history['data'],history['start'],history['end']))
print("--------------------------------------------------------------")
print("==============================================================")