WebSVN – nexmon – Blame – Rev 1 – /utilities/wireshark/tools/indexcap.py

1

office

1

#!/usr/bin/env python

2

#

3

# Tool to index protocols that appears in the given capture files

4

#

5

# The script list_protos_in_cap.sh does the same thing.

#

#

# Wireshark - Network traffic analyzer

10

# By Gerald Combs <gerald@wireshark.org>

11

12

#

13

# This program is free software; you can redistribute it and/or

14

# modify it under the terms of the GNU General Public License

15

# as published by the Free Software Foundation; either version 2

16

# of the License, or (at your option) any later version.

17

#

18

# This program is distributed in the hope that it will be useful,

19

# but WITHOUT ANY WARRANTY; without even the implied warranty of

20

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

21

# GNU General Public License for more details.

22

#

23

# You should have received a copy of the GNU General Public License

24

# along with this program; if not, write to the Free Software

25

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

26

#

27

28

from optparse import OptionParser

29

import multiprocessing

import sys

import os

import subprocess

import re

import pickle

import tempfile

import filecmp

import random

def extract_protos_from_file_proces(tshark, file):

40

try:

41

cmd = [tshark, "-Tfields", "-e", "frame.protocols", "-r", file]

42

p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

43

(stdout, stderr) = p.communicate()

44

if sys.version_info[0] >= 3:

45

stdout = stdout.decode('utf-8')

46

if p.returncode != 0:

return (file, {})

proto_hash = {}

for line in stdout.splitlines():

51

if not re.match(r'^[\w:-]+$', line):

52

continue

53

54

for proto in line.split(':'):

55

proto_hash[proto] = 1 + proto_hash.setdefault(proto, 0)

56

57

return (file, proto_hash)

58

except KeyboardInterrupt:

59

return None

60

61

def extract_protos_from_file(tshark, num_procs, max_files, cap_files, cap_hash, index_file_name):

62

pool = multiprocessing.Pool(num_procs)

63

results = [pool.apply_async(extract_protos_from_file_proces, [tshark, file]) for file in cap_files]

64

try:

65

for (cur_item_idx,result_async) in enumerate(results):

66

file_result = result_async.get()

67

action = "SKIPPED" if file_result[1] is {} else "PROCESSED"

68

            print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0])))

69

cap_hash.update(dict([file_result]))

70

except KeyboardInterrupt:

71

print("%s was interrupted by user" % (sys.argv[0]))

pool.terminate()

exit(1)

index_file = open(index_file_name, "wb")

76

pickle.dump(cap_hash, index_file)

index_file.close()

exit(0)

def dissect_file_process(tshark, tmpdir, file):

81

try:

82

(handle_o, tmpfile_o) = tempfile.mkstemp(suffix='_stdout', dir=tmpdir)

83

(handle_e, tmpfile_e) = tempfile.mkstemp(suffix='_stderr', dir=tmpdir)

84

cmd = [tshark, "-nxVr", file]

85

p = subprocess.Popen(cmd, stdout=handle_o, stderr=handle_e)

86

(stdout, stderr) = p.communicate()

87

if p.returncode == 0:

88

return (file, True, tmpfile_o, tmpfile_e)

89

else:

90

return (file, False, tmpfile_o, tmpfile_e)

91

92

except KeyboardInterrupt:

return False

finally:

os.close(handle_o)

os.close(handle_e)

def dissect_files(tshark, tmpdir, num_procs, max_files, cap_files):

100

pool = multiprocessing.Pool(num_procs)

101

results = [pool.apply_async(dissect_file_process, [tshark, tmpdir, file]) for file in cap_files]

102

try:

103

for (cur_item_idx,result_async) in enumerate(results):

104

file_result = result_async.get()

105

action = "FAILED" if file_result[1] is False else "PASSED"

106

            print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0])))

107

except KeyboardInterrupt:

108

print("%s was interrupted by user" % (sys.argv[0]))

pool.terminate()

exit(1)

def compare_files(tshark_bin, tmpdir, tshark_cmp, num_procs, max_files, cap_files):

113

pool = multiprocessing.Pool(num_procs)

114

results_bin = [pool.apply_async(dissect_file_process, [tshark_bin, tmpdir, file]) for file in cap_files]

115

results_cmp = [pool.apply_async(dissect_file_process, [tshark_cmp, tmpdir, file]) for file in cap_files]

116

try:

117

for (cur_item_idx,(result_async_bin, result_async_cmp)) in enumerate(zip(results_bin, results_cmp)):

118

file_result_bin = result_async_bin.get()

119

file_result_cmp = result_async_cmp.get()

120

if file_result_cmp[1] is False or file_result_bin[1] is False:

121

action = "FAILED (exitcode)"

122

if not filecmp.cmp(file_result_bin[2], file_result_cmp[2]):

123

action = "FAILED (stdout)"

124

if not filecmp.cmp(file_result_bin[3], file_result_cmp[3]):

125

action = "FAILED (stderr)"

126

else:

127

action = "PASSED"

128

os.remove(file_result_bin[2])

129

os.remove(file_result_cmp[2])

130

os.remove(file_result_bin[3])

131

os.remove(file_result_cmp[3])

132

133

            print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_bin[0], os.path.getsize(file_result_bin[0])))

134

            print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_cmp[0], os.path.getsize(file_result_cmp[0])))

135

except KeyboardInterrupt:

136

print("%s was interrupted by user" % (sys.argv[0]))

pool.terminate()

exit(1)

def list_all_proto(cap_hash):

141

proto_hash = {}

142

for files_hash in cap_hash.values():

143

for proto,count in files_hash.items():

144

proto_hash[proto] = count + proto_hash.setdefault(proto, 0)

return proto_hash

def list_all_files(cap_hash):

149

files = list(cap_hash.keys())

files.sort()

return files

def list_all_proto_files(cap_hash, proto_comma_delit):

155

protos = [ x.strip() for x in proto_comma_delit.split(',') ]

156

files = []

157

for (file, files_hash) in cap_hash.items():

158

for proto in files_hash.keys():

159

if proto in protos:

files.append(file)

break

return files

def index_file_action(options):

166

return options.list_all_proto or \

167

options.list_all_files or \

168

options.list_all_proto_files or \

169

options.dissect_files

170

171

def find_capture_files(paths, cap_hash):

172

cap_files = []

173

for path in paths:

174

if os.path.isdir(path):

175

path = os.path.normpath(path)

176

for root, dirs, files in os.walk(path):

177

cap_files += [os.path.join(root, name) for name in files if os.path.join(root, name) not in cap_hash]

178

elif path not in cap_hash:

179

cap_files.append(path)

180

return cap_files

181

182

def find_tshark_executable(bin_dir):

183

for file in ["tshark.exe", "tshark"]:

184

tshark = os.path.join(bin_dir, file)

185

if os.access(tshark, os.X_OK):

return tshark

return None

def main():

parser = OptionParser(usage="usage: %prog [options] index_file [file_1|dir_1 [.. file_n|dir_n]]")

192

parser.add_option("-d", "--dissect-files", dest="dissect_files", default=False, action="store_true",

193

help="Dissect all matching files")

194

parser.add_option("-m", "--max-files", dest="max_files", default=sys.maxsize, type="int",

195

help="Max number of files to process")

196

parser.add_option("-b", "--binary-dir", dest="bin_dir", default=os.getcwd(),

197

help="Directory containing tshark executable")

198

parser.add_option("-c", "--compare-dir", dest="compare_dir", default=None,

199

help="Directory containing tshark executable which is used for comparison")

200

parser.add_option("-j", dest="num_procs", default=multiprocessing.cpu_count(), type=int,

201

help="Max number of processes to spawn")

202

parser.add_option("-r", "--randomize", default=False, action="store_true",

203

help="Randomize the file list order")

204

parser.add_option("", "--list-all-proto", dest="list_all_proto", default=False, action="store_true",

205

help="List all protocols in index file")

206

parser.add_option("", "--list-all-files", dest="list_all_files", default=False, action="store_true",

207

help="List all files in index file")

208

parser.add_option("", "--list-all-proto-files", dest="list_all_proto_files", default=False,

209

metavar="PROTO_1[, .. PROTO_N]",

210

help="List all files in index file containing the given protocol")

211

212

(options, args) = parser.parse_args()

213

214

if len(args) == 0:

215

parser.error("index_file is a required argument")

216

217

if len(args) == 1 and not index_file_action(options):

218

parser.error("one capture file/directory must be specified")

219

220

if options.dissect_files and not options.list_all_files and not options.list_all_proto_files:

221

parser.error("--list-all-files or --list-all-proto-files must be specified")

222

223

if options.dissect_files and not options.compare_dir is None:

224

parser.error("--dissect-files and --compare-dir cannot be specified at the same time")

225

226

index_file_name = args.pop(0)

paths = args

cap_hash = {}

try:

index_file = open(index_file_name, "rb")

231

print("index file: %s [OPENED]" % index_file.name)

232

cap_hash = pickle.load(index_file)

233

index_file.close()

234

print("%d files" % len(cap_hash))

235

except IOError:

236

print("index file: %s [NEW]" % index_file_name)

237

238

if options.list_all_proto:

239

print(list_all_proto(cap_hash))

exit(0)

indexed_files = []

if options.list_all_files:

244

indexed_files = list_all_files(cap_hash)

245

print(indexed_files)

246

247

if options.list_all_proto_files:

248

indexed_files = list_all_proto_files(cap_hash, options.list_all_proto_files)

249

print(indexed_files)

250

251

tshark_bin = find_tshark_executable(options.bin_dir)

252

if not tshark_bin is None:

253

print("tshark: %s [FOUND]" % tshark_bin)

254

else:

255

print("tshark: %s [MISSING]" % tshark_bin)

256

exit(1)

257

258

if not options.compare_dir is None:

259

tshark_cmp = find_tshark_executable(options.compare_dir)

260

if not tshark_cmp is None:

261

print("tshark: %s [FOUND]" % tshark_cmp)

262

else:

263

print("tshark: %s [MISSING]" % tshark_cmp)

264

exit(1)

265

266

if options.dissect_files or options.compare_dir:

267

cap_files = indexed_files

268

elif options.list_all_proto_files or options.list_all_files:

269

exit(0)

270

else:

271

cap_files = find_capture_files(paths, cap_hash)

272

273

if options.randomize:

274

random.shuffle(cap_files)

else:

cap_files.sort()

options.max_files = min(options.max_files, len(cap_files))

279

print("%u total files, %u working files" % (len(cap_files), options.max_files))

280

cap_files = cap_files[:options.max_files]

281

if options.compare_dir or options.dissect_files:

282

tmpdir = tempfile.mkdtemp()

283

print("Temporary working dir: %s" % tmpdir)

284

try:

285

if options.compare_dir:

286

compare_files(tshark_bin, tmpdir, tshark_cmp, options.num_procs, options.max_files, cap_files)

287

elif options.dissect_files:

288

dissect_files(tshark_bin, tmpdir, options.num_procs, options.max_files, cap_files)

289

else:

290

            extract_protos_from_file(tshark_bin, options.num_procs, options.max_files, cap_files, cap_hash, index_file_name)

291

finally:

292

# Dissection may result in a non-empty directory.

293

if options.compare_dir:

294

os.rmdir(tmpdir)

295

if __name__ == "__main__":

296

main()

nexmon – Blame information for rev 1