parser.py 14 KB
Newer Older
1
#!/usr/bin/env python3
2
3
#SCT log parser

4

5
import sys
Vincent Stehlé's avatar
Vincent Stehlé committed
6
import argparse
Vincent Stehlé's avatar
Vincent Stehlé committed
7
import csv
Vincent Stehlé's avatar
Vincent Stehlé committed
8
import logging
Vincent Stehlé's avatar
Vincent Stehlé committed
9
import json
Vincent Stehlé's avatar
Vincent Stehlé committed
10

11

12
#based loosley on https://stackoverflow.com/a/4391978
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
13
# returns a filtered dict of dicts that meet some Key-value pair.
14
# I.E. key="result" value="FAILURE"
15
def key_value_find(list_1, key, value):
16
    found = list()
17
    for test in list_1:
18
        if test[key] == value:
19
            found.append(test)
20
    return found
21

22
23

#Were we intrept test logs into test dicts
24
def test_parser(string, current):
25
    test_list = {
Vincent Stehlé's avatar
Vincent Stehlé committed
26
      "name": string[2], #FIXME:Sometimes, SCT has name and Description,
27
      "result": string[1],
28
      **current,
29
      "guid": string[0], #FIXME:GUID's overlap
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
30
      #"comment": string[-1], #FIXME:need to hash this out, sometime there is no comments
31
      "log": ' '.join(string[3:])
32
    }
33
    return test_list
Vincent Stehlé's avatar
Vincent Stehlé committed
34

35
#Parse the ekl file, and create a map of the tests
36
37
def ekl_parser (file):
    #create our "database" dict
38
    temp_list = list()
39
40
41
42
43
44
45
46
    # All tests are grouped by the "HEAD" line, which precedes them.
    current = {}

    # Count number of tests since beginning of the set
    n = 0

    # Number of skipped tests sets
    s = 0
47

48
    for i, line in enumerate(file):
Vincent Stehlé's avatar
Vincent Stehlé committed
49
50
51
52
53
54
55
        # Strip the line from trailing whitespaces
        line = line.rstrip()

        # Skip empty line
        if line == '':
            continue

56
57
        # strip the line of | & || used for sepration
        split_line = line.split('|')
58

59
60
61
62
        # TERM marks the end of a test set
        # In case of empty test set we generate an artificial skipped test
        # entry. Then reset current as a precaution, as well as our test
        # counter.
63
        if split_line[0] == '' and split_line[1] == "TERM":
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
            if not n:
                logging.debug(f"Skipped test set `{current['sub set']}'")

                temp_list.append({
                    **current,
                    'name': '',
                    'guid': '',
                    'log': '',
                    'result': 'SKIPPED',
                })

                s += 1

            current = {}
            n = 0
79
80
            continue

81
82
83
        # The "HEAD" tag is the only indcation we are on a new test set
        if split_line[0] == '' and split_line[1] == "HEAD":
            # split the header into test group and test set.
84
            try:
85
86
87
                group, Set = split_line[12].split('\\')
            except Exception:
                group, Set = '', split_line[12]
88
89
90
            current = {
                'group': group,
                'test set': Set,
91
92
93
94
95
96
97
98
                'sub set': split_line[10],
                'set guid': split_line[8],
                'iteration': split_line[4],
                'start date': split_line[6],
                'start time': split_line[7],
                'revision': split_line[9],
                'descr': split_line[11],
                'device path': '|'.join(split_line[13:]),
99
            }
100

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
101
        #FIXME:? EKL file has an inconsistent line structure,
102
103
        # sometime we see a line that consits ' dump of GOP->I\n'
        #easiest way to skip is check for blank space in the first char
104
        elif split_line[0] != '' and split_line[0][0] != " ":
105
106
107
108
            try:
                #deliminiate on ':' for tests
                split_test = [new_string for old_string in split_line for new_string in old_string.split(':')]
                #put the test into a dict, and then place that dict in another dict with GUID as key
109
                tmp_dict = test_parser(split_test, current)
110
                temp_list.append(tmp_dict)
111
                n += 1
112
113
114
            except:
                print("Line:",split_line)
                sys.exit("your log may be corrupted")
115
116
117
        else:
            logging.error(f"Unparsed line {i} `{line}'")

118
119
120
    if s:
        logging.debug(f'{s} skipped test set(s)')

121
    return temp_list
122

Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
123
#Parse Seq file, used to tell which tests should run.
124
def seq_parser(file):
125
    temp_dict = list()
126
    lines=file.readlines()
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
127
    magic=7 #a test in a seq file is 7 lines, if not mod7, something wrong..
128
129
    if len(lines)%magic != 0:
        sys.exit("seqfile cut short, should be mod7")
130
131
132
133
134
135
136
137
    #the utf-16 char makes this looping a bit harder, so we use x+(i) where i is next 0-6th
    for x in range(0,len(lines),magic): #loop ever "7 lines"
        #(x+0)[Test Case]
        #(x+1)Revision=0x10000
        #(x+2)Guid=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
        #(x+3)Name=InstallAcpiTableFunction
        #(x+4)Order=0xFFFFFFFF
        #(x+5)Iterations=0xFFFFFFFF
Vincent Stehlé's avatar
Vincent Stehlé committed
138
        #(x+6)(utf-16 char)
139
        #currently only add tests that are supposed to run, should add all?
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
140
        #0xFFFFFFFF in "Iterations" means the test is NOT supposed to run
141
142
143
144
145
146
147
148
        if not "0xFFFFFFFF" in lines[x+5]:
            seq_dict = {
                "name": lines[x+3][5:-1],#from after "Name=" to end (5char long)
                "guid": lines[x+2][5:-1],#from after"Guid=" to the end, (5char long)
                "Iteration": lines[x+5][11:-1],#from after "Iterations=" (11char long)
                "rev": lines[x+1][9:-1],#from after "Revision=" (9char long)
                "Order": lines[x+4][6:-1]#from after "Order=" (6char long)
            }
149
            temp_dict.append(seq_dict) #put in a dict based on guid
150

151
    return temp_dict
152

153
154
155
156
#group items by key, and print by key
#we slowly iterate through the list, group and print groups
def key_tree_2_md(input_list,file,key):
    #make a copy so we don't destroy the first list.
Vincent Stehlé's avatar
Vincent Stehlé committed
157
    temp_list = input_list.copy()
158
159
160
161
162
163
164
165
166
167
168
169
    while temp_list:
        test_dict = temp_list.pop()
        found, not_found = [test_dict],[]
        #go through whole list looking for key match
        while temp_list:
            next_dict = temp_list.pop()
            if next_dict[key] == test_dict[key]: #if match add to found
                found.append(next_dict)
            else: #else not found
                not_found.append(next_dict)
        temp_list = not_found #start over with found items removed
        file.write("### " + test_dict[key])
170
        dict_2_md(found,file)
Vincent Stehlé's avatar
Vincent Stehlé committed
171

172
173


174
175
176
#generic writer, takes a list of dicts and turns the dicts into an MD table.
def dict_2_md(input_list,file):
    if len(input_list) > 0:
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
177
        file.write("\n\n")
178
179
180
181
        #create header for MD table using dict keys
        temp_string1, temp_string2 = "|", "|"
        for x in (input_list[0].keys()):
            temp_string1 += (x + "|")
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
182
            temp_string2 += ("---|")
183
184
185
186
187
188
189
        file.write(temp_string1+"\n"+temp_string2+"\n")
        #print each item from the dict into the table
        for x in input_list:
            test_string = "|"
            for y in x.keys():
                test_string += (x[y] + "|")
            file.write(test_string+'\n')
Vincent Stehlé's avatar
Vincent Stehlé committed
190
    #seprate table from other items in MD
191
    file.write("\n\n")
192

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
193

Vincent Stehlé's avatar
Vincent Stehlé committed
194
195
196
197
198
199
200
201
202
203
# Sort tests data in-place
# sort_keys is a comma-separated list
# The first key has precedence, then the second, etc.
# To use python list in-place sorting, we use the keys in reverse order.
def sort_data(cross_check, sort_keys):
    logging.debug(f"Sorting on `{sort_keys}'")
    for k in reversed(sort_keys.split(',')):
        cross_check.sort(key=lambda x: x[k])


Vincent Stehlé's avatar
Vincent Stehlé committed
204
205
206
207
208
209
210
211
212
# Generate csv
def gen_csv(cross_check, filename):
    # Find keys
    keys = set()

    for x in cross_check:
        keys = keys.union(x.keys())

    # Write csv
Vincent Stehlé's avatar
Vincent Stehlé committed
213
214
    logging.debug(f'Generate {filename}')

Vincent Stehlé's avatar
Vincent Stehlé committed
215
216
217
218
219
220
221
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.DictWriter(
            csvfile, fieldnames=sorted(keys), delimiter=';')
        writer.writeheader()
        writer.writerows(cross_check)


Vincent Stehlé's avatar
Vincent Stehlé committed
222
223
224
225
226
227
228
229
# Generate json
def gen_json(cross_check, filename):
    logging.debug(f'Generate {filename}')

    with open(filename, 'w') as jsonfile:
        json.dump(cross_check, jsonfile, sort_keys=True, indent=2)


230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
# Combine or two databases db1 and db2 coming from ekl and seq files
# respectively into a single cross_check database
# Tests in db1, which were not meant to be run according to db2 have their
# results forced to SPURIOUS.
# Tests sets in db2, which were not run according to db1 have an artificial
# test entry created with result DROPPED.
def combine_dbs(db1, db2):
    cross_check = db1

    # Do a pass to verify that all tests in db1 were meant to be run.
    # Otherwise, force the result to SPURIOUS.
    s = set()

    for x in db2:
        s.add(x['guid'])

    n = 0

    for i in range(len(cross_check)):
        if cross_check[i]['set guid'] not in s:
            logging.debug(f"Spurious test {i} `{cross_check[i]['name']}'")
            cross_check[i]['result'] = 'SPURIOUS'
            n += 1

    if n:
        logging.debug(f'{n} spurious test(s)')

    # Do a pass to autodetect all tests fields in case we need to merge dropped
    # tests sets entries
    f = {}

    for x in cross_check:
        for k in x.keys():
            f[k] = ''

    logging.debug(f'Test fields: {f.keys()}')

    # Do a pass to find the test sets that did not run for whatever reason.
    s = set()

    for x in cross_check:
        s.add(x['set guid'])

    n = 0

    for i in range(len(db2)):
        x = db2[i]

        if not x['guid'] in s:
            logging.debug(f"Dropped test set {i} `{x['name']}'")

            # Create an artificial test entry to reflect the dropped test set
            cross_check.append({
                **f,
                'sub set': x['name'],
                'set guid': x['guid'],
                'revision': x['rev'],
                'group': 'Unknown',
                'result': 'DROPPED',
            })

            n += 1

    if n:
        logging.debug(f'{n} dropped test set(s)')

    return cross_check


299
def main():
Vincent Stehlé's avatar
Vincent Stehlé committed
300
301
302
303
    parser = argparse.ArgumentParser(
        description='Process SCT results.'
                    ' This program takes the SCT summary and sequence files,'
                    ' and generates a nice report in mardown format.',
Vincent Stehlé's avatar
Vincent Stehlé committed
304
305
306
307
308
        epilog='When sorting is requested, tests data are sorted'
               ' according to the first sort key, then the second, etc.'
               ' Sorting happens after update by the configuration rules.'
               ' Useful example: --sort'
               ' "group,descr,set guid,test set,sub set,guid,name,log"',
Vincent Stehlé's avatar
Vincent Stehlé committed
309
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
Vincent Stehlé's avatar
Vincent Stehlé committed
310
    parser.add_argument('--csv', help='Output .csv filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
311
    parser.add_argument('--json', help='Output .json filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
312
313
    parser.add_argument(
        '--md', help='Output .md filename', default='result.md')
Vincent Stehlé's avatar
Vincent Stehlé committed
314
315
    parser.add_argument(
        '--debug', action='store_true', help='Turn on debug messages')
Vincent Stehlé's avatar
Vincent Stehlé committed
316
317
    parser.add_argument(
        '--sort', help='Comma-separated list of keys to sort output on')
Vincent Stehlé's avatar
Vincent Stehlé committed
318
319
320
321
322
323
324
325
326
327
    parser.add_argument(
        'log_file', nargs='?', default='sample.ekl',
        help='Input .ekl filename')
    parser.add_argument(
        'seq_file', nargs='?', default='sample.seq',
        help='Input .seq filename')
    parser.add_argument('find_key', nargs='?', help='Search key')
    parser.add_argument('find_value', nargs='?', help='Search value')
    args = parser.parse_args()

Vincent Stehlé's avatar
Vincent Stehlé committed
328
329
330
331
    logging.basicConfig(
        format='%(levelname)s %(funcName)s: %(message)s',
        level=logging.DEBUG if args.debug else logging.INFO)

Vincent Stehlé's avatar
Vincent Stehlé committed
332
    #Command line argument 1, ekl file to open
333
    db1 = list() #"database 1" all tests.
Vincent Stehlé's avatar
Vincent Stehlé committed
334
335
    logging.debug(f'Read {args.log_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
336
    with open(args.log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
337
338
        db1 = ekl_parser(f.readlines())

Vincent Stehlé's avatar
Vincent Stehlé committed
339
340
    logging.debug('{} test(s)'.format(len(db1)))

Vincent Stehlé's avatar
Vincent Stehlé committed
341
    #Command line argument 2, seq file to open
342
    db2 = dict() #"database 2" all test sets that should run
Vincent Stehlé's avatar
Vincent Stehlé committed
343
344
    logging.debug(f'Read {args.seq_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
345
    with open(args.seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
346
        db2 = seq_parser(f)
Vincent Stehlé's avatar
Vincent Stehlé committed
347

Vincent Stehlé's avatar
Vincent Stehlé committed
348
349
    logging.debug('{} test set(s)'.format(len(db2)))

350
351
    # Produce a single cross_check database from our two db1 and db2 databases.
    cross_check = combine_dbs(db1, db2)
352

Vincent Stehlé's avatar
Vincent Stehlé committed
353

Vincent Stehlé's avatar
Vincent Stehlé committed
354
355
356
357
    # Sort tests data in-place, if requested
    if args.sort is not None:
        sort_data(cross_check, args.sort)

358
359
360
    # search for failures, warnings, passes & others
    # We detect all present keys in additions to the expected ones. This is
    # handy with config rules overriding the result field with arbitrary values.
361
    res_keys = set(['DROPPED', 'FAILURE', 'WARNING', 'PASS'])
Vincent Stehlé's avatar
Vincent Stehlé committed
362

363
364
365
366
367
368
369
370
371
372
    for x in cross_check:
        res_keys.add(x['result'])

    # Now we fill some bins with tests according to their result
    bins = {}

    for k in res_keys:
        bins[k] = key_value_find(cross_check, "result", k)

    # Print a one-line summary
373
    s = map(
374
375
376
377
        lambda k: '{} {}(s)'.format(len(bins[k]), k.lower()),
        sorted(res_keys))

    logging.info(', '.join(s))
378
379

    # generate MD summary
Vincent Stehlé's avatar
Vincent Stehlé committed
380
381
    logging.debug(f'Generate {args.md}')

Vincent Stehlé's avatar
Vincent Stehlé committed
382
    with open(args.md, 'w') as resultfile:
383
384
385
        resultfile.write("# SCT Summary \n\n")
        resultfile.write("|  |  |\n")
        resultfile.write("|--|--|\n")
386
387
388
389
390
391

        # Loop on all the result values we found for the summary
        for k in sorted(res_keys):
            resultfile.write(
                "|{}:|{}|\n".format(k.title(), len(bins[k])))

392
393
        resultfile.write("\n\n")

394
395
        # Loop on all the result values we found (except PASS) for the sections
        # listing the tests by group
396
        n = 1
397
398
        res_keys_np = set(res_keys)
        res_keys_np.remove('PASS')
Vincent Stehlé's avatar
Vincent Stehlé committed
399

400
401
402
403
        for k in sorted(res_keys_np):
            resultfile.write("## {}. {} by group\n\n".format(n, k.title()))
            key_tree_2_md(bins[k], resultfile, "group")
            n += 1
404

Vincent Stehlé's avatar
Vincent Stehlé committed
405
406
407
    # Generate csv if requested
    if args.csv is not None:
        gen_csv(cross_check, args.csv)
Vincent Stehlé's avatar
Vincent Stehlé committed
408

Vincent Stehlé's avatar
Vincent Stehlé committed
409
410
411
412
    # Generate json if requested
    if args.json is not None:
        gen_json(cross_check, args.json)

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
413
414
    #command line argument 3&4, key are to support a key & value search.
    #these will be displayed in CLI
Vincent Stehlé's avatar
Vincent Stehlé committed
415
416
    if args.find_key is not None and args.find_value is not None:
        found = key_value_find(db1, args.find_key, args.find_value)
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
417
418
419
        #print the dict
        print("found:",len(found),"items with search constraints")
        for x in found:
Vincent Stehlé's avatar
Vincent Stehlé committed
420
421
422
423
            print(
                x["guid"], ":", x["name"], "with", args.find_key, ":",
                x[args.find_key])

424

425
main()