parser.py 13.2 KB
Newer Older
1
#!/usr/bin/env python3
2
3
#SCT log parser

4

5
import sys
Vincent Stehlé's avatar
Vincent Stehlé committed
6
import argparse
Vincent Stehlé's avatar
Vincent Stehlé committed
7
import csv
Vincent Stehlé's avatar
Vincent Stehlé committed
8
import logging
Vincent Stehlé's avatar
Vincent Stehlé committed
9
import json
Vincent Stehlé's avatar
Vincent Stehlé committed
10

11

12
#based loosley on https://stackoverflow.com/a/4391978
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
13
# returns a filtered dict of dicts that meet some Key-value pair.
14
# I.E. key="result" value="FAILURE"
15
def key_value_find(list_1, key, value):
16
    found = list()
17
    for test in list_1:
18
        if test[key] == value:
19
            found.append(test)
20
    return found
21

22
23

#Were we intrept test logs into test dicts
24
def test_parser(string, current):
25
    test_list = {
Vincent Stehlé's avatar
Vincent Stehlé committed
26
      "name": string[2], #FIXME:Sometimes, SCT has name and Description,
27
      "result": string[1],
28
      **current,
29
      "guid": string[0], #FIXME:GUID's overlap
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
30
      #"comment": string[-1], #FIXME:need to hash this out, sometime there is no comments
31
      "log": ' '.join(string[3:])
32
    }
33
    return test_list
Vincent Stehlé's avatar
Vincent Stehlé committed
34

35
#Parse the ekl file, and create a map of the tests
36
37
def ekl_parser (file):
    #create our "database" dict
38
    temp_list = list()
39
    #All tests are grouped by the "HEAD" line the procedes them.
40
41
42
43
44
45
    current = {
        'group': "N/A",
        'test set': "N/A",
        'sub set': "N/A",
        'set guid': "N/A",
    }
46
47

    for line in file:
Vincent Stehlé's avatar
Vincent Stehlé committed
48
49
50
51
52
53
54
        # Strip the line from trailing whitespaces
        line = line.rstrip()

        # Skip empty line
        if line == '':
            continue

55
56
57
        #strip the line of | & || used for sepration
        split_line = [string for string in line.split('|') if string != ""]

Vincent Stehlé's avatar
Vincent Stehlé committed
58
59
        # Skip TERM
        if split_line[0] == "TERM":
60
61
62
63
64
            continue

        #The "HEAD" tag is the only indcation we are on a new test set
        if split_line[0]=="HEAD":
            #split the header into test group and test set.
65
            try:
66
                group, Set = split_line[8].split('\\')
67
            except:
68
69
70
71
72
73
                group, Set = '', split_line[8]
            current = {
                'group': group,
                'test set': Set,
                'sub set': split_line[6],
                'set guid': split_line[4],
74
75
76
77
78
79
                'iteration': split_line[1],
                'start date': split_line[2],
                'start time': split_line[3],
                'revision': split_line[5],
                'descr': split_line[7],
                'device path': split_line[9],
80
            }
81

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
82
        #FIXME:? EKL file has an inconsistent line structure,
83
84
85
        # sometime we see a line that consits ' dump of GOP->I\n'
        #easiest way to skip is check for blank space in the first char
        elif split_line[0][0] != " ":
86
87
88
89
            try:
                #deliminiate on ':' for tests
                split_test = [new_string for old_string in split_line for new_string in old_string.split(':')]
                #put the test into a dict, and then place that dict in another dict with GUID as key
90
                tmp_dict = test_parser(split_test, current)
91
92
93
94
95
                temp_list.append(tmp_dict)
            except:
                print("Line:",split_line)
                sys.exit("your log may be corrupted")
    return temp_list
96

Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
97
#Parse Seq file, used to tell which tests should run.
98
def seq_parser(file):
99
    temp_dict = list()
100
    lines=file.readlines()
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
101
    magic=7 #a test in a seq file is 7 lines, if not mod7, something wrong..
102
103
    if len(lines)%magic != 0:
        sys.exit("seqfile cut short, should be mod7")
104
105
106
107
108
109
110
111
    #the utf-16 char makes this looping a bit harder, so we use x+(i) where i is next 0-6th
    for x in range(0,len(lines),magic): #loop ever "7 lines"
        #(x+0)[Test Case]
        #(x+1)Revision=0x10000
        #(x+2)Guid=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
        #(x+3)Name=InstallAcpiTableFunction
        #(x+4)Order=0xFFFFFFFF
        #(x+5)Iterations=0xFFFFFFFF
Vincent Stehlé's avatar
Vincent Stehlé committed
112
        #(x+6)(utf-16 char)
113
        #currently only add tests that are supposed to run, should add all?
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
114
        #0xFFFFFFFF in "Iterations" means the test is NOT supposed to run
115
116
117
118
119
120
121
122
        if not "0xFFFFFFFF" in lines[x+5]:
            seq_dict = {
                "name": lines[x+3][5:-1],#from after "Name=" to end (5char long)
                "guid": lines[x+2][5:-1],#from after"Guid=" to the end, (5char long)
                "Iteration": lines[x+5][11:-1],#from after "Iterations=" (11char long)
                "rev": lines[x+1][9:-1],#from after "Revision=" (9char long)
                "Order": lines[x+4][6:-1]#from after "Order=" (6char long)
            }
123
            temp_dict.append(seq_dict) #put in a dict based on guid
124

125
    return temp_dict
126

127
128
129
130
#group items by key, and print by key
#we slowly iterate through the list, group and print groups
def key_tree_2_md(input_list,file,key):
    #make a copy so we don't destroy the first list.
Vincent Stehlé's avatar
Vincent Stehlé committed
131
    temp_list = input_list.copy()
132
133
134
135
136
137
138
139
140
141
142
143
    while temp_list:
        test_dict = temp_list.pop()
        found, not_found = [test_dict],[]
        #go through whole list looking for key match
        while temp_list:
            next_dict = temp_list.pop()
            if next_dict[key] == test_dict[key]: #if match add to found
                found.append(next_dict)
            else: #else not found
                not_found.append(next_dict)
        temp_list = not_found #start over with found items removed
        file.write("### " + test_dict[key])
144
        dict_2_md(found,file)
Vincent Stehlé's avatar
Vincent Stehlé committed
145

146
147


148
149
150
#generic writer, takes a list of dicts and turns the dicts into an MD table.
def dict_2_md(input_list,file):
    if len(input_list) > 0:
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
151
        file.write("\n\n")
152
153
154
155
        #create header for MD table using dict keys
        temp_string1, temp_string2 = "|", "|"
        for x in (input_list[0].keys()):
            temp_string1 += (x + "|")
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
156
            temp_string2 += ("---|")
157
158
159
160
161
162
163
        file.write(temp_string1+"\n"+temp_string2+"\n")
        #print each item from the dict into the table
        for x in input_list:
            test_string = "|"
            for y in x.keys():
                test_string += (x[y] + "|")
            file.write(test_string+'\n')
Vincent Stehlé's avatar
Vincent Stehlé committed
164
    #seprate table from other items in MD
165
    file.write("\n\n")
166

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
167

Vincent Stehlé's avatar
Vincent Stehlé committed
168
169
170
171
172
173
174
175
176
177
# Sort tests data in-place
# sort_keys is a comma-separated list
# The first key has precedence, then the second, etc.
# To use python list in-place sorting, we use the keys in reverse order.
def sort_data(cross_check, sort_keys):
    logging.debug(f"Sorting on `{sort_keys}'")
    for k in reversed(sort_keys.split(',')):
        cross_check.sort(key=lambda x: x[k])


Vincent Stehlé's avatar
Vincent Stehlé committed
178
179
180
181
182
183
184
185
186
# Generate csv
def gen_csv(cross_check, filename):
    # Find keys
    keys = set()

    for x in cross_check:
        keys = keys.union(x.keys())

    # Write csv
Vincent Stehlé's avatar
Vincent Stehlé committed
187
188
    logging.debug(f'Generate {filename}')

Vincent Stehlé's avatar
Vincent Stehlé committed
189
190
191
192
193
194
195
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.DictWriter(
            csvfile, fieldnames=sorted(keys), delimiter=';')
        writer.writeheader()
        writer.writerows(cross_check)


Vincent Stehlé's avatar
Vincent Stehlé committed
196
197
198
199
200
201
202
203
# Generate json
def gen_json(cross_check, filename):
    logging.debug(f'Generate {filename}')

    with open(filename, 'w') as jsonfile:
        json.dump(cross_check, jsonfile, sort_keys=True, indent=2)


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
# Combine or two databases db1 and db2 coming from ekl and seq files
# respectively into a single cross_check database
# Tests in db1, which were not meant to be run according to db2 have their
# results forced to SPURIOUS.
# Tests sets in db2, which were not run according to db1 have an artificial
# test entry created with result DROPPED.
def combine_dbs(db1, db2):
    cross_check = db1

    # Do a pass to verify that all tests in db1 were meant to be run.
    # Otherwise, force the result to SPURIOUS.
    s = set()

    for x in db2:
        s.add(x['guid'])

    n = 0

    for i in range(len(cross_check)):
        if cross_check[i]['set guid'] not in s:
            logging.debug(f"Spurious test {i} `{cross_check[i]['name']}'")
            cross_check[i]['result'] = 'SPURIOUS'
            n += 1

    if n:
        logging.debug(f'{n} spurious test(s)')

    # Do a pass to autodetect all tests fields in case we need to merge dropped
    # tests sets entries
    f = {}

    for x in cross_check:
        for k in x.keys():
            f[k] = ''

    logging.debug(f'Test fields: {f.keys()}')

    # Do a pass to find the test sets that did not run for whatever reason.
    s = set()

    for x in cross_check:
        s.add(x['set guid'])

    n = 0

    for i in range(len(db2)):
        x = db2[i]

        if not x['guid'] in s:
            logging.debug(f"Dropped test set {i} `{x['name']}'")

            # Create an artificial test entry to reflect the dropped test set
            cross_check.append({
                **f,
                'sub set': x['name'],
                'set guid': x['guid'],
                'revision': x['rev'],
                'group': 'Unknown',
                'result': 'DROPPED',
            })

            n += 1

    if n:
        logging.debug(f'{n} dropped test set(s)')

    return cross_check


273
def main():
Vincent Stehlé's avatar
Vincent Stehlé committed
274
275
276
277
    parser = argparse.ArgumentParser(
        description='Process SCT results.'
                    ' This program takes the SCT summary and sequence files,'
                    ' and generates a nice report in mardown format.',
Vincent Stehlé's avatar
Vincent Stehlé committed
278
279
280
281
282
        epilog='When sorting is requested, tests data are sorted'
               ' according to the first sort key, then the second, etc.'
               ' Sorting happens after update by the configuration rules.'
               ' Useful example: --sort'
               ' "group,descr,set guid,test set,sub set,guid,name,log"',
Vincent Stehlé's avatar
Vincent Stehlé committed
283
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
Vincent Stehlé's avatar
Vincent Stehlé committed
284
    parser.add_argument('--csv', help='Output .csv filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
285
    parser.add_argument('--json', help='Output .json filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
286
287
    parser.add_argument(
        '--md', help='Output .md filename', default='result.md')
Vincent Stehlé's avatar
Vincent Stehlé committed
288
289
    parser.add_argument(
        '--debug', action='store_true', help='Turn on debug messages')
Vincent Stehlé's avatar
Vincent Stehlé committed
290
291
    parser.add_argument(
        '--sort', help='Comma-separated list of keys to sort output on')
Vincent Stehlé's avatar
Vincent Stehlé committed
292
293
294
295
296
297
298
299
300
301
    parser.add_argument(
        'log_file', nargs='?', default='sample.ekl',
        help='Input .ekl filename')
    parser.add_argument(
        'seq_file', nargs='?', default='sample.seq',
        help='Input .seq filename')
    parser.add_argument('find_key', nargs='?', help='Search key')
    parser.add_argument('find_value', nargs='?', help='Search value')
    args = parser.parse_args()

Vincent Stehlé's avatar
Vincent Stehlé committed
302
303
304
305
    logging.basicConfig(
        format='%(levelname)s %(funcName)s: %(message)s',
        level=logging.DEBUG if args.debug else logging.INFO)

Vincent Stehlé's avatar
Vincent Stehlé committed
306
    #Command line argument 1, ekl file to open
307
    db1 = list() #"database 1" all tests.
Vincent Stehlé's avatar
Vincent Stehlé committed
308
309
    logging.debug(f'Read {args.log_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
310
    with open(args.log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
311
312
        db1 = ekl_parser(f.readlines())

Vincent Stehlé's avatar
Vincent Stehlé committed
313
314
    logging.debug('{} test(s)'.format(len(db1)))

Vincent Stehlé's avatar
Vincent Stehlé committed
315
    #Command line argument 2, seq file to open
316
    db2 = dict() #"database 2" all test sets that should run
Vincent Stehlé's avatar
Vincent Stehlé committed
317
318
    logging.debug(f'Read {args.seq_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
319
    with open(args.seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
320
        db2 = seq_parser(f)
Vincent Stehlé's avatar
Vincent Stehlé committed
321

Vincent Stehlé's avatar
Vincent Stehlé committed
322
323
    logging.debug('{} test set(s)'.format(len(db2)))

324
325
    # Produce a single cross_check database from our two db1 and db2 databases.
    cross_check = combine_dbs(db1, db2)
326

Vincent Stehlé's avatar
Vincent Stehlé committed
327

Vincent Stehlé's avatar
Vincent Stehlé committed
328
329
330
331
    # Sort tests data in-place, if requested
    if args.sort is not None:
        sort_data(cross_check, args.sort)

332
333
334
    # search for failures, warnings, passes & others
    # We detect all present keys in additions to the expected ones. This is
    # handy with config rules overriding the result field with arbitrary values.
335
    res_keys = set(['DROPPED', 'FAILURE', 'WARNING', 'PASS'])
Vincent Stehlé's avatar
Vincent Stehlé committed
336

337
338
339
340
341
342
343
344
345
346
    for x in cross_check:
        res_keys.add(x['result'])

    # Now we fill some bins with tests according to their result
    bins = {}

    for k in res_keys:
        bins[k] = key_value_find(cross_check, "result", k)

    # Print a one-line summary
347
    s = map(
348
349
350
351
        lambda k: '{} {}(s)'.format(len(bins[k]), k.lower()),
        sorted(res_keys))

    logging.info(', '.join(s))
352
353

    # generate MD summary
Vincent Stehlé's avatar
Vincent Stehlé committed
354
355
    logging.debug(f'Generate {args.md}')

Vincent Stehlé's avatar
Vincent Stehlé committed
356
    with open(args.md, 'w') as resultfile:
357
358
359
        resultfile.write("# SCT Summary \n\n")
        resultfile.write("|  |  |\n")
        resultfile.write("|--|--|\n")
360
361
362
363
364
365

        # Loop on all the result values we found for the summary
        for k in sorted(res_keys):
            resultfile.write(
                "|{}:|{}|\n".format(k.title(), len(bins[k])))

366
367
        resultfile.write("\n\n")

368
369
        # Loop on all the result values we found (except PASS) for the sections
        # listing the tests by group
370
        n = 1
371
372
        res_keys_np = set(res_keys)
        res_keys_np.remove('PASS')
Vincent Stehlé's avatar
Vincent Stehlé committed
373

374
375
376
377
        for k in sorted(res_keys_np):
            resultfile.write("## {}. {} by group\n\n".format(n, k.title()))
            key_tree_2_md(bins[k], resultfile, "group")
            n += 1
378

Vincent Stehlé's avatar
Vincent Stehlé committed
379
380
381
    # Generate csv if requested
    if args.csv is not None:
        gen_csv(cross_check, args.csv)
Vincent Stehlé's avatar
Vincent Stehlé committed
382

Vincent Stehlé's avatar
Vincent Stehlé committed
383
384
385
386
    # Generate json if requested
    if args.json is not None:
        gen_json(cross_check, args.json)

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
387
388
    #command line argument 3&4, key are to support a key & value search.
    #these will be displayed in CLI
Vincent Stehlé's avatar
Vincent Stehlé committed
389
390
    if args.find_key is not None and args.find_value is not None:
        found = key_value_find(db1, args.find_key, args.find_value)
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
391
392
393
        #print the dict
        print("found:",len(found),"items with search constraints")
        for x in found:
Vincent Stehlé's avatar
Vincent Stehlé committed
394
395
396
397
            print(
                x["guid"], ":", x["name"], "with", args.find_key, ":",
                x[args.find_key])

398

399
main()