parser.py 13.3 KB
Newer Older
1
#!/usr/bin/env python3
2
3
#SCT log parser

4

5
import sys
Vincent Stehlé's avatar
Vincent Stehlé committed
6
import argparse
Vincent Stehlé's avatar
Vincent Stehlé committed
7
import csv
Vincent Stehlé's avatar
Vincent Stehlé committed
8
import logging
Vincent Stehlé's avatar
Vincent Stehlé committed
9
import json
Vincent Stehlé's avatar
Vincent Stehlé committed
10

11

12
#based loosley on https://stackoverflow.com/a/4391978
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
13
# returns a filtered dict of dicts that meet some Key-value pair.
14
# I.E. key="result" value="FAILURE"
15
def key_value_find(list_1, key, value):
16
    found = list()
17
    for test in list_1:
18
        if test[key] == value:
19
            found.append(test)
20
    return found
21

22
23

#Were we intrept test logs into test dicts
24
def test_parser(string, current):
25
    test_list = {
Vincent Stehlé's avatar
Vincent Stehlé committed
26
      "name": string[2], #FIXME:Sometimes, SCT has name and Description,
27
      "result": string[1],
28
      **current,
29
      "guid": string[0], #FIXME:GUID's overlap
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
30
      #"comment": string[-1], #FIXME:need to hash this out, sometime there is no comments
31
      "log": ' '.join(string[3:])
32
    }
33
    return test_list
Vincent Stehlé's avatar
Vincent Stehlé committed
34

35
#Parse the ekl file, and create a map of the tests
36
37
def ekl_parser (file):
    #create our "database" dict
38
    temp_list = list()
39
    #All tests are grouped by the "HEAD" line the procedes them.
40
41
42
43
44
45
    current = {
        'group': "N/A",
        'test set': "N/A",
        'sub set': "N/A",
        'set guid': "N/A",
    }
46

47
    for i, line in enumerate(file):
Vincent Stehlé's avatar
Vincent Stehlé committed
48
49
50
51
52
53
54
        # Strip the line from trailing whitespaces
        line = line.rstrip()

        # Skip empty line
        if line == '':
            continue

55
56
        # strip the line of | & || used for sepration
        split_line = line.split('|')
57

Vincent Stehlé's avatar
Vincent Stehlé committed
58
        # Skip TERM
59
        if split_line[0] == '' and split_line[1] == "TERM":
60
61
            continue

62
63
64
        # The "HEAD" tag is the only indcation we are on a new test set
        if split_line[0] == '' and split_line[1] == "HEAD":
            # split the header into test group and test set.
65
            try:
66
67
68
                group, Set = split_line[12].split('\\')
            except Exception:
                group, Set = '', split_line[12]
69
70
71
            current = {
                'group': group,
                'test set': Set,
72
73
74
75
76
77
78
79
                'sub set': split_line[10],
                'set guid': split_line[8],
                'iteration': split_line[4],
                'start date': split_line[6],
                'start time': split_line[7],
                'revision': split_line[9],
                'descr': split_line[11],
                'device path': '|'.join(split_line[13:]),
80
            }
81

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
82
        #FIXME:? EKL file has an inconsistent line structure,
83
84
        # sometime we see a line that consits ' dump of GOP->I\n'
        #easiest way to skip is check for blank space in the first char
85
        elif split_line[0] != '' and split_line[0][0] != " ":
86
87
88
89
            try:
                #deliminiate on ':' for tests
                split_test = [new_string for old_string in split_line for new_string in old_string.split(':')]
                #put the test into a dict, and then place that dict in another dict with GUID as key
90
                tmp_dict = test_parser(split_test, current)
91
92
93
94
                temp_list.append(tmp_dict)
            except:
                print("Line:",split_line)
                sys.exit("your log may be corrupted")
95
96
97
        else:
            logging.error(f"Unparsed line {i} `{line}'")

98
    return temp_list
99

Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
100
#Parse Seq file, used to tell which tests should run.
101
def seq_parser(file):
102
    temp_dict = list()
103
    lines=file.readlines()
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
104
    magic=7 #a test in a seq file is 7 lines, if not mod7, something wrong..
105
106
    if len(lines)%magic != 0:
        sys.exit("seqfile cut short, should be mod7")
107
108
109
110
111
112
113
114
    #the utf-16 char makes this looping a bit harder, so we use x+(i) where i is next 0-6th
    for x in range(0,len(lines),magic): #loop ever "7 lines"
        #(x+0)[Test Case]
        #(x+1)Revision=0x10000
        #(x+2)Guid=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
        #(x+3)Name=InstallAcpiTableFunction
        #(x+4)Order=0xFFFFFFFF
        #(x+5)Iterations=0xFFFFFFFF
Vincent Stehlé's avatar
Vincent Stehlé committed
115
        #(x+6)(utf-16 char)
116
        #currently only add tests that are supposed to run, should add all?
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
117
        #0xFFFFFFFF in "Iterations" means the test is NOT supposed to run
118
119
120
121
122
123
124
125
        if not "0xFFFFFFFF" in lines[x+5]:
            seq_dict = {
                "name": lines[x+3][5:-1],#from after "Name=" to end (5char long)
                "guid": lines[x+2][5:-1],#from after"Guid=" to the end, (5char long)
                "Iteration": lines[x+5][11:-1],#from after "Iterations=" (11char long)
                "rev": lines[x+1][9:-1],#from after "Revision=" (9char long)
                "Order": lines[x+4][6:-1]#from after "Order=" (6char long)
            }
126
            temp_dict.append(seq_dict) #put in a dict based on guid
127

128
    return temp_dict
129

130
131
132
133
#group items by key, and print by key
#we slowly iterate through the list, group and print groups
def key_tree_2_md(input_list,file,key):
    #make a copy so we don't destroy the first list.
Vincent Stehlé's avatar
Vincent Stehlé committed
134
    temp_list = input_list.copy()
135
136
137
138
139
140
141
142
143
144
145
146
    while temp_list:
        test_dict = temp_list.pop()
        found, not_found = [test_dict],[]
        #go through whole list looking for key match
        while temp_list:
            next_dict = temp_list.pop()
            if next_dict[key] == test_dict[key]: #if match add to found
                found.append(next_dict)
            else: #else not found
                not_found.append(next_dict)
        temp_list = not_found #start over with found items removed
        file.write("### " + test_dict[key])
147
        dict_2_md(found,file)
Vincent Stehlé's avatar
Vincent Stehlé committed
148

149
150


151
152
153
#generic writer, takes a list of dicts and turns the dicts into an MD table.
def dict_2_md(input_list,file):
    if len(input_list) > 0:
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
154
        file.write("\n\n")
155
156
157
158
        #create header for MD table using dict keys
        temp_string1, temp_string2 = "|", "|"
        for x in (input_list[0].keys()):
            temp_string1 += (x + "|")
Jeff Booher-Kaeding's avatar
Jeff Booher-Kaeding committed
159
            temp_string2 += ("---|")
160
161
162
163
164
165
166
        file.write(temp_string1+"\n"+temp_string2+"\n")
        #print each item from the dict into the table
        for x in input_list:
            test_string = "|"
            for y in x.keys():
                test_string += (x[y] + "|")
            file.write(test_string+'\n')
Vincent Stehlé's avatar
Vincent Stehlé committed
167
    #seprate table from other items in MD
168
    file.write("\n\n")
169

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
170

Vincent Stehlé's avatar
Vincent Stehlé committed
171
172
173
174
175
176
177
178
179
180
# Sort tests data in-place
# sort_keys is a comma-separated list
# The first key has precedence, then the second, etc.
# To use python list in-place sorting, we use the keys in reverse order.
def sort_data(cross_check, sort_keys):
    logging.debug(f"Sorting on `{sort_keys}'")
    for k in reversed(sort_keys.split(',')):
        cross_check.sort(key=lambda x: x[k])


Vincent Stehlé's avatar
Vincent Stehlé committed
181
182
183
184
185
186
187
188
189
# Generate csv
def gen_csv(cross_check, filename):
    # Find keys
    keys = set()

    for x in cross_check:
        keys = keys.union(x.keys())

    # Write csv
Vincent Stehlé's avatar
Vincent Stehlé committed
190
191
    logging.debug(f'Generate {filename}')

Vincent Stehlé's avatar
Vincent Stehlé committed
192
193
194
195
196
197
198
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.DictWriter(
            csvfile, fieldnames=sorted(keys), delimiter=';')
        writer.writeheader()
        writer.writerows(cross_check)


Vincent Stehlé's avatar
Vincent Stehlé committed
199
200
201
202
203
204
205
206
# Generate json
def gen_json(cross_check, filename):
    logging.debug(f'Generate {filename}')

    with open(filename, 'w') as jsonfile:
        json.dump(cross_check, jsonfile, sort_keys=True, indent=2)


207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
# Combine or two databases db1 and db2 coming from ekl and seq files
# respectively into a single cross_check database
# Tests in db1, which were not meant to be run according to db2 have their
# results forced to SPURIOUS.
# Tests sets in db2, which were not run according to db1 have an artificial
# test entry created with result DROPPED.
def combine_dbs(db1, db2):
    cross_check = db1

    # Do a pass to verify that all tests in db1 were meant to be run.
    # Otherwise, force the result to SPURIOUS.
    s = set()

    for x in db2:
        s.add(x['guid'])

    n = 0

    for i in range(len(cross_check)):
        if cross_check[i]['set guid'] not in s:
            logging.debug(f"Spurious test {i} `{cross_check[i]['name']}'")
            cross_check[i]['result'] = 'SPURIOUS'
            n += 1

    if n:
        logging.debug(f'{n} spurious test(s)')

    # Do a pass to autodetect all tests fields in case we need to merge dropped
    # tests sets entries
    f = {}

    for x in cross_check:
        for k in x.keys():
            f[k] = ''

    logging.debug(f'Test fields: {f.keys()}')

    # Do a pass to find the test sets that did not run for whatever reason.
    s = set()

    for x in cross_check:
        s.add(x['set guid'])

    n = 0

    for i in range(len(db2)):
        x = db2[i]

        if not x['guid'] in s:
            logging.debug(f"Dropped test set {i} `{x['name']}'")

            # Create an artificial test entry to reflect the dropped test set
            cross_check.append({
                **f,
                'sub set': x['name'],
                'set guid': x['guid'],
                'revision': x['rev'],
                'group': 'Unknown',
                'result': 'DROPPED',
            })

            n += 1

    if n:
        logging.debug(f'{n} dropped test set(s)')

    return cross_check


276
def main():
Vincent Stehlé's avatar
Vincent Stehlé committed
277
278
279
280
    parser = argparse.ArgumentParser(
        description='Process SCT results.'
                    ' This program takes the SCT summary and sequence files,'
                    ' and generates a nice report in mardown format.',
Vincent Stehlé's avatar
Vincent Stehlé committed
281
282
283
284
285
        epilog='When sorting is requested, tests data are sorted'
               ' according to the first sort key, then the second, etc.'
               ' Sorting happens after update by the configuration rules.'
               ' Useful example: --sort'
               ' "group,descr,set guid,test set,sub set,guid,name,log"',
Vincent Stehlé's avatar
Vincent Stehlé committed
286
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
Vincent Stehlé's avatar
Vincent Stehlé committed
287
    parser.add_argument('--csv', help='Output .csv filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
288
    parser.add_argument('--json', help='Output .json filename')
Vincent Stehlé's avatar
Vincent Stehlé committed
289
290
    parser.add_argument(
        '--md', help='Output .md filename', default='result.md')
Vincent Stehlé's avatar
Vincent Stehlé committed
291
292
    parser.add_argument(
        '--debug', action='store_true', help='Turn on debug messages')
Vincent Stehlé's avatar
Vincent Stehlé committed
293
294
    parser.add_argument(
        '--sort', help='Comma-separated list of keys to sort output on')
Vincent Stehlé's avatar
Vincent Stehlé committed
295
296
297
298
299
300
301
302
303
304
    parser.add_argument(
        'log_file', nargs='?', default='sample.ekl',
        help='Input .ekl filename')
    parser.add_argument(
        'seq_file', nargs='?', default='sample.seq',
        help='Input .seq filename')
    parser.add_argument('find_key', nargs='?', help='Search key')
    parser.add_argument('find_value', nargs='?', help='Search value')
    args = parser.parse_args()

Vincent Stehlé's avatar
Vincent Stehlé committed
305
306
307
308
    logging.basicConfig(
        format='%(levelname)s %(funcName)s: %(message)s',
        level=logging.DEBUG if args.debug else logging.INFO)

Vincent Stehlé's avatar
Vincent Stehlé committed
309
    #Command line argument 1, ekl file to open
310
    db1 = list() #"database 1" all tests.
Vincent Stehlé's avatar
Vincent Stehlé committed
311
312
    logging.debug(f'Read {args.log_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
313
    with open(args.log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
314
315
        db1 = ekl_parser(f.readlines())

Vincent Stehlé's avatar
Vincent Stehlé committed
316
317
    logging.debug('{} test(s)'.format(len(db1)))

Vincent Stehlé's avatar
Vincent Stehlé committed
318
    #Command line argument 2, seq file to open
319
    db2 = dict() #"database 2" all test sets that should run
Vincent Stehlé's avatar
Vincent Stehlé committed
320
321
    logging.debug(f'Read {args.seq_file}')

Vincent Stehlé's avatar
Vincent Stehlé committed
322
    with open(args.seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
323
        db2 = seq_parser(f)
Vincent Stehlé's avatar
Vincent Stehlé committed
324

Vincent Stehlé's avatar
Vincent Stehlé committed
325
326
    logging.debug('{} test set(s)'.format(len(db2)))

327
328
    # Produce a single cross_check database from our two db1 and db2 databases.
    cross_check = combine_dbs(db1, db2)
329

Vincent Stehlé's avatar
Vincent Stehlé committed
330

Vincent Stehlé's avatar
Vincent Stehlé committed
331
332
333
334
    # Sort tests data in-place, if requested
    if args.sort is not None:
        sort_data(cross_check, args.sort)

335
336
337
    # search for failures, warnings, passes & others
    # We detect all present keys in additions to the expected ones. This is
    # handy with config rules overriding the result field with arbitrary values.
338
    res_keys = set(['DROPPED', 'FAILURE', 'WARNING', 'PASS'])
Vincent Stehlé's avatar
Vincent Stehlé committed
339

340
341
342
343
344
345
346
347
348
349
    for x in cross_check:
        res_keys.add(x['result'])

    # Now we fill some bins with tests according to their result
    bins = {}

    for k in res_keys:
        bins[k] = key_value_find(cross_check, "result", k)

    # Print a one-line summary
350
    s = map(
351
352
353
354
        lambda k: '{} {}(s)'.format(len(bins[k]), k.lower()),
        sorted(res_keys))

    logging.info(', '.join(s))
355
356

    # generate MD summary
Vincent Stehlé's avatar
Vincent Stehlé committed
357
358
    logging.debug(f'Generate {args.md}')

Vincent Stehlé's avatar
Vincent Stehlé committed
359
    with open(args.md, 'w') as resultfile:
360
361
362
        resultfile.write("# SCT Summary \n\n")
        resultfile.write("|  |  |\n")
        resultfile.write("|--|--|\n")
363
364
365
366
367
368

        # Loop on all the result values we found for the summary
        for k in sorted(res_keys):
            resultfile.write(
                "|{}:|{}|\n".format(k.title(), len(bins[k])))

369
370
        resultfile.write("\n\n")

371
372
        # Loop on all the result values we found (except PASS) for the sections
        # listing the tests by group
373
        n = 1
374
375
        res_keys_np = set(res_keys)
        res_keys_np.remove('PASS')
Vincent Stehlé's avatar
Vincent Stehlé committed
376

377
378
379
380
        for k in sorted(res_keys_np):
            resultfile.write("## {}. {} by group\n\n".format(n, k.title()))
            key_tree_2_md(bins[k], resultfile, "group")
            n += 1
381

Vincent Stehlé's avatar
Vincent Stehlé committed
382
383
384
    # Generate csv if requested
    if args.csv is not None:
        gen_csv(cross_check, args.csv)
Vincent Stehlé's avatar
Vincent Stehlé committed
385

Vincent Stehlé's avatar
Vincent Stehlé committed
386
387
388
389
    # Generate json if requested
    if args.json is not None:
        gen_json(cross_check, args.json)

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
390
391
    #command line argument 3&4, key are to support a key & value search.
    #these will be displayed in CLI
Vincent Stehlé's avatar
Vincent Stehlé committed
392
393
    if args.find_key is not None and args.find_value is not None:
        found = key_value_find(db1, args.find_key, args.find_value)
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
394
395
396
        #print the dict
        print("found:",len(found),"items with search constraints")
        for x in found:
Vincent Stehlé's avatar
Vincent Stehlé committed
397
398
399
400
            print(
                x["guid"], ":", x["name"], "with", args.find_key, ":",
                x[args.find_key])

401

402
main()