Parser.py 7.08 KB
Newer Older
1
2
#SCT log parser

3
import sys
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
4
import csv
5

6
#based loosley on https://stackoverflow.com/a/4391978
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
7
# returns a filtered dict of dicts that meet some Key-value pair.
8
# I.E. key="result" value="FAILURE"
9
def key_value_find(dict1, key, value):
10
    found = {}
11
12
    for key2 in dict1:
        test = dict1[key2]
13
        if test[key] == value:
14
            found[key2]=test
15
    return found
16
17

#Were we intrept test logs into test dicts
18
def test_parser(string,current_group,current_test_set,current_set_guid,current_sub_set):
19
20
21
22
23
    test_dict = {
      "name": string[2], #FIXME:ACS just the name, SCT has name and Description. 
                         # ACS tests don't follow the same format as the rest of UEFI tests
      "result": string[1],
      "group": current_group,
24
25
26
      "test set": current_test_set,  
      "sub set": current_sub_set,
      "set guid": current_set_guid,
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
27
28
      "guid": string[0], #FIXME:ACS GUID's overlap... need fix... 
      #"comment": string[-1], #FIXME:need to hash this out, sometime there is no comments
29
30
31
      "log": string
    }
    return test_dict["guid"], test_dict
32
    
33
#Parse the ekl file, and create a map of the tests
34
35
36
def ekl_parser (file):
    #create our "database" dict
    db_dict = dict()
37
    #All tests are grouped by the "HEAD" line the procedes them.
38
39
    current_group = "N/A"
    current_set = "N/A"
40
41
    current_set_guid = "N/A"
    current_sub_set = "N/A"
42
43
44
45
46
47
48
49
50
51

    for line in file:
        #strip the line of | & || used for sepration
        split_line = [string for string in line.split('|') if string != ""]

        #TODO:I can skip TERM, but I feel like "\n" should be able to be handled in the above list comprehension 
        if split_line[0]=="TERM" or split_line[0]=="\n":
            continue

        #The "HEAD" tag is the only indcation we are on a new test set
52
        #i.e. 
53
54
55
        if split_line[0]=="HEAD":
            #split the header into test group and test set.
            current_group, current_set = split_line[8].split('\\')
56
57
            current_set_guid = split_line[4]
            current_sub_set = split_line[6]
58

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
59
        #FIXME:? EKL file has an inconsistent line structure,
60
61
62
        # sometime we see a line that consits ' dump of GOP->I\n'
        #easiest way to skip is check for blank space in the first char
        elif split_line[0][0] != " ":
63
            #deliminiate on ':' for tests
64
            split_test = [new_string for old_string in split_line for new_string in old_string.split(':')]
65
            #put the test into a dict, and then place that dict in another dict with GUID as key
66
            guid,tmp_dict = test_parser(split_test,current_group,current_set,current_set_guid,current_sub_set)
67
            db_dict[guid]=tmp_dict
68
69
70

    return db_dict

71
72
73
def seq_parser(file):
    db_dict = dict()
    lines=file.readlines()
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
74
    magic=7 #a test in a seq file is 7 lines, if not mod7, something wrong..
75
76
    if len(lines)%magic != 0:
        sys.exit("seqfile cut short, should be mod7")
77
78
79
80
81
82
83
84
85
86
    #the utf-16 char makes this looping a bit harder, so we use x+(i) where i is next 0-6th
    for x in range(0,len(lines),magic): #loop ever "7 lines"
        #(x+0)[Test Case]
        #(x+1)Revision=0x10000
        #(x+2)Guid=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
        #(x+3)Name=InstallAcpiTableFunction
        #(x+4)Order=0xFFFFFFFF
        #(x+5)Iterations=0xFFFFFFFF
        #(x+6)(utf-16 char) 
        #currently only add tests that are supposed to run, should add all?
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
87
        #0xFFFFFFFF in "Iterations" means the test is NOT supposed to run
88
89
90
91
92
93
94
95
        if not "0xFFFFFFFF" in lines[x+5]:
            seq_dict = {
                "name": lines[x+3][5:-1],#from after "Name=" to end (5char long)
                "guid": lines[x+2][5:-1],#from after"Guid=" to the end, (5char long)
                "Iteration": lines[x+5][11:-1],#from after "Iterations=" (11char long)
                "rev": lines[x+1][9:-1],#from after "Revision=" (9char long)
                "Order": lines[x+4][6:-1]#from after "Order=" (6char long)
            }
96
            db_dict[seq_dict["guid"]]=seq_dict #put in a dict based on guid
97
98
99

    return db_dict

100

Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
101

102
def main():
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
103
104
105
106
107
108
109
110
111
112
    #Command line argument 1, ekl file to open, else open sample
    log_file = sys.argv[1] if len(sys.argv) >= 2 else "sample.ekl"
    db1 = {} #"database 1" all tests.
    with open(log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
        db1 = ekl_parser(f.readlines())

    #Command line argument 2, seq file to open, else open sample
    seq_file = sys.argv[2] if len(sys.argv) >= 3 else "sample.seq"
    db2 = {} #"database 2" all test sets that should run
    with open(seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
113
        db2 = seq_parser(f)
114
    
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
115
    #cross check is filled only with tests labled as "run" int the seq file
116
    cross_check_dict = dict()
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
117
118
119
120
121
    #combine a list of test sets that did not run for whatever reason.
    would_not_run = []
    for x in db2: #for each "set guid" in db2
        temp_dict = key_value_find(db1,"set guid",x)#find tests in db1 with given set guid
        if bool(temp_dict): #if its not empty, apprend it to our dict
122
            cross_check_dict = {**cross_check_dict, **temp_dict}
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
123
124
        else: #if it is empty, this test set was not run.
            would_not_run.append(db2[x]) 
125
126

    
Jeff Booher-Kaeding's avatar
V1.0?    
Jeff Booher-Kaeding committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
    #search for failures and warnings & passes,
    failures = key_value_find(cross_check_dict,"result","FAILURE")
    warnings = key_value_find(cross_check_dict,"result","WARNING")
    passes = key_value_find(cross_check_dict,"result","PASS")
    fail_and_warn = {**failures, **warnings}#dict of failures and warnings


    # generate CSV summary
    with open('result.csv', 'w') as csvfile:
        result_writer = csv.writer(csvfile, delimiter=',')
        result_writer.writerow(['']*9) 
        result_writer.writerow(["Failures:",len(failures)])
        result_writer.writerow(["Warnings:",len(warnings)])
        result_writer.writerow(["Pass:",len(passes)])
        result_writer.writerow(['']*9) 

        #If there were any silently dropped, lets print them
        if len(would_not_run) > 0:
            result_writer.writerow(["Silently Dropped:"]) 
            not_run_writer = csv.DictWriter(csvfile, would_not_run[0])
            not_run_writer.writeheader()
            for x in would_not_run:
                not_run_writer.writerow(x)
            result_writer.writerow(['']*9)
        
        #lets print every test that failed or had a wanring. if any did!
        result_writer.writerow(["Fail & Warn tests:"]) 
        first_guid = (list(fail_and_warn.keys())[0])
        test_writer = csv.DictWriter(csvfile, fail_and_warn[first_guid])
        test_writer.writeheader()
        for x in fail_and_warn:
            test_writer.writerow(fail_and_warn[x])


    #command line argument 3&4, key are to support a key & value search.
    #these will be displayed in CLI
    if len(sys.argv) >= 5:
        find_key = sys.argv[3]
        find_value = sys.argv[4]
        found = key_value_find(db1,find_key,find_value)
        #print the dict
        print("found:",len(found),"items with search constraints")
        for x in found:
            print(found[x]["guid"],":",found[x]["name"],"with",found[x][find_key],":",found[x][find_value])
171

172
main()