Unverified Commit 3276c39d authored by Jeffrey Booher-Kaeding's avatar Jeffrey Booher-Kaeding Committed by GitHub
Browse files

Merge pull request #6 from vstehle/for-jeffrey

Usability upgrades

Rework command line using argparse
Add logging and debug
Add csv output
Use iconv to cat logs
parents e7fbc0c5 5a6f6b7e
......@@ -3,6 +3,10 @@
import sys
import argparse
import csv
import logging
#based loosley on https://stackoverflow.com/a/4391978
# returns a filtered dict of dicts that meet some Key-value pair.
......@@ -16,14 +20,11 @@ def key_value_find(list_1, key, value):
#Were we intrept test logs into test dicts
def test_parser(string,current_group,current_test_set,current_set_guid,current_sub_set):
def test_parser(string, current):
test_list = {
"name": string[2], #FIXME:Sometimes, SCT has name and Description,
"result": string[1],
"group": current_group,
"test set": current_test_set,
"sub set": current_sub_set,
"set guid": current_set_guid,
"guid": string[0], #FIXME:GUID's overlap
#"comment": string[-1], #FIXME:need to hash this out, sometime there is no comments
"log": ' '.join(string[3:])
......@@ -35,10 +36,12 @@ def ekl_parser (file):
#create our "database" dict
temp_list = list()
#All tests are grouped by the "HEAD" line the procedes them.
current_group = "N/A"
current_set = "N/A"
current_set_guid = "N/A"
current_sub_set = "N/A"
current = {
'group': "N/A",
'test set': "N/A",
'sub set': "N/A",
'set guid': "N/A",
for line in file:
# Strip the line from trailing whitespaces
......@@ -59,11 +62,21 @@ def ekl_parser (file):
if split_line[0]=="HEAD":
#split the header into test group and test set.
current_group, current_set = split_line[8].split('\\')
group, Set = split_line[8].split('\\')
current_group, current_set = '', split_line[8]
current_set_guid = split_line[4]
current_sub_set = split_line[6]
group, Set = '', split_line[8]
current = {
'group': group,
'test set': Set,
'sub set': split_line[6],
'set guid': split_line[4],
'iteration': split_line[1],
'start date': split_line[2],
'start time': split_line[3],
'revision': split_line[5],
'descr': split_line[7],
'device path': split_line[9],
#FIXME:? EKL file has an inconsistent line structure,
# sometime we see a line that consits ' dump of GOP->I\n'
......@@ -73,7 +86,7 @@ def ekl_parser (file):
#deliminiate on ':' for tests
split_test = [new_string for old_string in split_line for new_string in old_string.split(':')]
#put the test into a dict, and then place that dict in another dict with GUID as key
tmp_dict = test_parser(split_test,current_group,current_set,current_set_guid,current_sub_set)
tmp_dict = test_parser(split_test, current)
......@@ -151,19 +164,67 @@ def dict_2_md(input_list,file):
# Generate csv
def gen_csv(cross_check, filename):
# Find keys
keys = set()
for x in cross_check:
keys = keys.union(x.keys())
# Write csv
logging.debug(f'Generate {filename}')
with open(filename, 'w', newline='') as csvfile:
writer = csv.DictWriter(
csvfile, fieldnames=sorted(keys), delimiter=';')
def main():
#Command line argument 1, ekl file to open, else open sample
log_file = sys.argv[1] if len(sys.argv) >= 2 else "sample.ekl"
parser = argparse.ArgumentParser(
description='Process SCT results.'
' This program takes the SCT summary and sequence files,'
' and generates a nice report in mardown format.',
parser.add_argument('--csv', help='Output .csv filename')
'--md', help='Output .md filename', default='result.md')
'--debug', action='store_true', help='Turn on debug messages')
'log_file', nargs='?', default='sample.ekl',
help='Input .ekl filename')
'seq_file', nargs='?', default='sample.seq',
help='Input .seq filename')
parser.add_argument('find_key', nargs='?', help='Search key')
parser.add_argument('find_value', nargs='?', help='Search value')
args = parser.parse_args()
format='%(levelname)s %(funcName)s: %(message)s',
level=logging.DEBUG if args.debug else logging.INFO)
#Command line argument 1, ekl file to open
db1 = list() #"database 1" all tests.
with open(log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
logging.debug(f'Read {args.log_file}')
with open(args.log_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
db1 = ekl_parser(f.readlines())
#Command line argument 2, seq file to open, else open sample
seq_file = sys.argv[2] if len(sys.argv) >= 3 else "sample.seq"
logging.debug('{} test(s)'.format(len(db1)))
#Command line argument 2, seq file to open
db2 = dict() #"database 2" all test sets that should run
with open(seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
logging.debug(f'Read {args.seq_file}')
with open(args.seq_file,"r",encoding="utf-16") as f: #files are encoded in utf-16
db2 = seq_parser(f)
logging.debug('{} test set(s)'.format(len(db2)))
#cross check is filled only with tests labled as "run" int the seq file
cross_check = list()
#combine a list of test sets that did not run for whatever reason.
......@@ -182,9 +243,10 @@ def main():
warnings = key_value_find(cross_check,"result","WARNING")
passes = key_value_find(cross_check,"result","PASS")
# generate MD summary
with open('result.md', 'w') as resultfile:
logging.debug(f'Generate {args.md}')
with open(args.md, 'w') as resultfile:
resultfile.write("# SCT Summary \n\n")
resultfile.write("| | |\n")
......@@ -206,16 +268,20 @@ def main():
# Generate csv if requested
if args.csv is not None:
gen_csv(cross_check, args.csv)
#command line argument 3&4, key are to support a key & value search.
#these will be displayed in CLI
if len(sys.argv) >= 5:
find_key = sys.argv[3]
find_value = sys.argv[4]
found = key_value_find(db1,find_key,find_value)
if args.find_key is not None and args.find_value is not None:
found = key_value_find(db1, args.find_key, args.find_value)
#print the dict
print("found:",len(found),"items with search constraints")
for x in found:
x["guid"], ":", x["name"], "with", args.find_key, ":",
......@@ -10,7 +10,9 @@ It will proceed to generate a Markdown file listing number of failures, passes,
## Usage
Usage to generate a "result md" is such. `python3 parser.py <log_file.ekl> <seq_file.seq>`
If you do no provided any command line arguments it will use `sample.ekl` and `sample.seq`.
The output filename can be specified with `--md <filename>`.
An online help is available with the `-h` option.
### Custom search
For a custom Key:value search, the next two arguments *MUST be included together.* The program will search and display files that met that constraint, without the crosscheck, and display the names, guid, and key:value to the command line. `python3 Parser.py <file.ekl> <file.seq> <search key> <search value>`
set -eu
find log/ -name '*.ekl' -exec cat {} \; > cat_summary.ekl
find "$d" -name '*.ekl' -exec iconv -f UTF-16 -t UTF-8 {} \; |\
iconv -f UTF-8 -t UTF-16 > cat_summary.ekl
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment