Commit 8bd1de3c authored by Vincent Stehlé's avatar Vincent Stehlé
Browse files

parser.py: add an option to keep only certain fields



Keeping only some fields is handy to generate summaries for triaging.

This does not apply to the markdown and config template formats, which
depend on certain fields being present.
Signed-off-by: Vincent Stehlé's avatarVincent Stehlé <vincent.stehle@arm.com>
parent 2949991d
...@@ -65,6 +65,19 @@ $ ./parser.py \ ...@@ -65,6 +65,19 @@ $ ./parser.py \
or True" ... or True" ...
``` ```
### Keeping only certain fields
Except for the markdown and the config template formats, it is possible to
specify which tests data fields to actually write using the `--fields` option.
Example command, suitable for triaging:
``` {.sh}
$ ./parser.py --fields 'result,sub set,descr,name,log' ...
```
The csv format can retain the fields order.
## Configuration file ## Configuration file
It is possible to use a configuration file with command line option `--config It is possible to use a configuration file with command line option `--config
......
...@@ -349,20 +349,49 @@ def sort_data(cross_check, sort_keys): ...@@ -349,20 +349,49 @@ def sort_data(cross_check, sort_keys):
cross_check.sort(key=lambda x: x[k]) cross_check.sort(key=lambda x: x[k])
# Generate csv # Keep only certain fields in data, in-place
def gen_csv(cross_check, filename): # The fields to write are supplied as a comma-separated list
# Find keys def keep_fields(cross_check, fields):
keys = set() logging.debug(f"Keeping fields: `{fields}'")
s = set(fields.split(','))
for x in cross_check: for x in cross_check:
keys = keys.union(x.keys()) for k in list(x.keys()):
if k not in s:
del x[k]
# Discover fields
# The fields can be supplied as a comma-separated list
# Order is preserved
# Additional fields are auto-discovered and added to the list, sorted
def discover_fields(cross_check, fields=None):
if fields is not None:
keys = fields.split(',')
else:
keys = []
# Find keys, not already listed
s = set()
# Write csv for x in cross_check:
logging.debug(f'Generate {filename}') s = s.union(x.keys())
s = s.difference(keys)
keys += sorted(s)
logging.debug(f'Fields: {keys}')
return keys
# Generate csv
# The fields to write are supplied as a list
def gen_csv(cross_check, filename, fields):
logging.debug(f'Generate {filename} (fields: {fields})')
with open(filename, 'w', newline='') as csvfile: with open(filename, 'w', newline='') as csvfile:
writer = csv.DictWriter( writer = csv.DictWriter(
csvfile, fieldnames=sorted(keys), delimiter=';') csvfile, fieldnames=fields, delimiter=';')
writer.writeheader() writer.writeheader()
writer.writerows(cross_check) writer.writerows(cross_check)
...@@ -511,6 +540,8 @@ def main(): ...@@ -511,6 +540,8 @@ def main():
parser.add_argument( parser.add_argument(
'--sort', help='Comma-separated list of keys to sort output on') '--sort', help='Comma-separated list of keys to sort output on')
parser.add_argument('--filter', help='Python expression to filter results') parser.add_argument('--filter', help='Python expression to filter results')
parser.add_argument(
'--fields', help='Comma-separated list of fields to write')
parser.add_argument( parser.add_argument(
'log_file', nargs='?', default='sample.ekl', 'log_file', nargs='?', default='sample.ekl',
help='Input .ekl filename') help='Input .ekl filename')
...@@ -616,9 +647,21 @@ def main(): ...@@ -616,9 +647,21 @@ def main():
key_tree_2_md(bins[k], resultfile, "group") key_tree_2_md(bins[k], resultfile, "group")
n += 1 n += 1
# Generate yaml config template if requested
if 'template' in args and args.template is not None:
gen_template(cross_check, args.template)
# Filter fields before writing any other type of output
# Do not rely on specific fields being present after this step
if args.fields is not None:
keep_fields(cross_check, args.fields)
# Auto-discover the fields and take the option into account
fields = discover_fields(cross_check, args.fields)
# Generate csv if requested # Generate csv if requested
if args.csv is not None: if args.csv is not None:
gen_csv(cross_check, args.csv) gen_csv(cross_check, args.csv, fields)
# Generate json if requested # Generate json if requested
if args.json is not None: if args.json is not None:
...@@ -628,10 +671,6 @@ def main(): ...@@ -628,10 +671,6 @@ def main():
if 'yaml' in args and args.yaml is not None: if 'yaml' in args and args.yaml is not None:
gen_yaml(cross_check, args.yaml) gen_yaml(cross_check, args.yaml)
# Generate yaml config template if requested
if 'template' in args and args.template is not None:
gen_template(cross_check, args.template)
# command line argument 3&4, key are to support a key & value search. # command line argument 3&4, key are to support a key & value search.
# these will be displayed in CLI # these will be displayed in CLI
if args.find_key is not None and args.find_value is not None: if args.find_key is not None and args.find_value is not None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment