gallium/tools: update trace scripts to Python 3

Bring the scripts for parsing, dumping state and diffing of Gallium
trace files to modern day by updating them to Python 3.

Add option '-p' to some tools for outputting only plaintext
instead of ANSI / colorized format.

Also fix state parsing of some dumps by adding 'clear_render_target'
and 'get_disk_shader_cache' to ignored calls list.

Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/4321
Signed-off-by: Matti Hamalainen <ccr@tnsp.org>
Acked-By: Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9830>

7 files changed, 81 insertions, 76 deletions

diff --git a/src/gallium/tools/trace/diff_state.py b/src/gallium/tools/trace/diff_state.py
index 877dc388a8e..4f7e5b076fa 100755
--- a/src/gallium/tools/trace/diff_state.py
+++ b/src/gallium/tools/trace/diff_state.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 #
 # Copyright 2011 Jose Fonseca
@@ -26,7 +26,7 @@
 
 
 import json
-import optparse
+import argparse
 import re
 import difflib
 import sys
@@ -79,8 +79,7 @@ class Dumper(Visitor):
     def visitObject(self, node):
         self.enter_object()
 
-        members = node.keys()
-        members.sort()
+        members = sorted(node)
         for i in range(len(members)):
             name = members[i]
             value = node[name]
@@ -147,10 +146,8 @@ class Comparer(Visitor):
             return False
         if len(a) != len(b) and not self.ignore_added:
             return False
-        ak = a.keys()
-        bk = b.keys()
-        ak.sort()
-        bk.sort()
+        ak = sorted(a)
+        bk = sorted(b)
         if ak != bk and not self.ignore_added:
             return False
         for k in ak:
@@ -174,11 +171,11 @@ class Comparer(Visitor):
         return True
 
     def visitValue(self, a, b):
-        if isinstance(a, float) or isinstance(b, float):
+        if isinstance(a, float) and isinstance(b, float):
             if a == 0:
                 return abs(b) < self.tolerance
             else:
-                return abs((b - a)/a) < self.tolerance
+                return abs((b - a) / a) < self.tolerance
         else:
             return a == b
 
@@ -247,7 +244,7 @@ class Differ(Visitor):
             self.replace(a, b)
 
     def replace(self, a, b):
-        if isinstance(a, basestring) and isinstance(b, basestring):
+        if isinstance(a, str) and isinstance(b, str):
             if '\n' in a or '\n' in b:
                 a = a.splitlines()
                 b = b.splitlines()
@@ -276,7 +273,7 @@ class Differ(Visitor):
         self.dumper.visit(b)
 
     def isMultilineString(self, value):
-        return isinstance(value, basestring) and '\n' in value
+        return isinstance(value, str) and '\n' in value
     
     def replaceMultilineString(self, a, b):
         self.dumper.visit(a)
@@ -330,20 +327,21 @@ def load(stream, strip_images = True, strip_comments = True):
 
 
 def main():
-    optparser = optparse.OptionParser(
-        usage="\n\t%prog [options] <ref_json> <src_json>")
-    optparser.add_option(
-        '--keep-images',
+    optparser = argparse.ArgumentParser(
+        description="Diff JSON format state dump files")
+    optparser.add_argument("-k", "--keep-images",
         action="store_false", dest="strip_images", default=True,
         help="compare images")
 
-    (options, args) = optparser.parse_args(sys.argv[1:])
+    optparser.add_argument("ref_json", action="store",
+        type=str, help="reference state file")
+    optparser.add_argument("src_json", action="store",
+        type=str, help="source state file")
 
-    if len(args) != 2:
-        optparser.error('incorrect number of arguments')
+    args = optparser.parse_args()
 
-    a = load(open(sys.argv[1], 'rt'), options.strip_images)
-    b = load(open(sys.argv[2], 'rt'), options.strip_images)
+    a = load(open(args.ref_json, 'rt'), args.strip_images)
+    b = load(open(args.src_json, 'rt'), args.strip_images)
 
     if False:
         dumper = Dumper()
diff --git a/src/gallium/tools/trace/dump.py b/src/gallium/tools/trace/dump.py
index 10211c87c56..b80e27ae423 100755
--- a/src/gallium/tools/trace/dump.py
+++ b/src/gallium/tools/trace/dump.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 # 
 # Copyright 2008 VMware, Inc.
diff --git a/src/gallium/tools/trace/dump_state.py b/src/gallium/tools/trace/dump_state.py
index 2622d130bf8..8b68d83f9a2 100755
--- a/src/gallium/tools/trace/dump_state.py
+++ b/src/gallium/tools/trace/dump_state.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 # 
 # Copyright 2008-2013, VMware, Inc.
@@ -58,8 +58,7 @@ PIPE_SHADER_TYPES    = 4
 
 def serialize(obj):
     '''JSON serializer function for non-standard Python objects.'''
-
-    if isinstance(obj, bytearray):
+    if isinstance(obj, bytearray) or isinstance(obj, bytes):
         # TODO: Decide on a single way of dumping blobs
         if False:
             # Don't dump full blobs, but merely a description of their size and
@@ -72,10 +71,10 @@ def serialize(obj):
             # Dump blobs as an array of 16byte hexadecimals
             res = []
             for i in range(0, len(obj), 16):
-                res.append(binascii.b2a_hex(obj[i: i+16]))
+                res.append(obj[i : i + 16].hex())
             return res
         # Dump blobs as a single hexadecimal string
-        return binascii.b2a_hex(obj)
+        return obj.hex()
 
     # If the object has a __json__ method, use it.
     try:
@@ -207,7 +206,7 @@ class Screen(Dispatcher):
     def resource_destroy(self, resource):
         self.interpreter.unregister_object(resource)
 
-    def fence_finish(self, fence, timeout=None):
+    def fence_finish(self, fence, timeout=None, ctx=None):
         pass
     
     def fence_signalled(self, fence):
@@ -473,7 +472,7 @@ class Context(Dispatcher):
 
         count = min(info.count, self.MAX_ELEMENTS)
         indices = []
-        for i in xrange(info.start, info.start + count):
+        for i in range(info.start, info.start + count):
             offset = self._state.index_buffer.offset + i*index_size
             if offset + index_size > len(data):
                 index = 0
@@ -492,7 +491,7 @@ class Context(Dispatcher):
 
         count = min(count, self.MAX_ELEMENTS)
         vertices = []
-        for index in xrange(start, start + count):
+        for index in range(start, start + count):
             if index >= start + 16:
                 sys.stdout.write('\t...\n')
                 break
@@ -669,7 +668,7 @@ class Context(Dispatcher):
         # Return a fake fence
         return self.interpreter.call_no
 
-    def clear(self, buffers, color, depth, stencil):
+    def clear(self, buffers, color, depth, stencil, scissor_state=None):
         pass
         
     def clear_render_target(self, dst, rgba, dstx, dsty, width, height):
@@ -717,11 +716,13 @@ class Interpreter(parser.TraceDumper):
             ('pipe_screen', 'get_param'),
             ('pipe_screen', 'get_paramf'),
             ('pipe_screen', 'get_shader_param'),
+            ('pipe_screen', 'get_disk_shader_cache'),
             ('pipe_context', 'clear_render_target'), # XXX workaround trace bugs
+            ('pipe_context', 'flush_resource'),
     ))
 
     def __init__(self, stream, options):
-        parser.TraceDumper.__init__(self, stream, sys.stderr)
+        parser.TraceDumper.__init__(self, stream, options, sys.stderr)
         self.options = options
         self.objects = {}
         self.result = None
@@ -767,7 +768,7 @@ class Interpreter(parser.TraceDumper):
             args = args[1:]
         else:
             obj = self.globl
-            
+
         method = getattr(obj, call.method)
         ret = method(**dict(args))
         
@@ -793,10 +794,10 @@ class Main(parser.Main):
         '''Custom options.'''
 
         optparser = parser.Main.get_optparser(self)
-        optparser.add_option("-q", "--quiet", action="store_const", const=0, dest="verbosity", help="no messages")
-        optparser.add_option("-v", "--verbose", action="count", dest="verbosity", default=0, help="increase verbosity level")
-        optparser.add_option("-c", "--call", action="store", type="int", dest="call", default=0xffffffff, help="dump on this call")
-        optparser.add_option("-d", "--draw", action="store", type="int", dest="draw", default=0xffffffff, help="dump on this draw")
+        optparser.add_argument("-v", "--verbose", action="count", default=0, dest="verbosity", help="increase verbosity level")
+        optparser.add_argument("-q", "--quiet", action="store_const", const=0, dest="verbosity", help="no messages")
+        optparser.add_argument("-c", "--call", action="store", type=int, dest="call", default=0xffffffff, help="dump on this call")
+        optparser.add_argument("-d", "--draw", action="store", type=int, dest="draw", default=0xffffffff, help="dump on this draw")
         return optparser
 
     def process_arg(self, stream, options):
diff --git a/src/gallium/tools/trace/format.py b/src/gallium/tools/trace/format.py
index de6f4021597..d50675c45d0 100755
--- a/src/gallium/tools/trace/format.py
+++ b/src/gallium/tools/trace/format.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 #
 # Copyright 2008 VMware, Inc.
@@ -164,9 +164,9 @@ class WindowsConsoleFormatter(Formatter):
 
 
 def DefaultFormatter(stream):
-    if sys.platform in ('linux2', 'cygwin'):
+    if sys.platform in ('linux2', 'linux', 'cygwin'):
         return AnsiFormatter(stream)
-    elif sys.platform in ('win32',):
+    elif sys.platform in ('win32', ):
         return WindowsConsoleFormatter(stream)
     else:
         return Formatter(stream)
diff --git a/src/gallium/tools/trace/model.py b/src/gallium/tools/trace/model.py
index 6a421847113..913a4ef13d2 100755
--- a/src/gallium/tools/trace/model.py
+++ b/src/gallium/tools/trace/model.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 # 
 # Copyright 2008 VMware, Inc.
@@ -33,12 +33,7 @@
 import sys
 import string
 import binascii
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
+from io import StringIO
 import format
 
 
@@ -177,7 +172,7 @@ class PrettyPrinter:
             self.formatter.literal('NULL')
             return
 
-        if isinstance(node.value, basestring):
+        if isinstance(node.value, str):
             self.formatter.literal('"' + node.value + '"')
             return
 
diff --git a/src/gallium/tools/trace/parse.py b/src/gallium/tools/trace/parse.py
index 004b585f95a..a529f389723 100755
--- a/src/gallium/tools/trace/parse.py
+++ b/src/gallium/tools/trace/parse.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 ##########################################################################
 # 
 # Copyright 2008 VMware, Inc.
@@ -27,9 +27,10 @@
 ##########################################################################
 
 
+import io
 import sys
-import xml.parsers.expat
-import optparse
+import xml.parsers.expat as xpat
+import argparse
 
 from model import *
 
@@ -72,7 +73,7 @@ class XmlTokenizer:
         self.character_pos = 0, 0
         self.character_data = ''
         
-        self.parser = xml.parsers.expat.ParserCreate()
+        self.parser = xpat.ParserCreate()
         self.parser.StartElementHandler  = self.handle_element_start
         self.parser.EndElementHandler    = self.handle_element_end
         self.parser.CharacterDataHandler = self.handle_character_data
@@ -112,8 +113,8 @@ class XmlTokenizer:
             data = data.rstrip('\0')
             try:
                 self.parser.Parse(data, self.final)
-            except xml.parsers.expat.ExpatError, e:
-                #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
+            except xpat.ExpatError as e:
+                #if e.code == xpat.errors.XML_ERROR_NO_ELEMENTS:
                 if e.code == 3:
                     pass
                 else:
@@ -205,7 +206,7 @@ class TraceParser(XmlParser):
         attrs = self.element_start('call')
         try:
             no = int(attrs['no'])
-        except KeyError:
+        except KeyError as e:
             self.last_call_no += 1
             no = self.last_call_no
         else:
@@ -352,9 +353,12 @@ class TraceParser(XmlParser):
     
 class TraceDumper(TraceParser):
     
-    def __init__(self, fp, outStream = sys.stdout):
+    def __init__(self, fp, options, outStream = sys.stdout):
         TraceParser.__init__(self, fp)
-        self.formatter = format.DefaultFormatter(outStream)
+        if options.plain:
+            self.formatter = format.Formatter(outStream)
+        else:
+            self.formatter = format.DefaultFormatter(outStream)
         self.pretty_printer = PrettyPrinter(self.formatter)
 
     def handle_call(self, call):
@@ -370,29 +374,36 @@ class Main:
 
     def main(self):
         optparser = self.get_optparser()
-        (options, args) = optparser.parse_args(sys.argv[1:])
-    
-        if not args:
-            optparser.error('insufficient number of arguments')
-
-        for arg in args:
-            if arg.endswith('.gz'):
-                from gzip import GzipFile
-                stream = GzipFile(arg, 'rt')
-            elif arg.endswith('.bz2'):
-                from bz2 import BZ2File
-                stream = BZ2File(arg, 'rU')
-            else:
-                stream = open(arg, 'rt')
-            self.process_arg(stream, options)
+        args = optparser.parse_args()
+
+        for fname in args.filename:
+            try:
+                if fname.endswith('.gz'):
+                    from gzip import GzipFile
+                    stream = io.TextIOWrapper(GzipFile(fname, 'rb'))
+                elif fname.endswith('.bz2'):
+                    from bz2 import BZ2File
+                    stream = io.TextIOWrapper(BZ2File(fname, 'rb'))
+                else:
+                    stream = open(fname, 'rt')
+            except Exception as e:
+                print("ERROR: {}".format(str(e)))
+                sys.exit(1)
+
+            self.process_arg(stream, args)
 
     def get_optparser(self):
-        optparser = optparse.OptionParser(
-            usage="\n\t%prog [options] TRACE  [...]")
+        optparser = argparse.ArgumentParser(
+            description="Parse and dump Gallium trace(s)")
+        optparser.add_argument("filename", action="extend", nargs="+",
+            type=str, metavar="filename", help="Gallium trace filename (plain or .gz, .bz2)")
+        optparser.add_argument("-p", "--plain",
+            action="store_const", const=True, default=False,
+            dest="plain", help="disable ANSI color etc. formatting")
         return optparser
 
     def process_arg(self, stream, options):
-        parser = TraceDumper(stream)
+        parser = TraceDumper(stream, options)
         parser.parse()
 
 
diff --git a/src/gallium/tools/trace/tracediff.sh b/src/gallium/tools/trace/tracediff.sh
index dccb7a3d519..da9dd4a9276 100755
--- a/src/gallium/tools/trace/tracediff.sh
+++ b/src/gallium/tools/trace/tracediff.sh
@@ -29,7 +29,7 @@ set -e
 TRACEDUMP=${TRACEDUMP:-`dirname "$0"`/dump.py}
 
 stripdump () {
-	python $TRACEDUMP "$1" \
+	python3 $TRACEDUMP "$1" \
 	| sed \
 		-e 's@ // time .*@@' \
 		-e 's/\x1b\[[0-9]\{1,2\}\(;[0-9]\{1,2\}\)\{0,2\}m//g' \