glthread: compile marshal_generated.c faster by breaking it up into 8 files

Reviewed-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4270>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4270>

src/mapi/glapi/gen/gl_marshal.py

@@ -46,6 +46,8 @@ static inline int safe_mul(int a, int b)
 """
 
 
+file_index = 0
+file_count = 1
 current_indent = 0
 
 
@@ -324,18 +326,27 @@ class PrintCode(gl_XML.gl_print_base):
         out('')
 
     def printBody(self, api):
-        async_funcs = []
+        # The first file only contains the dispatch tables
+        if file_index == 0:
+            self.print_unmarshal_dispatch_cmd(api)
+            self.print_create_marshal_table(api)
+            return
+
+        # The remaining files contain the marshal and unmarshal functions
+        func_per_file = (len(api.functionIterateAll()) // (file_count - 1)) + 1
+        i = -1
         for func in api.functionIterateAll():
+            i += 1
+            if i // func_per_file != (file_index - 1):
+                continue
+
             flavor = func.marshal_flavor()
             if flavor in ('skip', 'custom'):
                 continue
             elif flavor == 'async':
                 self.print_async_body(func)
-                async_funcs.append(func)
             elif flavor == 'sync':
                 self.print_sync_body(func)
-        self.print_unmarshal_dispatch_cmd(api)
-        self.print_create_marshal_table(api)
 
 
 def show_usage():
@@ -347,14 +358,19 @@ if __name__ == '__main__':
     file_name = 'gl_API.xml'
 
     try:
-        (args, trail) = getopt.getopt(sys.argv[1:], 'm:f:')
+        (args, trail) = getopt.getopt(sys.argv[1:], 'm:f:i:n:')
     except Exception:
         show_usage()
 
     for (arg,val) in args:
         if arg == '-f':
             file_name = val
+        elif arg == '-i':
+            file_index = int(val)
+        elif arg == '-n':
+            file_count = int(val)
 
+    assert file_index < file_count
     printer = PrintCode()
 
     api = gl_XML.parse_GL_API(file_name, marshal_XML.marshal_item_factory())

src/mapi/glapi/gen/meson.build

@@ -222,14 +222,17 @@ main_api_exec_c = custom_target(
   capture : true,
 )
 
-main_marshal_generated_c = custom_target(
+main_marshal_generated_c = []
-  'marshal_generated.c',
+foreach x : ['0', '1', '2', '3', '4', '5', '6', '7']
-  input : ['gl_marshal.py', 'gl_and_es_API.xml'],
+  main_marshal_generated_c += custom_target(
-  output : 'marshal_generated.c',
+    'marshal_generated' + x + '.c',
-  command : [prog_python, '@INPUT0@', '-f', '@INPUT1@'],
+    input : ['gl_marshal.py', 'gl_and_es_API.xml'],
-  depend_files : files('marshal_XML.py') + glapi_gen_depends,
+    output : 'marshal_generated' + x + '.c',
-  capture : true,
+    command : [prog_python, '@INPUT0@', '-f', '@INPUT1@', '-i', x, '-n', '8'],
-)
+    depend_files : files('marshal_XML.py') + glapi_gen_depends,
+    capture : true,
+  )
+endforeach
 
 glx_generated = []
 

src/mesa/Android.gen.mk

@@ -87,10 +87,52 @@ $(intermediates)/main/api_exec.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml
 $(intermediates)/main/api_exec.c: $(dispatch_deps)
 	$(call es-gen)
 
-$(intermediates)/main/marshal_generated.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated0.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
-$(intermediates)/main/marshal_generated.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml
+$(intermediates)/main/marshal_generated0.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 0 -n 8
 
-$(intermediates)/main/marshal_generated.c: $(dispatch_deps)
+$(intermediates)/main/marshal_generated0.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated1.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated1.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 1 -n 8
+
+$(intermediates)/main/marshal_generated1.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated2.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated2.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 2 -n 8
+
+$(intermediates)/main/marshal_generated2.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated3.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated3.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 3 -n 8
+
+$(intermediates)/main/marshal_generated3.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated4.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated4.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 4 -n 8
+
+$(intermediates)/main/marshal_generated4.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated5.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated5.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 5 -n 8
+
+$(intermediates)/main/marshal_generated5.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated6.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated6.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 6 -n 8
+
+$(intermediates)/main/marshal_generated6.c: $(dispatch_deps)
+	$(call es-gen)
+
+$(intermediates)/main/marshal_generated7.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal.py
+$(intermediates)/main/marshal_generated7.c: PRIVATE_XML := -f $(glapi)/gl_and_es_API.xml -i 7 -n 8
+
+$(intermediates)/main/marshal_generated7.c: $(dispatch_deps)
 	$(call es-gen)
 
 $(intermediates)/main/marshal_generated.h: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(glapi)/gl_marshal_h.py

src/mesa/Makefile.sources

@@ -145,7 +145,14 @@ MAIN_FILES = \
 	main/lines.c \
 	main/lines.h \
 	main/macros.h \
-	main/marshal_generated.c \
+	main/marshal_generated0.c \
+	main/marshal_generated1.c \
+	main/marshal_generated2.c \
+	main/marshal_generated3.c \
+	main/marshal_generated4.c \
+	main/marshal_generated5.c \
+	main/marshal_generated6.c \
+	main/marshal_generated7.c \
 	main/marshal_generated.h \
 	main/matrix.c \
 	main/matrix.h \

src/mesa/SConscript

@@ -104,12 +104,13 @@ if env['platform'] not in ('cygwin', 'darwin', 'windows', 'haiku'):
         pass
 
 # The marshal_generated.c file is generated from the GL/ES API.xml file
-env.CodeGenerate(
+for i in range(8):
-    target = 'main/marshal_generated.c',
+    env.CodeGenerate(
-    script = GLAPI + 'gen/gl_marshal.py',
+        target = 'main/marshal_generated{0}.c'.format(i),
-    source = [GLAPI + 'gen/gl_and_es_API.xml'] + env.Glob(GLAPI + 'gen/*.xml'),
+        script = GLAPI + 'gen/gl_marshal.py',
-    command = python_cmd + ' $SCRIPT -f $SOURCE > $TARGET'
+        source = [GLAPI + 'gen/gl_and_es_API.xml'] + env.Glob(GLAPI + 'gen/*.xml'),
-    )
+        command = python_cmd + ' $SCRIPT -f $SOURCE -i {0} -n 8 > $TARGET'.format(i)
+        )
 
 # The marshal_generated.h file is generated from the GL/ES API.xml file
 env.CodeGenerate(

src/mesa/meson.build

@@ -692,13 +692,12 @@ files_libmesa_common += [
   main_enums_c,
   format_fallback_c,
   get_hash_h,
-  main_marshal_generated_c,
   main_marshal_generated_h,
   main_dispatch_h,
   ir_expression_operation_h,
   main_remap_helper_h,
   sha1_h,
-]
+] + main_marshal_generated_c
 files_libmesa_gallium += [
   ir_expression_operation_h,
   sha1_h,