Add test-JL_stdlibs target for precompiling stdlibs with JuliaLowering

Makefile

@@ -123,6 +123,10 @@ julia-cli-release julia-cli-debug: julia-cli-% : julia-deps
 julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$*
 
+.PHONY: julia-sysimg-JL-release julia-sysimg-JL-debug
+julia-sysimg-JL-release julia-sysimg-JL-debug : julia-sysimg-JL-% : julia-sysimg-% julia-stdlib | $(build_private_libdir)
+	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-JL-$*
+
 # Useful for cross-bootstrapping
 .PHONY: julia-sysbase-release julia-sysbase-debug
 julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)

base/Base.jl

@@ -319,6 +319,9 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 Core.println("JuliaSyntax/src/JuliaSyntax.jl")
 include(@__MODULE__, string(DATAROOT, "julia/JuliaSyntax/src/JuliaSyntax.jl"))
 
+# May be replaced in incremental sysimage build after-the-fact
+const JuliaLowering = nothing
+
 end_base_include = time_ns()
 
 const _sysimage_modules = PkgId[]
@@ -397,6 +400,11 @@ function __init__()
     if get_bool_env("JULIA_USE_FLISP_PARSER", false) === false
         JuliaSyntax.enable_in_core!()
     end
+    if JuliaLowering !== nothing && get_bool_env("JULIA_USE_FLISP_LOWERING", true) === false
+        # This is not available by default, but JuliaLowering can be added to
+        # Base after-the-fact via an incremental sysimage build.
+        JuliaLowering.activate!()
+    end
 
     CoreLogging.global_logger(CoreLogging.ConsoleLogger())
     nothing

sysimage.mk

@@ -11,6 +11,8 @@ sysimg-ji: $(build_private_libdir)/sysbase.ji
 sysimg-bc: $(build_private_libdir)/sys-bc.a
 sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT)
 sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
+sysimg-JL-release: $(build_private_libdir)/sys-JL.$(SHLIB_EXT)
+sysimg-JL-debug: $(build_private_libdir)/sys-JL-debug.$(SHLIB_EXT)
 sysbase-release: $(build_private_libdir)/sysbase.$(SHLIB_EXT)
 sysbase-debug: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT)
 
@@ -147,7 +149,31 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil
 .SECONDARY: $$(build_private_libdir)/sys$1-o.a $(build_private_libdir)/sys$1-bc.a # request Make to keep these files around
 .SECONDARY: $$(build_private_libdir)/sysbase$1-o.a $(build_private_libdir)/sysbase$1-bc.a # request Make to keep these files around
 endef
+
+JULIALOWERING_SRCS := $(shell find $(build_datarootdir)/julia/JuliaLowering/src -name \*.jl)
+
+define JL_sysimg_builder
+$$(build_private_libdir)/sys-JL$1-o.a $$(build_private_libdir)/sys-JL$1-bc.a : $$(build_private_libdir)/sys-JL$1-%.a : $$(build_private_libdir)/sys$1.$$(SHLIB_EXT) $$(JULIALOWERING_SRCS)
+	@$$(call PRINT_JULIA, cd $$(JULIAHOME)/JuliaLowering/src/ && \
+	if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \
+		 WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
+		 JULIA_LOAD_PATH='@stdlib' \
+		 JULIA_PROJECT= \
+		 JULIA_DEPOT_PATH=':' \
+		 JULIA_NUM_THREADS=1 \
+			$$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
+			$(bootstrap_julia_flags) \
+			--startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) -e "Core.include(Base, \"$$(call cygpath_w,$$(BUILDROOT)/JuliaLowering/src/JuliaLowering.jl)\")"; then \
+		echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
+		false; \
+	fi )
+	@mv [email protected] $$@
+.SECONDARY: $$(build_private_libdir)/sys-JL$1-o.a $(build_private_libdir)/sys-JL$1-bc.a # request Make to keep these files around
+endef
+
 $(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release)))
 $(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
 $(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
 $(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
+$(eval $(call JL_sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
+$(eval $(call JL_sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))

test/JuliaLowering_stdlibs.jl

@@ -0,0 +1,38 @@
+import Libdl
+
+### 34 / 51 (non-sysimage) stdlibs precompile successfully with JuliaLowering ###
+const INCOMPATIBLE_STDLIBS = String[
+    "Statistics", "JuliaSyntaxHighlighting", "Markdown", "Downloads", "LibGit2",
+    "InteractiveUtils", "Test", "REPL", "Pkg", "LazyArtifacts", "SparseArrays",
+    "Dates", "TOML", "StyledStrings", "Profile",
+]
+
+# ensure JL-inclusive sysimage is built / available
+run(`$(ENV["MAKE"]) -C $(ENV["BUILDROOT"]) -f sysimage.mk sysimg-JL-$(ENV["JULIA_BUILD_MODE"])`)
+
+stdlibs_to_test = filter(name -> !in(name, INCOMPATIBLE_STDLIBS), String[
+    strip(split(line, "=")[1])
+    for line in readlines("$(ENV["JULIAHOME"])/stdlib/Project.toml")[2:end]
+])
+configs = [
+    ``=>Base.CacheFlags(check_bounds=0, debug_level=2, opt_level=3),
+    ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3),
+]
+compilecache_command = "using Base: CacheFlags; Base.Precompilation.precompilepkgs($(stdlibs_to_test); configs=$(configs))"
+
+debug = ENV["JULIA_BUILD_MODE"] == "debug" ? "-debug" : ""
+sysimage = joinpath(dirname(unsafe_string(Base.JLOptions().image_file)), "sys-JL$(debug).$(Libdl.dlext)")
+
+# pre-compile stdlibs (into temporary depot)
+mktempdir() do tmp_depot
+    cmd = addenv(
+        `$(ENV["JULIA_EXECUTABLE"]) --sysimage $(sysimage) --startup-file=no -e $compilecache_command`,
+        "JULIA_LOAD_PATH" => "@stdlib$(Base.Linking.pathsep)$(joinpath(ENV["JULIAHOME"], "stdlib"))",
+        "JULIA_CPU_TARGET" => "sysimage",
+        "JULIA_USE_FLISP_LOWERING" => "0",
+        "JULIA_USE_FALLBACK_REPL" => "0",
+        "JULIA_DEPOT_PATH" => tmp_depot,
+        ; inherit = true
+    )
+    success(run(cmd))
+end

test/Makefile

@@ -11,6 +11,13 @@ export JULIA_LOAD_PATH := @$(PATHSEP)@stdlib
 unexport JULIA_PROJECT :=
 unexport JULIA_BINDIR :=
 
+# used by JuliaLowering_stdlibs.jl test (invokes `make` for JL sysimage)
+export MAKE
+export BUILDROOT
+export JULIAHOME
+export JULIA_EXECUTABLE
+export JULIA_BUILD_MODE
+
 TESTGROUPS = unicode strings compiler Compiler JuliaSyntax JuliaLowering
 TESTS = all default stdlib $(TESTGROUPS) \
 		$(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \
@@ -33,7 +40,7 @@ default:
 
 .PHONY: $(TESTS)
 $(TESTS):
-	@cd $(SRCDIR) && \
+	+@cd $(SRCDIR) && \
 	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@)
 
 .PHONY: install-revise-deps

test/choosetests.jl

@@ -31,7 +31,7 @@ const TESTNAMES = [
         "smallarrayshrink", "opaque_closure", "filesystem", "download",
         "scopedvalues", "compileall", "rebinding",
         "faulty_constructor_method_should_not_cause_stack_overflows",
-        "JuliaSyntax", "JuliaLowering",
+        "JuliaSyntax", "JuliaLowering", "JuliaLowering_stdlibs",
 ]
 
 const INTERNET_REQUIRED_LIST = [