Fix a WPF<>TSF crash by avoiding TF_TMAE_CONSOLE (#19584)

As explained in detail in the diff.

Closes #19562

Author: lhecker

.github/actions/spelling/expect/expect.txt

@@ -1091,6 +1091,8 @@ NEXTLINE
 nfe
 NLSMODE
 NOACTIVATE
+NOACTIVATEKEYBOARDLAYOUT
+NOACTIVATETIP
 NOAPPLYNOW
 NOCLIP
 NOCOMM
@@ -1504,7 +1506,6 @@ scrolllock
 scrolloffset
 SCROLLSCALE
 SCROLLSCREENBUFFER
-scursor
 sddl
 SDKDDK
 segfault
@@ -1703,9 +1704,11 @@ TEXTMETRIC
 TEXTMETRICW
 textmode
 texttests
+TFCAT
 THUMBPOSITION
 THUMBTRACK
 tilunittests
+TIPCAP
 titlebars
 TITLEISLINKNAME
 TLDP
@@ -1757,6 +1760,8 @@ UIACCESS
 uiacore
 uiautomationcore
 uielem
+UIELEMENTENABLED
+UIELEMENTENABLEDONLY
 UINTs
 uld
 uldash

src/cascadia/TerminalControl/HwndTerminal.cpp

@@ -462,6 +462,11 @@ void HwndTerminal::SendOutput(std::wstring_view data)
     _terminal->Write(data);
 }
 
+void _stdcall AvoidBuggyTSFConsoleFlags()
+{
+    Microsoft::Console::TSF::Handle::AvoidBuggyTSFConsoleFlags();
+}
+
 HRESULT _stdcall CreateTerminal(HWND parentHwnd, _Out_ void** hwnd, _Out_ void** terminal)
 {
     auto publicTerminal = std::make_unique<HwndTerminal>(parentHwnd);

src/cascadia/TerminalControl/HwndTerminal.hpp

@@ -41,6 +41,7 @@ typedef struct _TerminalTheme
 } TerminalTheme, *LPTerminalTheme;
 
 extern "C" {
+__declspec(dllexport) void _stdcall AvoidBuggyTSFConsoleFlags();
 __declspec(dllexport) HRESULT _stdcall CreateTerminal(HWND parentHwnd, _Out_ void** hwnd, _Out_ void** terminal);
 __declspec(dllexport) void _stdcall TerminalSendOutput(void* terminal, LPCWSTR data);
 __declspec(dllexport) void _stdcall TerminalRegisterScrollCallback(void* terminal, void __stdcall callback(int, int, int));

src/cascadia/TerminalControl/dll/Microsoft.Terminal.Control.def

@@ -4,6 +4,7 @@ EXPORTS
   DllGetActivationFactory = WINRT_GetActivationFactory    PRIVATE
 
   ; Flat C ABI
+  AvoidBuggyTSFConsoleFlags
   CreateTerminal
   DestroyTerminal
   TerminalCalculateResize

src/cascadia/WpfTerminalControl/NativeMethods.cs

@@ -171,6 +171,9 @@ public enum SetWindowPosFlags : uint
             SWP_SHOWWINDOW = 0x0040,
         }
 
+        [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall, ExactSpelling = true)]
+        public static extern void AvoidBuggyTSFConsoleFlags();
+
         [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall, PreserveSig = false)]
         public static extern void CreateTerminal(IntPtr parent, out IntPtr hwnd, out IntPtr terminal);
 

src/cascadia/WpfTerminalControl/TerminalContainer.cs

@@ -11,7 +11,6 @@ namespace Microsoft.Terminal.Wpf
     using System.Windows.Automation.Peers;
     using System.Windows.Interop;
     using System.Windows.Media;
-    using System.Windows.Threading;
 
     /// <summary>
     /// The container class that hosts the native hwnd terminal.
@@ -32,6 +31,11 @@ public class TerminalContainer : HwndHost
         /// </summary>
         public TerminalContainer()
         {
+            // WPF & TSF can't deal with us setting TF_TMAE_CONSOLE on the UI thread.
+            // It simply crashes on Windows 10 if you use the Emoji picker.
+            // (On later versions of Windows it just doesn't work.)
+            NativeMethods.AvoidBuggyTSFConsoleFlags();
+
             this.MessageHook += this.TerminalContainer_MessageHook;
             this.GotFocus += this.TerminalContainer_GotFocus;
             this.Focusable = true;

src/tsf/Handle.cpp

@@ -16,6 +16,11 @@ Handle Handle::Create()
     return handle;
 }
 
+void Handle::AvoidBuggyTSFConsoleFlags()
+{
+    Implementation::AvoidBuggyTSFConsoleFlags();
+}
+
 void Handle::SetDefaultScopeAlphanumericHalfWidth(bool enable)
 {
     Implementation::SetDefaultScopeAlphanumericHalfWidth(enable);

src/tsf/Handle.h

@@ -33,6 +33,7 @@ namespace Microsoft::Console::TSF
     struct Handle
     {
         static Handle Create();
+        static void AvoidBuggyTSFConsoleFlags();
         static void SetDefaultScopeAlphanumericHalfWidth(bool enable);
 
         Handle() = default;

src/tsf/Implementation.cpp

@@ -27,9 +27,45 @@ static void TfPropertyvalClose(TF_PROPERTYVAL* val)
 }
 using unique_tf_propertyval = wil::unique_struct<TF_PROPERTYVAL, decltype(&TfPropertyvalClose), &TfPropertyvalClose>;
 
+// The flags passed to ActivateEx don't replace the flags during previous calls.
+// Instead, they're additive. So, if we pass flags that some concurrently running
+// TSF clients don't expect we may blow them up accidentally.
+//
+// Such is the case with WPF (and TSF, which is actually at fault there).
+// If you pass TF_TMAE_CONSOLE it'll instantly crash on Windows 10 on first text input.
+// On Windows 11 it'll at least not crash but still make emoji input completely non-functional.
+//
+// --------
+//
+// In any case, we pass the same flags as conhost v1:
+// - TF_TMAE_UIELEMENTENABLEDONLY: TSF activates only text services that are
+//   categorized in GUID_TFCAT_TIPCAP_UIELEMENTENABLED.
+// - TF_TMAE_NOACTIVATEKEYBOARDLAYOUT: TSF does not sync the current keyboard layout
+//   while this method is called. The keyboard layout will be adjusted when the
+//   calling thread gets focus. This flag must be used with TF_TMAE_NOACTIVATETIP.
+// - TF_TMAE_CONSOLE: A text service is activated for console usage.
+//   Some IMEs are known to use this as a hint. Particularly a Korean IME can benefit
+//   from this, because Korean relies on "recomposing" previously finished compositions.
+//   That can't work in a terminal, since we submit composed text to the shell immediately.
+//
+// I'm not sure what TF_TMAE_UIELEMENTENABLEDONLY does. I tried to figure it out but failed.
+//
+// For TF_TMAE_NOACTIVATEKEYBOARDLAYOUT, I'm 99% sure it doesn't do anything, including in
+// conhost v1. This is because IMM will be initialized on WM_ACTIVATE, which calls ActivateEx(0).
+// Any subsequent ActivateEx() calls will update the flags, _except_ for this one and
+// TF_TMAE_NOACTIVATETIP which are explicitly filtered out.
+//
+// TF_TMAE_NOACTIVATETIP however is important. Without it, TIPs are immediately initialized.
+static std::atomic<DWORD> s_activationFlags{ TF_TMAE_NOACTIVATETIP | TF_TMAE_UIELEMENTENABLEDONLY | TF_TMAE_NOACTIVATEKEYBOARDLAYOUT | TF_TMAE_CONSOLE };
+void Implementation::AvoidBuggyTSFConsoleFlags() noexcept
+{
+    s_activationFlags.fetch_and(~static_cast<DWORD>(TF_TMAE_CONSOLE), std::memory_order_relaxed);
+}
+
+static std::atomic<bool> s_wantsAnsiInputScope{ false };
 void Implementation::SetDefaultScopeAlphanumericHalfWidth(bool enable) noexcept
 {
-    _wantsAnsiInputScope.store(enable, std::memory_order_relaxed);
+    s_wantsAnsiInputScope.store(enable, std::memory_order_relaxed);
 }
 
 void Implementation::Initialize()
@@ -40,7 +76,7 @@ void Implementation::Initialize()
     // There's no point in calling TF_GetThreadMgr. ITfThreadMgr is a per-thread singleton.
     _threadMgrEx = wil::CoCreateInstance<ITfThreadMgrEx>(CLSID_TF_ThreadMgr, CLSCTX_INPROC_SERVER);
 
-    THROW_IF_FAILED(_threadMgrEx->ActivateEx(&_clientId, TF_TMAE_CONSOLE));
+    THROW_IF_FAILED(_threadMgrEx->ActivateEx(&_clientId, s_activationFlags.load(std::memory_order_relaxed)));
     THROW_IF_FAILED(_threadMgrEx->CreateDocumentMgr(_documentMgr.addressof()));
 
     TfEditCookie ecTextStore;
@@ -319,7 +355,7 @@ STDMETHODIMP Implementation::GetWnd(HWND* phwnd) noexcept
 
 STDMETHODIMP Implementation::GetAttribute(REFGUID rguidAttribute, VARIANT* pvarValue) noexcept
 {
-    if (_wantsAnsiInputScope.load(std::memory_order_relaxed) && IsEqualGUID(rguidAttribute, GUID_PROP_INPUTSCOPE))
+    if (s_wantsAnsiInputScope.load(std::memory_order_relaxed) && IsEqualGUID(rguidAttribute, GUID_PROP_INPUTSCOPE))
     {
         _ansiInputScope.AddRef();
         pvarValue->vt = VT_UNKNOWN;

src/tsf/Implementation.h

@@ -16,6 +16,7 @@ namespace Microsoft::Console::TSF
 
     struct Implementation : ITfContextOwner, ITfContextOwnerCompositionSink, ITfTextEditSink
     {
+        static void AvoidBuggyTSFConsoleFlags() noexcept;
         static void SetDefaultScopeAlphanumericHalfWidth(bool enable) noexcept;
 
         virtual ~Implementation() = default;
@@ -129,6 +130,5 @@ namespace Microsoft::Console::TSF
         int _compositions = 0;
 
         AnsiInputScope _ansiInputScope{ this };
-        inline static std::atomic<bool> _wantsAnsiInputScope{ false };
     };
 }