[lldb] Display autosuggestion part in gray if there is one possible suggestion

This is relanding D81001. The patch originally failed as on newer editline
versions it seems CC_REFRESH will move the cursor to the start of the line via
\r and then back to the original position. On older editline versions like
the one used by default on macOS, CC_REFRESH doesn't move the cursor at all.
As the patch changed the way we handle tab completion (previously we did
REDISPLAY but now we're doing CC_REFRESH), this caused a few completion tests
to receive this unexpected cursor movement in the output stream.
This patch updates those tests to also accept output that contains the specific
cursor movement commands (\r and then \x1b[XC). lldbpexpect.py received an
utility method for generating the cursor movement escape sequence.

Original summary:

I implemented autosuggestion if there is one possible suggestion.
I set the keybinds for every character. When a character is typed, Editline::TypedCharacter is called.
Then, autosuggestion part is displayed in gray, and you can actually input by typing C-k.
Editline::Autosuggest is a function for finding completion, and it is like Editline::TabCommand now, but I will add more features to it.

Testing does not work well in my environment, so I can't confirm that it goes well, sorry. I am dealing with it now.

Reviewed By: teemperor, JDevlieghere, #lldb

Differential Revision: https://reviews.llvm.org/D81001

lldb/include/lldb/Core/Debugger.h

@@ -273,6 +273,8 @@ public:
 
   bool SetUseColor(bool use_color);
 
+  bool GetUseAutosuggestion() const;
+
   bool GetUseSourceCache() const;
 
   bool SetUseSourceCache(bool use_source_cache);

lldb/include/lldb/Core/IOHandler.h

@@ -203,6 +203,9 @@ public:
 
   virtual void IOHandlerDeactivated(IOHandler &io_handler) {}
 
+  virtual llvm::Optional<std::string> IOHandlerSuggestion(IOHandler &io_handler,
+                                                          llvm::StringRef line);
+
   virtual void IOHandlerComplete(IOHandler &io_handler,
                                  CompletionRequest &request);
 
@@ -420,6 +423,9 @@ private:
   static int FixIndentationCallback(Editline *editline, const StringList &lines,
                                     int cursor_position, void *baton);
 
+  static llvm::Optional<std::string> SuggestionCallback(llvm::StringRef line,
+                                                        void *baton);
+
   static void AutoCompleteCallback(CompletionRequest &request, void *baton);
 #endif
 

lldb/include/lldb/Host/Editline.h

@@ -98,6 +98,9 @@ typedef int (*FixIndentationCallbackType)(Editline *editline,
                                           const StringList &lines,
                                           int cursor_position, void *baton);
 
+typedef llvm::Optional<std::string> (*SuggestionCallbackType)(
+    llvm::StringRef line, void *baton);
+
 typedef void (*CompleteCallbackType)(CompletionRequest &request, void *baton);
 
 /// Status used to decide when and how to start editing another line in
@@ -184,6 +187,9 @@ public:
   /// Cancel this edit and oblitarate all trace of it
   bool Cancel();
 
+  /// Register a callback for autosuggestion.
+  void SetSuggestionCallback(SuggestionCallbackType callback, void *baton);
+
   /// Register a callback for the tab key
   void SetAutoCompleteCallback(CompleteCallbackType callback, void *baton);
 
@@ -312,6 +318,12 @@ private:
   /// tab key is typed.
   unsigned char TabCommand(int ch);
 
+  /// Apply autosuggestion part in gray as editline.
+  unsigned char ApplyAutosuggestCommand(int ch);
+
+  /// Command used when a character is typed.
+  unsigned char TypedCharacter(int ch);
+
   /// Respond to normal character insertion by fixing line indentation
   unsigned char FixIndentationCommand(int ch);
 
@@ -360,7 +372,9 @@ private:
   const char *m_fix_indentation_callback_chars = nullptr;
   CompleteCallbackType m_completion_callback = nullptr;
   void *m_completion_callback_baton = nullptr;
-
+  SuggestionCallbackType m_suggestion_callback = nullptr;
+  void *m_suggestion_callback_baton = nullptr;
+  std::size_t m_previous_autosuggestion_size = 0;
   std::mutex m_output_mutex;
 };
 }

lldb/include/lldb/Interpreter/CommandInterpreter.h

@@ -351,6 +351,10 @@ public:
 
   CommandObject *GetCommandObjectForCommand(llvm::StringRef &command_line);
 
+  /// Returns the auto-suggestion string that should be added to the given
+  /// command line.
+  llvm::Optional<std::string> GetAutoSuggestionForCommand(llvm::StringRef line);
+
   // This handles command line completion.
   void HandleCompletion(CompletionRequest &request);
 

lldb/packages/Python/lldbsuite/test/lldbpexpect.py

@@ -68,3 +68,10 @@ else:
             self.child.sendeof()
             self.child.close(force=not gracefully)
             self.child = None
+
+        def cursor_forward_escape_seq(self, chars_to_move):
+            """
+            Returns the escape sequence to move the cursor forward/right
+            by a certain amount of characters.
+            """
+            return b"\x1b\[" + str(chars_to_move).encode("utf-8") + b"C"

lldb/source/Core/CoreProperties.td

@@ -131,4 +131,8 @@ let Definition = "debugger" in {
     Global,
     DefaultStringValue<"frame #${frame.index}: ${ansi.fg.yellow}${frame.pc}${ansi.normal}{ ${module.file.basename}{`${function.name-without-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${ansi.fg.cyan}${line.file.basename}${ansi.normal}:${ansi.fg.yellow}${line.number}${ansi.normal}{:${ansi.fg.yellow}${line.column}${ansi.normal}}}{${function.is-optimized} [opt]}{${frame.is-artificial} [artificial]}\\\\n">,
     Desc<"The default frame format string to use when displaying stack frameinformation for threads from thread backtrace unique.">;
+  def ShowAutosuggestion: Property<"show-autosuggestion", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"If true, LLDB will show suggestions to complete the command the user typed.">;
 }

lldb/source/Core/Debugger.cpp

@@ -346,6 +346,12 @@ bool Debugger::SetUseColor(bool b) {
   return ret;
 }
 
+bool Debugger::GetUseAutosuggestion() const {
+  const uint32_t idx = ePropertyShowAutosuggestion;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+}
+
 bool Debugger::GetUseSourceCache() const {
   const uint32_t idx = ePropertyUseSourceCache;
   return m_collection_sp->GetPropertyAtIndexAsBoolean(

lldb/source/Core/IOHandler.cpp

@@ -195,6 +195,14 @@ void IOHandlerConfirm::IOHandlerInputComplete(IOHandler &io_handler,
   }
 }
 
+llvm::Optional<std::string>
+IOHandlerDelegate::IOHandlerSuggestion(IOHandler &io_handler,
+                                       llvm::StringRef line) {
+  return io_handler.GetDebugger()
+      .GetCommandInterpreter()
+      .GetAutoSuggestionForCommand(line);
+}
+
 void IOHandlerDelegate::IOHandlerComplete(IOHandler &io_handler,
                                           CompletionRequest &request) {
   switch (m_completion) {
@@ -258,6 +266,8 @@ IOHandlerEditline::IOHandlerEditline(
                                                m_color_prompts);
     m_editline_up->SetIsInputCompleteCallback(IsInputCompleteCallback, this);
     m_editline_up->SetAutoCompleteCallback(AutoCompleteCallback, this);
+    if (debugger.GetUseAutosuggestion() && debugger.GetUseColor())
+      m_editline_up->SetSuggestionCallback(SuggestionCallback, this);
     // See if the delegate supports fixing indentation
     const char *indent_chars = delegate.IOHandlerGetFixIndentationCharacters();
     if (indent_chars) {
@@ -430,6 +440,16 @@ int IOHandlerEditline::FixIndentationCallback(Editline *editline,
       *editline_reader, lines, cursor_position);
 }
 
+llvm::Optional<std::string>
+IOHandlerEditline::SuggestionCallback(llvm::StringRef line, void *baton) {
+  IOHandlerEditline *editline_reader = static_cast<IOHandlerEditline *>(baton);
+  if (editline_reader)
+    return editline_reader->m_delegate.IOHandlerSuggestion(*editline_reader,
+                                                           line);
+
+  return llvm::None;
+}
+
 void IOHandlerEditline::AutoCompleteCallback(CompletionRequest &request,
                                              void *baton) {
   IOHandlerEditline *editline_reader = (IOHandlerEditline *)baton;

lldb/source/Host/common/Editline.cpp

@@ -1009,7 +1009,10 @@ unsigned char Editline::TabCommand(int ch) {
         to_add.push_back(request.GetParsedArg().GetQuoteChar());
       to_add.push_back(' ');
       el_insertstr(m_editline, to_add.c_str());
-      break;
+      // Clear all the autosuggestion parts if the only single space can be completed.
+      if (to_add == " ")
+        return CC_REDISPLAY;
+      return CC_REFRESH;
     }
     case CompletionMode::Partial: {
       std::string to_add = completion.GetCompletion();
@@ -1043,6 +1046,52 @@ unsigned char Editline::TabCommand(int ch) {
   return CC_REDISPLAY;
 }
 
+unsigned char Editline::ApplyAutosuggestCommand(int ch) {
+  const LineInfo *line_info = el_line(m_editline);
+  llvm::StringRef line(line_info->buffer,
+                       line_info->lastchar - line_info->buffer);
+
+  if (llvm::Optional<std::string> to_add =
+          m_suggestion_callback(line, m_suggestion_callback_baton))
+    el_insertstr(m_editline, to_add->c_str());
+
+  return CC_REDISPLAY;
+}
+
+unsigned char Editline::TypedCharacter(int ch) {
+  std::string typed = std::string(1, ch);
+  el_insertstr(m_editline, typed.c_str());
+  const LineInfo *line_info = el_line(m_editline);
+  llvm::StringRef line(line_info->buffer,
+                       line_info->lastchar - line_info->buffer);
+
+  if (llvm::Optional<std::string> to_add =
+          m_suggestion_callback(line, m_suggestion_callback_baton)) {
+    std::string to_add_color = ANSI_FAINT + to_add.getValue() + ANSI_UNFAINT;
+    fputs(typed.c_str(), m_output_file);
+    fputs(to_add_color.c_str(), m_output_file);
+    size_t new_autosuggestion_size = line.size() + to_add->length();
+    // Print spaces to hide any remains of a previous longer autosuggestion.
+    if (new_autosuggestion_size < m_previous_autosuggestion_size) {
+      size_t spaces_to_print =
+          m_previous_autosuggestion_size - new_autosuggestion_size;
+      std::string spaces = std::string(spaces_to_print, ' ');
+      fputs(spaces.c_str(), m_output_file);
+    }
+    m_previous_autosuggestion_size = new_autosuggestion_size;
+
+    int editline_cursor_position =
+        (int)((line_info->cursor - line_info->buffer) + GetPromptWidth());
+    int editline_cursor_row = editline_cursor_position / m_terminal_width;
+    int toColumn =
+        editline_cursor_position - (editline_cursor_row * m_terminal_width);
+    fprintf(m_output_file, ANSI_SET_COLUMN_N, toColumn);
+    return CC_REFRESH;
+  }
+
+  return CC_REDISPLAY;
+}
+
 void Editline::ConfigureEditor(bool multiline) {
   if (m_editline && m_multiline_enabled == multiline)
     return;
@@ -1156,7 +1205,38 @@ void Editline::ConfigureEditor(bool multiline) {
   if (!multiline) {
     el_set(m_editline, EL_BIND, "^r", "em-inc-search-prev",
            NULL); // Cycle through backwards search, entering string
+
+    if (m_suggestion_callback) {
+      el_wset(m_editline, EL_ADDFN, EditLineConstString("lldb-apply-complete"),
+              EditLineConstString("Adopt autocompletion"),
+              (EditlineCommandCallbackType)([](EditLine *editline, int ch) {
+                return Editline::InstanceFor(editline)->ApplyAutosuggestCommand(
+                    ch);
+              }));
+
+      el_set(m_editline, EL_BIND, "^f", "lldb-apply-complete",
+             NULL); // Apply a part that is suggested automatically
+
+      el_wset(m_editline, EL_ADDFN, EditLineConstString("lldb-typed-character"),
+              EditLineConstString("Typed character"),
+              (EditlineCommandCallbackType)([](EditLine *editline, int ch) {
+                return Editline::InstanceFor(editline)->TypedCharacter(ch);
+              }));
+
+      char bind_key[2] = {0, 0};
+      llvm::StringRef ascii_chars =
+          "abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXZY1234567890!\"#$%"
+          "&'()*+,./:;<=>?@[]_`{|}~ ";
+      for (char c : ascii_chars) {
+        bind_key[0] = c;
+        el_set(m_editline, EL_BIND, bind_key, "lldb-typed-character", NULL);
+      }
+      el_set(m_editline, EL_BIND, "\\-", "lldb-typed-character", NULL);
+      el_set(m_editline, EL_BIND, "\\^", "lldb-typed-character", NULL);
+      el_set(m_editline, EL_BIND, "\\\\", "lldb-typed-character", NULL);
+    }
   }
+
   el_set(m_editline, EL_BIND, "^w", "ed-delete-prev-word",
          NULL); // Delete previous word, behave like bash in emacs mode
   el_set(m_editline, EL_BIND, "\t", "lldb-complete",
@@ -1367,6 +1447,12 @@ bool Editline::Cancel() {
   return result;
 }
 
+void Editline::SetSuggestionCallback(SuggestionCallbackType callback,
+                                     void *baton) {
+  m_suggestion_callback = callback;
+  m_suggestion_callback_baton = baton;
+}
+
 void Editline::SetAutoCompleteCallback(CompleteCallbackType callback,
                                        void *baton) {
   m_completion_callback = callback;

lldb/source/Interpreter/CommandInterpreter.cpp

@@ -1878,6 +1878,19 @@ void CommandInterpreter::HandleCompletion(CompletionRequest &request) {
   HandleCompletionMatches(request);
 }
 
+llvm::Optional<std::string>
+CommandInterpreter::GetAutoSuggestionForCommand(llvm::StringRef line) {
+  if (line.empty())
+    return llvm::None;
+  const size_t s = m_command_history.GetSize();
+  for (int i = s - 1; i >= 0; --i) {
+    llvm::StringRef entry = m_command_history.GetStringAtIndex(i);
+    if (entry.consume_front(line))
+      return entry.str();
+  }
+  return llvm::None;
+}
+
 CommandInterpreter::~CommandInterpreter() {}
 
 void CommandInterpreter::UpdatePrompt(llvm::StringRef new_prompt) {

lldb/test/API/commands/expression/multiline-completion/TestMultilineCompletion.py

@@ -40,7 +40,9 @@ class MultilineCompletionTest(PExpectTest):
 
         self.start_expression_editor()
         self.child.send("to_\t")
-        self.child.expect_exact("to_complete")
+        # editline might move the cursor back to the start of the line via \r
+        # and then back to its original position.
+        self.child.expect(re.compile(b"to_(\r" + self.cursor_forward_escape_seq(len("  1: to_")) + b")?complete"))
         self.exit_expression_editor()
 
         # Check that completion empty input in a function with only one

lldb/test/API/iohandler/autosuggestion/TestAutosuggestion.py

@@ -0,0 +1,105 @@
+"""
+Tests autosuggestion using pexpect.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.lldbpexpect import PExpectTest
+
+def cursor_horizontal_abs(s):
+    return "\x1b[" + str(len(s) + 1) + "G"
+
+
+
+class TestCase(PExpectTest):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    # PExpect uses many timeouts internally and doesn't play well
+    # under ASAN on a loaded machine..
+    @skipIfAsan
+    @skipIfEditlineSupportMissing
+    def test_autosuggestion_add_spaces(self):
+        self.launch(extra_args=["-o", "settings set show-autosuggestion true", "-o", "settings set use-color true"])
+
+        # Common input codes and escape sequences.
+        faint_color = "\x1b[2m"
+        reset = "\x1b[0m"
+
+        # Check if spaces are added to hide the previous gray characters.
+        self.expect("help frame var")
+        self.expect("help frame info")
+        self.child.send("help frame v")
+        self.child.expect_exact(cursor_horizontal_abs("(lldb) help frame ") + "v" + faint_color + "ar" + reset + " ")
+
+        self.quit()
+
+    @skipIfAsan
+    @skipIfEditlineSupportMissing
+    def test_autosuggestion(self):
+        self.launch(extra_args=["-o", "settings set show-autosuggestion true", "-o", "settings set use-color true"])
+
+        # Common input codes and escape sequences.
+        ctrl_f = "\x06"
+        faint_color = "\x1b[2m"
+        reset = "\x1b[0m"
+        delete = chr(127)
+
+        frame_output_needle = "Syntax: frame <subcommand>"
+        # Run 'help frame' once to put it into the command history.
+        self.expect("help frame", substrs=[frame_output_needle])
+
+        # Check that LLDB shows the autosuggestion in gray behind the text.
+        self.child.send("hel")
+        self.child.expect_exact(cursor_horizontal_abs("(lldb) he") + "l" + faint_color + "p frame" + reset)
+
+        # Apply the autosuggestion and press enter. This should print the
+        # 'help frame' output if everything went correctly.
+        self.child.send(ctrl_f + "\n")
+        self.child.expect_exact(frame_output_needle)
+
+        # Check that pressing Ctrl+F directly after Ctrl+F again does nothing.
+        self.child.send("hel" + ctrl_f + ctrl_f + "\n")
+        self.child.expect_exact(frame_output_needle)
+
+        # Try autosuggestion using tab and ^f.
+        # \t makes "help" and ^f makes "help frame". If everything went
+        # correct we should see the 'help frame' output again.
+        self.child.send("hel\t" + ctrl_f + "\n")
+        self.child.expect_exact(frame_output_needle)
+
+        # Check that autosuggestion works after delete.
+        self.child.send("a1234" + 5 * delete + "hel" + ctrl_f + "\n")
+        self.child.expect_exact(frame_output_needle)
+
+        # Check that autosuggestion works after delete.
+        self.child.send("help x" + delete + ctrl_f + "\n")
+        self.child.expect_exact(frame_output_needle)
+
+        # Check that autosuggestion complete to the most recent one.
+        self.child.send("help frame variable\n")
+        self.child.send("help fr")
+        self.child.expect_exact(faint_color + "ame variable" + reset)
+        self.child.send("\n")
+
+        # Try another command.
+        apropos_output_needle = "Syntax: apropos <search-word>"
+        # Run 'help frame' once to put it into the command history.
+        self.expect("help apropos", substrs=[apropos_output_needle])
+
+        # Check that 'hel' should have an autosuggestion for 'help apropos' now.
+        self.child.send("hel")
+        self.child.expect_exact(cursor_horizontal_abs("(lldb) he") + "l" + faint_color + "p apropos" + reset)
+
+        # Run the command and expect the 'help apropos' output.
+        self.child.send(ctrl_f + "\n")
+        self.child.expect_exact(apropos_output_needle)
+
+        # Check that pressing Ctrl+F in an empty prompt does nothing.
+        breakpoint_output_needle = "Syntax: breakpoint <subcommand>"
+        self.child.send(ctrl_f + "help breakpoint" +"\n")
+        self.child.expect_exact(breakpoint_output_needle)
+
+
+        self.quit()

lldb/test/API/iohandler/completion/TestIOHandlerCompletion.py

@@ -26,7 +26,9 @@ class IOHandlerCompletionTest(PExpectTest):
 
         # Try tab completing regi to register.
         self.child.send("regi\t")
-        self.child.expect_exact(self.PROMPT + "register")
+        # editline might move the cursor back to the start of the line and
+        # then back to its original position.
+        self.child.expect(re.compile(b"regi(\r" + self.cursor_forward_escape_seq(len(self.PROMPT + "regi")) + b")?ster"))
         self.child.send("\n")
         self.expect_prompt()
 
@@ -39,7 +41,11 @@ class IOHandlerCompletionTest(PExpectTest):
         # If we get a correct partial completion without a trailing space, then this
         # should complete the current test file.
         self.child.send("TestIOHandler\t")
-        self.child.expect_exact("TestIOHandlerCompletion.py")
+        # As above, editline might move the cursor to the start of the line and
+        # then back to its original position. We only care about the fact
+        # that this is completing a partial completion, so skip the exact cursor
+        # position calculation.
+        self.child.expect(re.compile(b"TestIOHandler(\r" + self.cursor_forward_escape_seq("\d+") + b")?Completion.py"))
         self.child.send("\n")
         self.expect_prompt()