PseudoConsole.h

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#ifndef LIBLLDB_HOST_WINDOWS_PSEUDOCONSOLE_H_
#define LIBLLDB_HOST_WINDOWS_PSEUDOCONSOLE_H_
 
#include "llvm/Support/Error.h"
#include <atomic>
#include <mutex>
#include <string>
 
#define PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE 0x20016
typedef void *HANDLE;
typedef void *HPCON;
 
namespace lldb_private {
 
class PseudoConsole {
 
public:
  enum class Mode { ConPTY, Pipe, None };
 
  PseudoConsole() = default;
  ~PseudoConsole();
 
  PseudoConsole(const PseudoConsole &) = delete;
  PseudoConsole(PseudoConsole &&) = delete;
  PseudoConsole &operator=(const PseudoConsole &) = delete;
  PseudoConsole &operator=(PseudoConsole &&) = delete;
 
  /// Creates a named pipe pair for overlapped I/O.
  /// On failure any handles that were successfully opened are closed and an
  /// error is returned.
  llvm::Error CreateOverlappedPipePair(HANDLE &out_read, HANDLE &out_write,
                                       bool inheritable);
 
  /// Creates and opens a new ConPTY instance with a default console size of
  /// 80x25. Also sets up the associated STDIN/STDOUT pipes and responds to
  /// the cursor-position query that ConPTY emits at startup.
  ///
  /// \param req_cols, req_rows Optional terminal dimensions.
  ///
  /// \return
  ///     An llvm::Error if the ConPTY could not be created, or if ConPTY is
  ///     not available on this version of Windows, llvm::Error::success()
  ///     otherwise.
  llvm::Error OpenPseudoConsole(uint16_t req_cols = 0, uint16_t req_rows = 0);
 
  /// Creates a pair of anonymous pipes to use for stdio instead of a ConPTY.
  ///
  /// \return
  ///     An llvm::Error if the pipes could not be created.
  llvm::Error OpenAnonymousPipes();
 
  /// Closes the ConPTY and invalidates its handle, without closing the STDIN
  /// and STDOUT pipes. Closing the ConPTY signals EOF to any process currently
  /// attached to it. The console host then writes its last frame to the STDOUT
  /// pipe and closes its end, so reading the pipe up to EOF gets all of the
  /// output.
  ///
  /// In pipe mode there is no ConPTY to close: this cancels the read pending
  /// on the STDOUT pipe and marks the pipes closed, so IsConnected() returns
  /// false from then on, as it does once a ConPTY is closed.
  void Close();
 
  /// Closes the STDIN and STDOUT pipe handles and invalidates them.
  void ClosePseudoConsolePipes();
 
  /// Closes the child-side pipe handles (stdin read end and stdout/stderr write
  /// end) that were passed to CreateProcessW. Must be called after a successful
  /// CreateProcessW to avoid keeping the pipes alive indefinitely.
  void CloseAnonymousPipes();
 
  /// Closes any open ConPTY/pipe handles and resets internal state to a
  /// freshly-constructed PseudoConsole.
  void Reset();
 
  /// Returns whether the ConPTY and its pipes are currently open and valid.
  bool IsConnected() const;
 
  /// The ConPTY HPCON handle accessor.
  ///
  /// This object retains ownership of the HPCON when this accessor is used.
  ///
  /// \return
  ///     The ConPTY HPCON handle, or INVALID_HANDLE_VALUE if it is currently
  ///     invalid.
  HPCON GetPseudoTerminalHandle() { return m_conpty_handle; };
 
  /// The STDOUT read HANDLE accessor.
  ///
  /// This object retains ownership of the HANDLE when this accessor is used.
  ///
  /// \return
  ///     The STDOUT read HANDLE, or INVALID_HANDLE_VALUE if it is currently
  ///     invalid.
  HANDLE GetSTDOUTHandle() const { return m_conpty_output; };
 
  /// The STDIN write HANDLE accessor.
  ///
  /// This object retains ownership of the HANDLE when this accessor is used.
  ///
  /// \return
  ///     The STDIN write HANDLE, or INVALID_HANDLE_VALUE if it is currently
  ///     invalid.
  HANDLE GetSTDINHandle() const { return m_conpty_input; };
 
  /// The child-side stdin read HANDLE (pipe mode only).
  HANDLE GetChildStdinHandle() const { return m_pipe_child_stdin; };
 
  /// The child-side stdout/stderr write HANDLE (pipe mode only).
  HANDLE GetChildStdoutHandle() const { return m_pipe_child_stdout; };
 
  Mode GetMode() const { return m_mode; };
 
protected:
  HANDLE m_conpty_handle = reinterpret_cast<HANDLE>(static_cast<intptr_t>(-1));
  HANDLE m_conpty_output = reinterpret_cast<HANDLE>(static_cast<intptr_t>(-1));
  HANDLE m_conpty_input = reinterpret_cast<HANDLE>(static_cast<intptr_t>(-1));
  // Pipe mode: child-side handles passed to CreateProcessW, closed after launch
  HANDLE m_pipe_child_stdin =
      reinterpret_cast<HANDLE>(static_cast<intptr_t>(-1));
  HANDLE m_pipe_child_stdout =
      reinterpret_cast<HANDLE>(static_cast<intptr_t>(-1));
  // Pipe mode: set by Close(). Its CancelIoEx only reaches a read that is
  // already pending, so a reader has to be able to tell that the pipes were
  // closed before it starts the next one.
  std::atomic<bool> m_pipes_closed = false;
  Mode m_mode = Mode::None;
  std::mutex m_mutex{};
};
} // namespace lldb_private
 
#endif // LIBLLDB_HOST_WINDOWS_PSEUDOCONSOLE_H_