LLDB mainline
NativeRegisterContextLinux_arm64.cpp
Go to the documentation of this file.
1//===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#if defined(__arm64__) || defined(__aarch64__)
10
14
15#include "lldb/Host/HostInfo.h"
19#include "lldb/Utility/Log.h"
21#include "lldb/Utility/Status.h"
22
29
30// System includes - They have to be included after framework includes because
31// they define some macros which collide with variable names in other modules
32#include <sys/uio.h>
33// NT_PRSTATUS and NT_FPREGSET definition
34#include <elf.h>
35#include <mutex>
36#include <optional>
37
38#ifndef NT_ARM_SVE
39#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
40#endif
41
42#ifndef NT_ARM_SSVE
43#define NT_ARM_SSVE \
44 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
45#endif
46
47#ifndef NT_ARM_ZA
48#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
49#endif
50
51#ifndef NT_ARM_ZT
52#define NT_ARM_ZT \
53 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
54#endif
55
56#ifndef NT_ARM_PAC_MASK
57#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
58#endif
59
60#ifndef NT_ARM_TAGGED_ADDR_CTRL
61#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
62#endif
63
64#ifndef NT_ARM_FPMR
65#define NT_ARM_FPMR 0x40e /* Floating point mode register */
66#endif
67
68#ifndef NT_ARM_GCS
69#define NT_ARM_GCS 0x410 /* Guarded Control Stack control registers */
70#endif
71
72#define HWCAP_PACA (1 << 30)
73
74#define HWCAP_GCS (1UL << 32)
75
76#define HWCAP2_MTE (1 << 18)
77
78#define HWCAP2_FPMR (1UL << 48)
79
80using namespace lldb;
81using namespace lldb_private;
82using namespace lldb_private::process_linux;
83
84// A NativeRegisterContext is constructed per thread, but all threads' registers
85// will contain the same fields. Therefore this mutex prevents each instance
86// competing with the other, and subsequent instances from having to detect the
87// fields all over again.
88static std::mutex g_register_flags_detector_mutex;
89static Arm64RegisterFlagsDetector g_register_flags_detector;
90
91std::unique_ptr<NativeRegisterContextLinux>
93 const ArchSpec &target_arch, NativeThreadLinux &native_thread) {
94 switch (target_arch.GetMachine()) {
95 case llvm::Triple::arm:
96 return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,
97 native_thread);
98 case llvm::Triple::aarch64: {
99 // Configure register sets supported by this AArch64 target.
100 // Read SVE header to check for SVE support.
101 struct sve::user_sve_header sve_header;
102 struct iovec ioVec;
103 ioVec.iov_base = &sve_header;
104 ioVec.iov_len = sizeof(sve_header);
105 unsigned int regset = NT_ARM_SVE;
106
107 Flags opt_regsets;
109 native_thread.GetID(), &regset,
110 &ioVec, sizeof(sve_header))
111 .Success()) {
113
114 // We may also have the Scalable Matrix Extension (SME) which adds a
115 // streaming SVE mode.
116 ioVec.iov_len = sizeof(sve_header);
117 regset = NT_ARM_SSVE;
119 native_thread.GetID(), &regset,
120 &ioVec, sizeof(sve_header))
121 .Success())
123 }
124
125 sve::user_za_header za_header;
126 ioVec.iov_base = &za_header;
127 ioVec.iov_len = sizeof(za_header);
128 regset = NT_ARM_ZA;
130 native_thread.GetID(), &regset,
131 &ioVec, sizeof(za_header))
132 .Success())
134
135 // SME's ZT0 is a 512 bit register.
136 std::array<uint8_t, 64> zt_reg;
137 ioVec.iov_base = zt_reg.data();
138 ioVec.iov_len = zt_reg.size();
139 regset = NT_ARM_ZT;
141 native_thread.GetID(), &regset,
142 &ioVec, zt_reg.size())
143 .Success())
145
146 NativeProcessLinux &process = native_thread.GetProcess();
147
148 std::optional<uint64_t> auxv_at_hwcap =
150 if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))
152
153 std::optional<uint64_t> auxv_at_hwcap2 =
155 if (auxv_at_hwcap2) {
156 if (*auxv_at_hwcap2 & HWCAP2_MTE)
158 if (*auxv_at_hwcap2 & HWCAP2_FPMR)
160 if (*auxv_at_hwcap & HWCAP_GCS)
162 }
163
165
166 std::optional<uint64_t> auxv_at_hwcap3 =
168 std::lock_guard<std::mutex> lock(g_register_flags_detector_mutex);
169 if (!g_register_flags_detector.HasDetected())
170 g_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0),
171 auxv_at_hwcap2.value_or(0),
172 auxv_at_hwcap3.value_or(0));
173
174 auto register_info_up =
175 std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
176 return std::make_unique<NativeRegisterContextLinux_arm64>(
177 target_arch, native_thread, std::move(register_info_up));
178 }
179 default:
180 llvm_unreachable("have no register context for architecture");
181 }
182}
183
184llvm::Expected<ArchSpec>
186 return DetermineArchitectureViaGPR(
188}
189
190NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
191 const ArchSpec &target_arch, NativeThreadProtocol &native_thread,
192 std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)
193 : NativeRegisterContextRegisterInfo(native_thread,
194 register_info_up.release()),
195 NativeRegisterContextLinux(native_thread) {
196 g_register_flags_detector.UpdateRegisterInfo(
197 GetRegisterInfoInterface().GetRegisterInfo(),
198 GetRegisterInfoInterface().GetRegisterCount());
199
200 ::memset(&m_fpr, 0, sizeof(m_fpr));
201 ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
202 ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
203 ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));
204 ::memset(&m_sve_header, 0, sizeof(m_sve_header));
205 ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
206 ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));
207 ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));
208 ::memset(&m_gcs_regs, 0, sizeof(m_gcs_regs));
209 std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);
210
211 m_mte_ctrl_reg = 0;
212 m_fpmr_reg = 0;
213
214 // 16 is just a maximum value, query hardware for actual watchpoint count
215 m_max_hwp_supported = 16;
216 m_max_hbp_supported = 16;
217
218 m_refresh_hwdebug_info = true;
219
220 m_gpr_is_valid = false;
221 m_fpu_is_valid = false;
222 m_sve_buffer_is_valid = false;
223 m_sve_header_is_valid = false;
224 m_pac_mask_is_valid = false;
225 m_mte_ctrl_is_valid = false;
226 m_tls_is_valid = false;
227 m_zt_buffer_is_valid = false;
228 m_fpmr_is_valid = false;
229 m_gcs_is_valid = false;
230
231 // SME adds the tpidr2 register
232 m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)
233 : sizeof(m_tls_regs.tpidr_reg);
234
235 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())
236 m_sve_state = SVEState::Unknown;
237 else
238 m_sve_state = SVEState::Disabled;
239}
240
242NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
243 return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
244}
245
246uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
247 return GetRegisterInfo().GetRegisterSetCount();
248}
249
250const RegisterSet *
251NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
252 return GetRegisterInfo().GetRegisterSet(set_index);
253}
254
255uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
256 uint32_t count = 0;
257 for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
258 count += GetRegisterSet(set_index)->num_registers;
259 return count;
260}
261
262Status
263NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
264 RegisterValue &reg_value) {
266
267 if (!reg_info) {
268 error = Status::FromErrorString("reg_info NULL");
269 return error;
270 }
271
272 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
273
274 if (reg == LLDB_INVALID_REGNUM)
276 "no lldb regnum for %s",
277 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
278
279 uint8_t *src;
280 uint32_t offset = LLDB_INVALID_INDEX32;
281 uint64_t sve_vg;
282 std::vector<uint8_t> sve_reg_non_live;
283
284 if (IsGPR(reg)) {
285 error = ReadGPR();
286 if (error.Fail())
287 return error;
288
289 offset = reg_info->byte_offset;
290 assert(offset < GetGPRSize());
291 src = (uint8_t *)GetGPRBuffer() + offset;
292
293 } else if (IsFPR(reg)) {
294 if (m_sve_state == SVEState::Disabled) {
295 // SVE is disabled take legacy route for FPU register access
296 error = ReadFPR();
297 if (error.Fail())
298 return error;
299
300 offset = CalculateFprOffset(reg_info);
301 assert(offset < GetFPRSize());
302 src = (uint8_t *)GetFPRBuffer() + offset;
303 } else {
304 // SVE or SSVE enabled, we will read and cache SVE ptrace data.
305 // In SIMD or Full mode, the data comes from the SVE regset. In streaming
306 // mode it comes from the streaming SVE regset.
307 error = ReadAllSVE();
308 if (error.Fail())
309 return error;
310
311 // FPSR and FPCR will be located right after Z registers in
312 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
313 // will be located at the end of register data after an alignment
314 // correction based on currently selected vector length.
315 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
316 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
317 sve_reg_num = reg;
318 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
319 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
320 else if (m_sve_state == SVEState::FPSIMD)
321 offset = sve::ptrace_fpsimd_offset + (32 * 16);
322 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
323 sve_reg_num = reg;
324 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
325 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
326 else if (m_sve_state == SVEState::FPSIMD)
327 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
328 } else {
329 // Extract SVE Z register value register number for this reg_info
330 if (reg_info->value_regs &&
331 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
332 sve_reg_num = reg_info->value_regs[0];
333 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
334 }
335
336 assert(offset < GetSVEBufferSize());
337 src = (uint8_t *)GetSVEBuffer() + offset;
338 }
339 } else if (IsTLS(reg)) {
340 error = ReadTLS();
341 if (error.Fail())
342 return error;
343
344 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
345 assert(offset < GetTLSBufferSize());
346 src = (uint8_t *)GetTLSBuffer() + offset;
347 } else if (IsSVE(reg)) {
348 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
349 return Status::FromErrorString("SVE disabled or not supported");
350
351 if (GetRegisterInfo().IsSVERegVG(reg)) {
352 sve_vg = GetSVERegVG();
353 src = (uint8_t *)&sve_vg;
354 } else {
355 // SVE enabled, we will read and cache SVE ptrace data
356 error = ReadAllSVE();
357 if (error.Fail())
358 return error;
359
360 if (m_sve_state == SVEState::FPSIMD) {
361 // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so
362 // just copy 16 bytes of v register to the start of z register. All
363 // other SVE register will be set to zero.
364 sve_reg_non_live.resize(reg_info->byte_size, 0);
365 src = sve_reg_non_live.data();
366
367 if (GetRegisterInfo().IsSVEZReg(reg)) {
368 offset = CalculateSVEOffset(reg_info);
369 assert(offset < GetSVEBufferSize());
370 ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,
371 16);
372 }
373 } else {
374 offset = CalculateSVEOffset(reg_info);
375 assert(offset < GetSVEBufferSize());
376 src = (uint8_t *)GetSVEBuffer() + offset;
377 }
378 }
379 } else if (IsPAuth(reg)) {
380 error = ReadPAuthMask();
381 if (error.Fail())
382 return error;
383
384 offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();
385 assert(offset < GetPACMaskSize());
386 src = (uint8_t *)GetPACMask() + offset;
387 } else if (IsMTE(reg)) {
388 error = ReadMTEControl();
389 if (error.Fail())
390 return error;
391
392 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
393 assert(offset < GetMTEControlSize());
394 src = (uint8_t *)GetMTEControl() + offset;
395 } else if (IsSME(reg)) {
396 if (GetRegisterInfo().IsSMERegZA(reg)) {
397 error = ReadZAHeader();
398 if (error.Fail())
399 return error;
400
401 // If there is only a header and no registers, ZA is inactive. Read as 0
402 // in this case.
403 if (m_za_header.size == sizeof(m_za_header)) {
404 // This will get reconfigured/reset later, so we are safe to use it.
405 // ZA is a square of VL * VL and the ptrace buffer also includes the
406 // header itself.
407 m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +
408 GetZAHeaderSize());
409 std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);
410 } else {
411 // ZA is active, read the real register.
412 error = ReadZA();
413 if (error.Fail())
414 return error;
415 }
416
417 // ZA is part of the SME set but uses a separate member buffer for
418 // storage. Therefore its effective byte offset is always 0 even if it
419 // isn't 0 within the SME register set.
420 src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
421 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
422 // Unlike ZA, the kernel will return register data for ZT0 when ZA is not
423 // enabled. This data will be all 0s so we don't have to invent anything
424 // like we did for ZA.
425 error = ReadZT();
426 if (error.Fail())
427 return error;
428
429 src = (uint8_t *)GetZTBuffer();
430 } else {
431 error = ReadSMESVG();
432 if (error.Fail())
433 return error;
434
435 // This is a psuedo so it never fails.
436 ReadSMEControl();
437
438 offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();
439 assert(offset < GetSMEPseudoBufferSize());
440 src = (uint8_t *)GetSMEPseudoBuffer() + offset;
441 }
442 } else if (IsFPMR(reg)) {
443 error = ReadFPMR();
444 if (error.Fail())
445 return error;
446
447 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
448 assert(offset < GetFPMRBufferSize());
449 src = (uint8_t *)GetFPMRBuffer() + offset;
450 } else if (IsGCS(reg)) {
451 error = ReadGCS();
452 if (error.Fail())
453 return error;
454
455 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
456 assert(offset < GetGCSBufferSize());
457 src = (uint8_t *)GetGCSBuffer() + offset;
458 } else
460 "failed - register wasn't recognized to be a GPR or an FPR, "
461 "write strategy unknown");
462
463 reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,
465
466 return error;
467}
468
469Status NativeRegisterContextLinux_arm64::WriteRegister(
470 const RegisterInfo *reg_info, const RegisterValue &reg_value) {
472
473 if (!reg_info)
474 return Status::FromErrorString("reg_info NULL");
475
476 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
477
478 if (reg == LLDB_INVALID_REGNUM)
480 "no lldb regnum for %s",
481 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
482
483 uint8_t *dst;
484 uint32_t offset = LLDB_INVALID_INDEX32;
485 std::vector<uint8_t> sve_reg_non_live;
486
487 if (IsGPR(reg)) {
488 error = ReadGPR();
489 if (error.Fail())
490 return error;
491
492 assert(reg_info->byte_offset < GetGPRSize());
493 dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;
494 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
495
496 return WriteGPR();
497 } else if (IsFPR(reg)) {
498 if (m_sve_state == SVEState::Disabled) {
499 // SVE is disabled take legacy route for FPU register access
500 error = ReadFPR();
501 if (error.Fail())
502 return error;
503
504 offset = CalculateFprOffset(reg_info);
505 assert(offset < GetFPRSize());
506 dst = (uint8_t *)GetFPRBuffer() + offset;
507 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
508
509 return WriteFPR();
510 } else {
511 // SVE enabled, we will read and cache SVE ptrace data.
512 error = ReadAllSVE();
513 if (error.Fail())
514 return error;
515
516 // FPSR and FPCR will be located right after Z registers in
517 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
518 // will be located at the end of register data after an alignment
519 // correction based on currently selected vector length.
520 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
521 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
522 sve_reg_num = reg;
523 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
524 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
525 else if (m_sve_state == SVEState::FPSIMD)
526 offset = sve::ptrace_fpsimd_offset + (32 * 16);
527 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
528 sve_reg_num = reg;
529 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
530 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
531 else if (m_sve_state == SVEState::FPSIMD)
532 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
533 } else {
534 // Extract SVE Z register value register number for this reg_info
535 if (reg_info->value_regs &&
536 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
537 sve_reg_num = reg_info->value_regs[0];
538 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
539 }
540
541 assert(offset < GetSVEBufferSize());
542 dst = (uint8_t *)GetSVEBuffer() + offset;
543 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
544 return WriteAllSVE();
545 }
546 } else if (IsSVE(reg)) {
547 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
548 return Status::FromErrorString("SVE disabled or not supported");
549 else {
550 // Target has SVE enabled, we will read and cache SVE ptrace data
551 error = ReadAllSVE();
552 if (error.Fail())
553 return error;
554
555 if (GetRegisterInfo().IsSVERegVG(reg)) {
556 uint64_t vg_value = reg_value.GetAsUInt64();
557
558 if (sve::vl_valid(vg_value * 8)) {
559 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
560 return error;
561
562 SetSVERegVG(vg_value);
563
564 error = WriteSVEHeader();
565 if (error.Success()) {
566 // Changing VG during streaming mode also changes the size of ZA.
567 if (m_sve_state == SVEState::Streaming)
568 m_za_header_is_valid = false;
569 ConfigureRegisterContext();
570 }
571
572 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
573 return error;
574 }
575
576 return Status::FromErrorString("SVE vector length update failed.");
577 }
578
579 // If target supports SVE but currently in FPSIMD mode.
580 if (m_sve_state == SVEState::FPSIMD) {
581 // Here we will check if writing this SVE register enables
582 // SVEState::Full
583 bool set_sve_state_full = false;
584 const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();
585 if (GetRegisterInfo().IsSVEZReg(reg)) {
586 for (uint32_t i = 16; i < reg_info->byte_size; i++) {
587 if (reg_bytes[i]) {
588 set_sve_state_full = true;
589 break;
590 }
591 }
592 } else if (GetRegisterInfo().IsSVEPReg(reg) ||
593 reg == GetRegisterInfo().GetRegNumSVEFFR()) {
594 for (uint32_t i = 0; i < reg_info->byte_size; i++) {
595 if (reg_bytes[i]) {
596 set_sve_state_full = true;
597 break;
598 }
599 }
600 }
601
602 if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {
603 // We are writing a Z register which is zero beyond 16 bytes so copy
604 // first 16 bytes only as SVE payload mirrors legacy fpsimd structure
605 offset = CalculateSVEOffset(reg_info);
606 assert(offset < GetSVEBufferSize());
607 dst = (uint8_t *)GetSVEBuffer() + offset;
608 ::memcpy(dst, reg_value.GetBytes(), 16);
609
610 return WriteAllSVE();
611 } else
613 "SVE state change operation not supported");
614 } else {
615 offset = CalculateSVEOffset(reg_info);
616 assert(offset < GetSVEBufferSize());
617 dst = (uint8_t *)GetSVEBuffer() + offset;
618 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
619 return WriteAllSVE();
620 }
621 }
622 } else if (IsMTE(reg)) {
623 error = ReadMTEControl();
624 if (error.Fail())
625 return error;
626
627 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
628 assert(offset < GetMTEControlSize());
629 dst = (uint8_t *)GetMTEControl() + offset;
630 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
631
632 return WriteMTEControl();
633 } else if (IsTLS(reg)) {
634 error = ReadTLS();
635 if (error.Fail())
636 return error;
637
638 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
639 assert(offset < GetTLSBufferSize());
640 dst = (uint8_t *)GetTLSBuffer() + offset;
641 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
642
643 return WriteTLS();
644 } else if (IsSME(reg)) {
645 if (GetRegisterInfo().IsSMERegZA(reg)) {
646 error = ReadZA();
647 if (error.Fail())
648 return error;
649
650 // ZA is part of the SME set but not stored with the other SME registers.
651 // So its byte offset is effectively always 0.
652 dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
653 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
654
655 // While this is writing a header that contains a vector length, the only
656 // way to change that is via the vg register. So here we assume the length
657 // will always be the current length and no reconfigure is needed.
658 return WriteZA();
659 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
660 error = ReadZT();
661 if (error.Fail())
662 return error;
663
664 dst = (uint8_t *)GetZTBuffer();
665 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
666
667 return WriteZT();
668 } else
670 "Writing to SVG or SVCR is not supported.");
671 } else if (IsFPMR(reg)) {
672 error = ReadFPMR();
673 if (error.Fail())
674 return error;
675
676 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
677 assert(offset < GetFPMRBufferSize());
678 dst = (uint8_t *)GetFPMRBuffer() + offset;
679 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
680
681 return WriteFPMR();
682 } else if (IsGCS(reg)) {
683 error = ReadGCS();
684 if (error.Fail())
685 return error;
686
687 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
688 assert(offset < GetGCSBufferSize());
689 dst = (uint8_t *)GetGCSBuffer() + offset;
690 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
691
692 return WriteGCS();
693 }
694
695 return Status::FromErrorString("Failed to write register value");
696}
697
698enum RegisterSetType : uint32_t {
699 GPR,
700 SVE, // Used for SVE and SSVE.
701 FPR, // When there is no SVE, or SVE in FPSIMD mode.
702 // Pointer authentication registers are read only, so not included here.
703 MTE,
704 TLS,
705 SME, // ZA only, because SVCR and SVG are pseudo registers.
706 SME2, // ZT only.
707 FPMR,
708 GCS, // Guarded Control Stack registers.
709};
710
711static uint8_t *AddRegisterSetType(uint8_t *dst,
712 RegisterSetType register_set_type) {
713 *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;
714 return dst + sizeof(uint32_t);
715}
716
717static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {
718 ::memcpy(dst, src, size);
719 return dst + size;
720}
721
722static uint8_t *AddSavedRegisters(uint8_t *dst,
723 enum RegisterSetType register_set_type,
724 void *src, size_t size) {
725 dst = AddRegisterSetType(dst, register_set_type);
726 return AddSavedRegistersData(dst, src, size);
727}
728
729Status
730NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {
732 cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();
733 error = ReadGPR();
734 if (error.Fail())
735 return error;
736
737 if (GetRegisterInfo().IsZAPresent()) {
738 error = ReadZAHeader();
739 if (error.Fail())
740 return error;
741 // Use header size here because the buffer may contain fake data when ZA is
742 // disabled. We do not want to write this fake data (all 0s) because this
743 // would tell the kernel that we want ZA to become active. Which is the
744 // opposite of what we want in the case where it is currently inactive.
745 cached_size += sizeof(RegisterSetType) + m_za_header.size;
746 // For the same reason, we need to force it to be re-read so that it will
747 // always contain the real header.
748 m_za_buffer_is_valid = false;
749 error = ReadZA();
750 if (error.Fail())
751 return error;
752
753 // We will only be restoring ZT data if ZA is active. As writing to an
754 // inactive ZT enables ZA, which may not be desireable.
755 if (
756 // If we have ZT0, or in other words, if we have SME2.
757 GetRegisterInfo().IsZTPresent() &&
758 // And ZA is active, which means that ZT0 is also active.
759 m_za_header.size > sizeof(m_za_header)) {
760 cached_size += sizeof(RegisterSetType) + GetZTBufferSize();
761 // The kernel handles an inactive ZT0 for us, and it will read as 0s if
762 // inactive (unlike ZA where we fake that behaviour).
763 error = ReadZT();
764 if (error.Fail())
765 return error;
766 }
767 }
768
769 // If SVE is enabled we need not copy FPR separately.
770 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
771 // Store mode and register data.
772 cached_size +=
773 sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();
774 error = ReadAllSVE();
775 } else {
776 cached_size += sizeof(RegisterSetType) + GetFPRSize();
777 error = ReadFPR();
778 }
779 if (error.Fail())
780 return error;
781
782 if (GetRegisterInfo().IsMTEPresent()) {
783 cached_size += sizeof(RegisterSetType) + GetMTEControlSize();
784 error = ReadMTEControl();
785 if (error.Fail())
786 return error;
787 }
788
789 if (GetRegisterInfo().IsFPMRPresent()) {
790 cached_size += sizeof(RegisterSetType) + GetFPMRBufferSize();
791 error = ReadFPMR();
792 if (error.Fail())
793 return error;
794 }
795
796 if (GetRegisterInfo().IsGCSPresent()) {
797 cached_size += sizeof(RegisterSetType) + GetGCSBufferSize();
798 error = ReadGCS();
799 if (error.Fail())
800 return error;
801 }
802
803 // tpidr is always present but tpidr2 depends on SME.
804 cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();
805 error = ReadTLS();
806
807 return error;
808}
809
810Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
812 // AArch64 register data must contain GPRs and either FPR or SVE registers.
813 // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).
814 // Finally an optional MTE register. Pointer Authentication (PAC) registers
815 // are read-only and will be skipped.
816
817 // In order to create register data checkpoint we first read all register
818 // values if not done already and calculate total size of register set data.
819 // We store all register values in data_sp by copying full PTrace data that
820 // corresponds to register sets enabled by current register context.
821
822 uint32_t reg_data_byte_size = 0;
823 Status error = CacheAllRegisters(reg_data_byte_size);
824 if (error.Fail())
825 return error;
826
827 data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
828 uint8_t *dst = data_sp->GetBytes();
829
830 dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),
831 GetGPRBufferSize());
832
833 // Streaming SVE and the ZA register both use the streaming vector length.
834 // When you change this, the kernel will invalidate parts of the process
835 // state. Therefore we need a specific order of restoration for each mode, if
836 // we also have ZA to restore.
837 //
838 // Streaming mode enabled, ZA enabled:
839 // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.
840 // * Write ZA, this set SVCR.ZA. The register data we provide is written to
841 // ZA.
842 // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both
843 // register sets.
844 //
845 // Streaming mode disabled, ZA enabled:
846 // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases
847 // the streaming vector length is changing, so the thread is converted into
848 // an FPSIMD thread if it is not already one. This also clears SVCR.SM.
849 // * Write SVE registers, which also clears SVCR.SM but most importantly, puts
850 // us into full SVE mode instead of FPSIMD mode (where the registers are
851 // actually the 128 bit Neon registers).
852 // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register
853 // state.
854 //
855 // Restoring in different orders leads to things like the SVE registers being
856 // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s
857 // (disabled and 0s looks the same from inside lldb since we fake the value
858 // when it's disabled).
859 //
860 // For more information on this, look up the uses of the relevant NT_ARM_
861 // constants and the functions vec_set_vector_length, sve_set_common and
862 // za_set in the Linux Kernel.
863
864 if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
865 // Use the header size not the buffer size, as we may be using the buffer
866 // for fake data, which we do not want to write out.
867 assert(m_za_header.size <= GetZABufferSize());
868 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
869 m_za_header.size);
870 }
871
872 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
873 dst = AddRegisterSetType(dst, RegisterSetType::SVE);
874 *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;
875 dst += sizeof(m_sve_state);
876 dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());
877 } else {
878 dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),
879 GetFPRSize());
880 }
881
882 if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
883 assert(m_za_header.size <= GetZABufferSize());
884 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
885 m_za_header.size);
886 }
887
888 // If ZT0 is present and we are going to be restoring an active ZA (which
889 // implies an active ZT0), then restore ZT0 after ZA has been set. This
890 // prevents us enabling ZA accidentally after the restore of ZA disabled it.
891 // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore
892 // there's nothing for us to restore if ZA was originally inactive.
893 if (
894 // If we have SME2 and therefore ZT0.
895 GetRegisterInfo().IsZTPresent() &&
896 // And ZA is enabled.
897 m_za_header.size > sizeof(m_za_header))
898 dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),
899 GetZTBufferSize());
900
901 if (GetRegisterInfo().IsMTEPresent()) {
902 dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),
903 GetMTEControlSize());
904 }
905
906 if (GetRegisterInfo().IsFPMRPresent()) {
907 dst = AddSavedRegisters(dst, RegisterSetType::FPMR, GetFPMRBuffer(),
908 GetFPMRBufferSize());
909 }
910
911 if (GetRegisterInfo().IsGCSPresent()) {
912 dst = AddSavedRegisters(dst, RegisterSetType::GCS, GetGCSBuffer(),
913 GetGCSBufferSize());
914 }
915
916 dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),
917 GetTLSBufferSize());
918
919 return error;
920}
921
922static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,
923 bool &is_valid, std::function<Status()> writer) {
924 ::memcpy(buffer, *src, len);
925 is_valid = true;
926 *src += len;
927 return writer();
928}
929
930Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
931 const lldb::DataBufferSP &data_sp) {
932 // AArch64 register data must contain GPRs, either FPR or SVE registers
933 // (which can be streaming or non-streaming) and optional MTE register.
934 // Pointer Authentication (PAC) registers are read-only and will be skipped.
935
936 // We store all register values in data_sp by copying full PTrace data that
937 // corresponds to register sets enabled by current register context. In order
938 // to restore from register data checkpoint we will first restore GPRs, based
939 // on size of remaining register data either SVE or FPRs should be restored
940 // next. SVE is not enabled if we have register data size less than or equal
941 // to size of GPR + FPR + MTE.
942
944 if (!data_sp) {
946 "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",
947 __FUNCTION__);
948 return error;
949 }
950
951 const uint8_t *src = data_sp->GetBytes();
952 if (src == nullptr) {
954 "NativeRegisterContextLinux_arm64::%s "
955 "DataBuffer::GetBytes() returned a null "
956 "pointer",
957 __FUNCTION__);
958 return error;
959 }
960
961 uint64_t reg_data_min_size =
962 GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));
963 if (data_sp->GetByteSize() < reg_data_min_size) {
965 "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "
966 "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,
967 __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());
968 return error;
969 }
970
971 const uint8_t *end = src + data_sp->GetByteSize();
972 while (src < end) {
973 const RegisterSetType kind =
974 *reinterpret_cast<const RegisterSetType *>(src);
975 src += sizeof(RegisterSetType);
976
977 switch (kind) {
978 case RegisterSetType::GPR:
979 error = RestoreRegisters(
980 GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,
981 std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));
982 break;
983 case RegisterSetType::SVE:
984 // Restore to the correct mode, streaming or not.
985 m_sve_state = static_cast<SVEState>(*src);
986 src += sizeof(m_sve_state);
987
988 // First write SVE header. We do not use RestoreRegisters because we do
989 // not want src to be modified yet.
990 ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
991 if (!sve::vl_valid(m_sve_header.vl)) {
992 m_sve_header_is_valid = false;
994 "NativeRegisterContextLinux_arm64::%s "
995 "Invalid SVE header in data_sp",
996 __FUNCTION__);
997 return error;
998 }
999 m_sve_header_is_valid = true;
1000 error = WriteSVEHeader();
1001 if (error.Fail())
1002 return error;
1003
1004 // SVE header has been written configure SVE vector length if needed.
1005 // This could change ZA data too, but that will be restored again later
1006 // anyway.
1007 ConfigureRegisterContext();
1008
1009 // Write header and register data, incrementing src this time.
1010 error = RestoreRegisters(
1011 GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,
1012 std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));
1013 break;
1014 case RegisterSetType::FPR:
1015 error = RestoreRegisters(
1016 GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,
1017 std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));
1018 break;
1019 case RegisterSetType::MTE:
1020 error = RestoreRegisters(
1021 GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,
1022 std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));
1023 break;
1024 case RegisterSetType::TLS:
1025 error = RestoreRegisters(
1026 GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,
1027 std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));
1028 break;
1029 case RegisterSetType::SME:
1030 // To enable or disable ZA you write the regset with or without register
1031 // data. The kernel detects this by looking at the ioVec's length, not the
1032 // ZA header size you pass in. Therefore we must write header and register
1033 // data (if present) in one go every time. Read the header only first just
1034 // to get the size.
1035 ::memcpy(GetZAHeader(), src, GetZAHeaderSize());
1036 // Read the header and register data. Can't use the buffer size here, it
1037 // may be incorrect due to being filled with dummy data previously. Resize
1038 // this so WriteZA uses the correct size.
1039 m_za_ptrace_payload.resize(m_za_header.size);
1040 ::memcpy(GetZABuffer(), src, GetZABufferSize());
1041 m_za_buffer_is_valid = true;
1042
1043 error = WriteZA();
1044 if (error.Fail())
1045 return error;
1046
1047 // Update size of ZA, which resizes the ptrace payload potentially
1048 // trashing our copy of the data we just wrote.
1049 ConfigureRegisterContext();
1050
1051 // ZA buffer now has proper size, read back the data we wrote above, from
1052 // ptrace.
1053 error = ReadZA();
1054 src += GetZABufferSize();
1055 break;
1056 case RegisterSetType::SME2:
1057 // Doing this would activate an inactive ZA, however we will only get here
1058 // if the state we are restoring had an active ZA. Restoring ZT0 will
1059 // always come after restoring ZA.
1060 error = RestoreRegisters(
1061 GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,
1062 std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));
1063 break;
1064 case RegisterSetType::FPMR:
1065 error = RestoreRegisters(
1066 GetFPMRBuffer(), &src, GetFPMRBufferSize(), m_fpmr_is_valid,
1067 std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));
1068 break;
1069 case RegisterSetType::GCS:
1070 // It is not permitted to enable GCS via ptrace. We can disable it, but
1071 // to keep things simple we will not revert any change to the
1072 // PR_SHADOW_STACK_ENABLE bit. Instead patch in the current enable bit
1073 // into the registers we are about to restore.
1074 m_gcs_is_valid = false;
1075 error = ReadGCS();
1076 if (error.Fail())
1077 return error;
1078
1079 uint64_t enable_bit = m_gcs_regs.features_enabled & 1UL;
1080 gcs_regs new_gcs_regs = *reinterpret_cast<const gcs_regs *>(src);
1081 new_gcs_regs.features_enabled =
1082 (new_gcs_regs.features_enabled & ~1UL) | enable_bit;
1083
1084 const uint8_t *new_gcs_src =
1085 reinterpret_cast<const uint8_t *>(&new_gcs_regs);
1086 error = RestoreRegisters(
1087 GetGCSBuffer(), &new_gcs_src, GetGCSBufferSize(), m_gcs_is_valid,
1088 std::bind(&NativeRegisterContextLinux_arm64::WriteGCS, this));
1089 src += GetGCSBufferSize();
1090
1091 break;
1092 }
1093
1094 if (error.Fail())
1095 return error;
1096 }
1097
1098 return error;
1099}
1100
1101bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
1102 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1104 return true;
1105 return false;
1106}
1107
1108bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
1109 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1111 return true;
1112 return false;
1113}
1114
1115bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {
1116 return GetRegisterInfo().IsSVEReg(reg);
1117}
1118
1119bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {
1120 return GetRegisterInfo().IsSMEReg(reg);
1121}
1122
1123bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {
1124 return GetRegisterInfo().IsPAuthReg(reg);
1125}
1126
1127bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
1128 return GetRegisterInfo().IsMTEReg(reg);
1129}
1130
1131bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
1132 return GetRegisterInfo().IsTLSReg(reg);
1133}
1134
1135bool NativeRegisterContextLinux_arm64::IsFPMR(unsigned reg) const {
1136 return GetRegisterInfo().IsFPMRReg(reg);
1137}
1138
1139bool NativeRegisterContextLinux_arm64::IsGCS(unsigned reg) const {
1140 return GetRegisterInfo().IsGCSReg(reg);
1141}
1142
1143llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
1144 if (!m_refresh_hwdebug_info) {
1145 return llvm::Error::success();
1146 }
1147
1148 ::pid_t tid = m_thread.GetID();
1149
1150 Status error = arm64::ReadHardwareDebugInfo(tid, m_max_hwp_supported,
1151 m_max_hbp_supported);
1152 if (error.Fail())
1153 return error.ToError();
1154
1155 m_refresh_hwdebug_info = false;
1156
1157 return llvm::Error::success();
1158}
1159
1160llvm::Error
1161NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
1162 uint32_t max_supported =
1163 (hwbType == eDREGTypeWATCH) ? m_max_hwp_supported : m_max_hbp_supported;
1164 auto &regs = (hwbType == eDREGTypeWATCH) ? m_hwp_regs : m_hbp_regs;
1165 return arm64::WriteHardwareDebugRegs(hwbType, m_thread.GetID(), max_supported,
1166 regs)
1167 .ToError();
1168}
1169
1170Status NativeRegisterContextLinux_arm64::ReadGPR() {
1171 Status error;
1172
1173 if (m_gpr_is_valid)
1174 return error;
1175
1176 struct iovec ioVec;
1177 ioVec.iov_base = GetGPRBuffer();
1178 ioVec.iov_len = GetGPRBufferSize();
1179
1180 error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1181
1182 if (error.Success())
1183 m_gpr_is_valid = true;
1184
1185 return error;
1186}
1187
1188Status NativeRegisterContextLinux_arm64::WriteGPR() {
1189 Status error = ReadGPR();
1190 if (error.Fail())
1191 return error;
1192
1193 struct iovec ioVec;
1194 ioVec.iov_base = GetGPRBuffer();
1195 ioVec.iov_len = GetGPRBufferSize();
1196
1197 m_gpr_is_valid = false;
1198
1199 return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1200}
1201
1202Status NativeRegisterContextLinux_arm64::ReadFPR() {
1203 Status error;
1204
1205 if (m_fpu_is_valid)
1206 return error;
1207
1208 struct iovec ioVec;
1209 ioVec.iov_base = GetFPRBuffer();
1210 ioVec.iov_len = GetFPRSize();
1211
1212 error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1213
1214 if (error.Success())
1215 m_fpu_is_valid = true;
1216
1217 return error;
1218}
1219
1220Status NativeRegisterContextLinux_arm64::WriteFPR() {
1221 Status error = ReadFPR();
1222 if (error.Fail())
1223 return error;
1224
1225 struct iovec ioVec;
1226 ioVec.iov_base = GetFPRBuffer();
1227 ioVec.iov_len = GetFPRSize();
1228
1229 m_fpu_is_valid = false;
1230
1231 return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1232}
1233
1234void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
1235 m_gpr_is_valid = false;
1236 m_fpu_is_valid = false;
1237 m_sve_buffer_is_valid = false;
1238 m_sve_header_is_valid = false;
1239 m_za_buffer_is_valid = false;
1240 m_za_header_is_valid = false;
1241 m_pac_mask_is_valid = false;
1242 m_mte_ctrl_is_valid = false;
1243 m_tls_is_valid = false;
1244 m_zt_buffer_is_valid = false;
1245 m_fpmr_is_valid = false;
1246 m_gcs_is_valid = false;
1247
1248 // Update SVE and ZA registers in case there is change in configuration.
1249 ConfigureRegisterContext();
1250}
1251
1252unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
1253 return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;
1254}
1255
1256Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {
1257 Status error;
1258
1259 if (m_sve_header_is_valid)
1260 return error;
1261
1262 struct iovec ioVec;
1263 ioVec.iov_base = GetSVEHeader();
1264 ioVec.iov_len = GetSVEHeaderSize();
1265
1266 error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1267
1268 if (error.Success())
1269 m_sve_header_is_valid = true;
1270
1271 return error;
1272}
1273
1274Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
1275 Status error;
1276
1277 if (m_pac_mask_is_valid)
1278 return error;
1279
1280 struct iovec ioVec;
1281 ioVec.iov_base = GetPACMask();
1282 ioVec.iov_len = GetPACMaskSize();
1283
1284 error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
1285
1286 if (error.Success())
1287 m_pac_mask_is_valid = true;
1288
1289 return error;
1290}
1291
1292Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {
1293 Status error;
1294
1295 error = ReadSVEHeader();
1296 if (error.Fail())
1297 return error;
1298
1299 struct iovec ioVec;
1300 ioVec.iov_base = GetSVEHeader();
1301 ioVec.iov_len = GetSVEHeaderSize();
1302
1303 m_sve_buffer_is_valid = false;
1304 m_sve_header_is_valid = false;
1305 m_fpu_is_valid = false;
1306
1307 return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1308}
1309
1310Status NativeRegisterContextLinux_arm64::ReadAllSVE() {
1311 Status error;
1312 if (m_sve_buffer_is_valid)
1313 return error;
1314
1315 struct iovec ioVec;
1316 ioVec.iov_base = GetSVEBuffer();
1317 ioVec.iov_len = GetSVEBufferSize();
1318
1319 error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1320
1321 if (error.Success())
1322 m_sve_buffer_is_valid = true;
1323
1324 return error;
1325}
1326
1327Status NativeRegisterContextLinux_arm64::WriteAllSVE() {
1328 Status error;
1329
1330 error = ReadAllSVE();
1331 if (error.Fail())
1332 return error;
1333
1334 struct iovec ioVec;
1335
1336 ioVec.iov_base = GetSVEBuffer();
1337 ioVec.iov_len = GetSVEBufferSize();
1338
1339 m_sve_buffer_is_valid = false;
1340 m_sve_header_is_valid = false;
1341 m_fpu_is_valid = false;
1342
1343 return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1344}
1345
1346Status NativeRegisterContextLinux_arm64::ReadSMEControl() {
1347 // The real register is SVCR and is accessible from EL0. However we don't want
1348 // to have to JIT code into the target process so we'll just recreate it using
1349 // what we know from ptrace.
1350
1351 // Bit 0 indicates whether streaming mode is active.
1352 m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;
1353
1354 // Bit 1 indicates whether the array storage is active.
1355 // It is active if we can read the header and the size field tells us that
1356 // there is register data following it.
1357 Status error = ReadZAHeader();
1358 if (error.Success() && (m_za_header.size > sizeof(m_za_header)))
1359 m_sme_pseudo_regs.ctrl_reg |= 2;
1360
1361 return error;
1362}
1363
1364Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
1365 Status error;
1366
1367 if (m_mte_ctrl_is_valid)
1368 return error;
1369
1370 struct iovec ioVec;
1371 ioVec.iov_base = GetMTEControl();
1372 ioVec.iov_len = GetMTEControlSize();
1373
1374 error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1375
1376 if (error.Success())
1377 m_mte_ctrl_is_valid = true;
1378
1379 return error;
1380}
1381
1382Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
1383 Status error;
1384
1385 error = ReadMTEControl();
1386 if (error.Fail())
1387 return error;
1388
1389 struct iovec ioVec;
1390 ioVec.iov_base = GetMTEControl();
1391 ioVec.iov_len = GetMTEControlSize();
1392
1393 m_mte_ctrl_is_valid = false;
1394
1395 return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1396}
1397
1398Status NativeRegisterContextLinux_arm64::ReadTLS() {
1399 Status error;
1400
1401 if (m_tls_is_valid)
1402 return error;
1403
1404 struct iovec ioVec;
1405 ioVec.iov_base = GetTLSBuffer();
1406 ioVec.iov_len = GetTLSBufferSize();
1407
1408 error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1409
1410 if (error.Success())
1411 m_tls_is_valid = true;
1412
1413 return error;
1414}
1415
1416Status NativeRegisterContextLinux_arm64::WriteTLS() {
1417 Status error;
1418
1419 error = ReadTLS();
1420 if (error.Fail())
1421 return error;
1422
1423 struct iovec ioVec;
1424 ioVec.iov_base = GetTLSBuffer();
1425 ioVec.iov_len = GetTLSBufferSize();
1426
1427 m_tls_is_valid = false;
1428
1429 return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1430}
1431
1432Status NativeRegisterContextLinux_arm64::ReadGCS() {
1433 Status error;
1434
1435 if (m_gcs_is_valid)
1436 return error;
1437
1438 struct iovec ioVec;
1439 ioVec.iov_base = GetGCSBuffer();
1440 ioVec.iov_len = GetGCSBufferSize();
1441
1442 error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1443
1444 if (error.Success())
1445 m_gcs_is_valid = true;
1446
1447 return error;
1448}
1449
1450Status NativeRegisterContextLinux_arm64::WriteGCS() {
1451 Status error;
1452
1453 error = ReadGCS();
1454 if (error.Fail())
1455 return error;
1456
1457 struct iovec ioVec;
1458 ioVec.iov_base = GetGCSBuffer();
1459 ioVec.iov_len = GetGCSBufferSize();
1460
1461 m_gcs_is_valid = false;
1462
1463 return WriteRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1464}
1465
1466Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
1467 Status error;
1468
1469 if (m_za_header_is_valid)
1470 return error;
1471
1472 struct iovec ioVec;
1473 ioVec.iov_base = GetZAHeader();
1474 ioVec.iov_len = GetZAHeaderSize();
1475
1476 error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
1477
1478 if (error.Success())
1479 m_za_header_is_valid = true;
1480
1481 return error;
1482}
1483
1484Status NativeRegisterContextLinux_arm64::ReadZA() {
1485 Status error;
1486
1487 if (m_za_buffer_is_valid)
1488 return error;
1489
1490 struct iovec ioVec;
1491 ioVec.iov_base = GetZABuffer();
1492 ioVec.iov_len = GetZABufferSize();
1493
1494 error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1495
1496 if (error.Success())
1497 m_za_buffer_is_valid = true;
1498
1499 return error;
1500}
1501
1502Status NativeRegisterContextLinux_arm64::WriteZA() {
1503 // Note that because the ZA ptrace payload contains the header also, this
1504 // method will write both. This is done because writing only the header
1505 // will disable ZA, even if .size in the header is correct for an enabled ZA.
1506 Status error;
1507
1508 error = ReadZA();
1509 if (error.Fail())
1510 return error;
1511
1512 struct iovec ioVec;
1513 ioVec.iov_base = GetZABuffer();
1514 ioVec.iov_len = GetZABufferSize();
1515
1516 m_za_buffer_is_valid = false;
1517 m_za_header_is_valid = false;
1518 // Writing to ZA may enable ZA, which means ZT0 may change too.
1519 m_zt_buffer_is_valid = false;
1520
1521 return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1522}
1523
1524Status NativeRegisterContextLinux_arm64::ReadZT() {
1525 Status error;
1526
1527 if (m_zt_buffer_is_valid)
1528 return error;
1529
1530 struct iovec ioVec;
1531 ioVec.iov_base = GetZTBuffer();
1532 ioVec.iov_len = GetZTBufferSize();
1533
1534 error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1535 m_zt_buffer_is_valid = error.Success();
1536
1537 return error;
1538}
1539
1540Status NativeRegisterContextLinux_arm64::WriteZT() {
1541 Status error;
1542
1543 error = ReadZT();
1544 if (error.Fail())
1545 return error;
1546
1547 struct iovec ioVec;
1548 ioVec.iov_base = GetZTBuffer();
1549 ioVec.iov_len = GetZTBufferSize();
1550
1551 m_zt_buffer_is_valid = false;
1552 // Writing to an inactive ZT0 will enable ZA as well, which invalidates our
1553 // current copy of it.
1554 m_za_buffer_is_valid = false;
1555 m_za_header_is_valid = false;
1556
1557 return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1558}
1559
1560Status NativeRegisterContextLinux_arm64::ReadFPMR() {
1561 Status error;
1562
1563 if (m_fpmr_is_valid)
1564 return error;
1565
1566 struct iovec ioVec;
1567 ioVec.iov_base = GetFPMRBuffer();
1568 ioVec.iov_len = GetFPMRBufferSize();
1569
1570 error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1571
1572 if (error.Success())
1573 m_fpmr_is_valid = true;
1574
1575 return error;
1576}
1577
1578Status NativeRegisterContextLinux_arm64::WriteFPMR() {
1579 Status error;
1580
1581 error = ReadFPMR();
1582 if (error.Fail())
1583 return error;
1584
1585 struct iovec ioVec;
1586 ioVec.iov_base = GetFPMRBuffer();
1587 ioVec.iov_len = GetFPMRBufferSize();
1588
1589 m_fpmr_is_valid = false;
1590
1591 return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1592}
1593
1594void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
1595 // ConfigureRegisterContext gets called from InvalidateAllRegisters
1596 // on every stop and configures SVE vector length and whether we are in
1597 // streaming SVE mode.
1598 // If m_sve_state is set to SVEState::Disabled on first stop, code below will
1599 // be deemed non operational for the lifetime of current process.
1600 if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {
1601 // If we have SVE we may also have the SVE streaming mode that SME added.
1602 // We can read the header of either mode, but only the active mode will
1603 // have valid register data.
1604
1605 // Check whether SME is present and the streaming SVE mode is active.
1606 m_sve_header_is_valid = false;
1607 m_sve_buffer_is_valid = false;
1608 m_sve_state = SVEState::Streaming;
1609 Status error = ReadSVEHeader();
1610
1611 // Streaming mode is active if the header has the SVE active flag set.
1612 if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1614 // Non-streaming might be active instead.
1615 m_sve_header_is_valid = false;
1616 m_sve_buffer_is_valid = false;
1617 m_sve_state = SVEState::Full;
1618 error = ReadSVEHeader();
1619 if (error.Success()) {
1620 // If SVE is enabled thread can switch between SVEState::FPSIMD and
1621 // SVEState::Full on every stop.
1622 if ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1624 m_sve_state = SVEState::FPSIMD;
1625 // Else we are in SVEState::Full.
1626 } else {
1627 m_sve_state = SVEState::Disabled;
1628 }
1629 }
1630
1631 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||
1632 m_sve_state == SVEState::Streaming) {
1633 // On every stop we configure SVE vector length by calling
1634 // ConfigureVectorLengthSVE regardless of current SVEState of this thread.
1636 if (sve::vl_valid(m_sve_header.vl))
1637 vq = sve::vq_from_vl(m_sve_header.vl);
1638
1639 GetRegisterInfo().ConfigureVectorLengthSVE(vq);
1640 m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));
1641 }
1642 }
1643
1644 if (!m_za_header_is_valid) {
1645 Status error = ReadZAHeader();
1646 if (error.Success()) {
1648 if (sve::vl_valid(m_za_header.vl))
1649 vq = sve::vq_from_vl(m_za_header.vl);
1650
1651 GetRegisterInfo().ConfigureVectorLengthZA(vq);
1652 m_za_ptrace_payload.resize(m_za_header.size);
1653 m_za_buffer_is_valid = false;
1654 }
1655 }
1656}
1657
1658uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
1659 const RegisterInfo *reg_info) const {
1660 return reg_info->byte_offset - GetGPRSize();
1661}
1662
1663uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(
1664 const RegisterInfo *reg_info) const {
1665 // Start of Z0 data is after GPRs plus 8 bytes of vg register
1666 uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
1667 if (m_sve_state == SVEState::FPSIMD) {
1668 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
1669 sve_reg_offset = sve::ptrace_fpsimd_offset +
1670 (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;
1671 // Between non-streaming and streaming mode, the layout is identical.
1672 } else if (m_sve_state == SVEState::Full ||
1673 m_sve_state == SVEState::Streaming) {
1674 uint32_t sve_z0_offset = GetGPRSize() + 16;
1675 sve_reg_offset =
1676 sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;
1677 }
1678 return sve_reg_offset;
1679}
1680
1681Status NativeRegisterContextLinux_arm64::ReadSMESVG() {
1682 // This register is the streaming vector length, so we will get it from
1683 // NT_ARM_ZA regardless of the current streaming mode.
1684 Status error = ReadZAHeader();
1685 if (error.Success())
1686 m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;
1687
1688 return error;
1689}
1690
1691std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
1692 ExpeditedRegs expType) const {
1693 std::vector<uint32_t> expedited_reg_nums =
1695 // SVE, non-streaming vector length.
1696 if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
1697 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
1698 // SME, streaming vector length. This is used by the ZA register which is
1699 // present even when streaming mode is not enabled.
1700 if (GetRegisterInfo().IsSSVEPresent())
1701 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());
1702
1703 return expedited_reg_nums;
1704}
1705
1706llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>
1707NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {
1709 return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),
1711 }
1712
1713 return llvm::createStringError(llvm::inconvertibleErrorCode(),
1714 "Unknown AArch64 memory tag type %d", type);
1715}
1716
1717lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(
1718 lldb::addr_t hit_addr) {
1719 // Linux configures user-space virtual addresses with top byte ignored.
1720 // We set default value of mask such that top byte is masked out.
1721 lldb::addr_t mask = ~((1ULL << 56) - 1);
1722
1723 // Try to read pointer authentication data_mask register and calculate a
1724 // consolidated data address mask after ignoring the top byte.
1725 if (ReadPAuthMask().Success())
1726 mask |= m_pac_mask.data_mask;
1727
1728 return hit_addr & ~mask;
1729 ;
1730}
1731
1732#endif // defined (__arm64__) || defined (__aarch64__)
#define GPR(r16)
Definition ABIX86.cpp:145
static llvm::raw_ostream & error(Stream &strm)
#define HWCAP2_MTE
#define PTRACE_PEEKMTETAGS
Definition Ptrace.h:54
#define PTRACE_POKEMTETAGS
Definition Ptrace.h:57
#define PTRACE_GETREGSET
Definition Ptrace.h:36
struct _FPR FPR
#define HWCAP2_FPMR
@ AUXV_AT_HWCAP2
Extension of AT_HWCAP.
Definition AuxVector.h:59
@ AUXV_AT_HWCAP3
Extension of AT_HWCAP.
Definition AuxVector.h:60
@ AUXV_AT_HWCAP
Machine dependent hints about processor capabilities.
Definition AuxVector.h:49
size_t GetRegisterSetCount() const override
This class manages the storage and detection of register field information.
void DetectFields(uint64_t hwcap, uint64_t hwcap2, uint64_t hwcap3)
For the registers listed in this class, detect which fields are present.
void UpdateRegisterInfo(const RegisterInfo *reg_info, uint32_t num_regs)
Add the field information of any registers named in this class, to the relevant RegisterInfo instance...
bool HasDetected() const
Returns true if field detection has been run at least once.
A subclass of DataBuffer that stores a data buffer on the heap.
std::optional< uint64_t > GetAuxValue(enum AuxVector::EntryType type)
virtual std::vector< uint32_t > GetExpeditedRegisters(ExpeditedRegs expType) const
uint32_t SetFromMemoryData(const RegisterInfo &reg_info, const void *src, uint32_t src_len, lldb::ByteOrder src_byte_order, Status &error)
uint64_t GetAsUInt64(uint64_t fail_value=UINT64_MAX, bool *success_ptr=nullptr) const
const void * GetBytes() const
static Status FromErrorStringWithFormat(const char *format,...) __attribute__((format(printf
Definition Status.cpp:106
static Status FromErrorString(const char *str)
Definition Status.h:141
Manages communication with the inferior (debugee) process.
static Status PtraceWrapper(int req, lldb::pid_t pid, void *addr=nullptr, void *data=nullptr, size_t data_size=0, long *result=nullptr)
}
static std::unique_ptr< NativeRegisterContextLinux > CreateHostNativeRegisterContextLinux(const ArchSpec &target_arch, NativeThreadLinux &native_thread)
static llvm::Expected< ArchSpec > DetermineArchitecture(lldb::tid_t tid)
#define LLDB_INVALID_INDEX32
#define LLDB_INVALID_REGNUM
Status WriteHardwareDebugRegs(int hwbType, ::pid_t tid, uint32_t max_supported, const std::array< NativeRegisterContextDBReg::DREG, 16 > &regs)
Status ReadHardwareDebugInfo(::pid_t tid, uint32_t &max_hwp_supported, uint32_t &max_hbp_supported)
uint16_t vq_from_vl(uint16_t vl)
uint32_t PTraceFPSROffset(uint16_t vq)
uint32_t PTraceFPCROffset(uint16_t vq)
uint16_t vl_valid(uint16_t vl)
uint32_t PTraceSize(uint16_t vq, uint16_t flags)
A class that represents a running process on the host machine.
uint64_t pid_t
Definition lldb-types.h:83
std::shared_ptr< lldb_private::DataBuffer > DataBufferSP
std::shared_ptr< lldb_private::WritableDataBuffer > WritableDataBufferSP
uint64_t addr_t
Definition lldb-types.h:80
uint64_t tid_t
Definition lldb-types.h:84
@ eRegisterKindLLDB
lldb's internal register numbers
Every register is described in detail including its name, alternate name (optional),...
uint32_t * value_regs
List of registers (terminated with LLDB_INVALID_REGNUM).
uint32_t byte_offset
The byte offset in the register context data where this register's value is found.
uint32_t byte_size
Size in bytes of the register.
uint32_t kinds[lldb::kNumRegisterKinds]
Holds all of the various register numbers for all register kinds.
const char * name
Name of this register, can't be NULL.
Registers are grouped into register sets.
size_t num_registers
The number of registers in REGISTERS array below.