caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[libc] major refactor of startup library (#76092)

Browse Source 
* separate initialization routines into _start and do_start for all
architectures.
* lift do_start as a separate object library to avoid code duplication.
* (addtionally) address the problem of building hermetic libc with
-fstack-pointer-*

The `crt1.o` is now a merged result of three components:

```
___
  |___ x86_64
  |      |_______ start.cpp.o    <- _start (loads process initial stack and aligns stack pointer)
  |      |_______ tls.cpp.o      <- init_tls, cleanup_tls, set_thread_pointer (TLS related routines) 
  |___ do_start.cpp.o            <- do_start (sets up global variables and invokes the main function) 
```
This commit is contained in:
Schrodinger ZHU Yifan
2024-01-04 15:51:14 -05:00
committed by GitHub
parent d67c2d8554
commit 2bc994456c
14 changed files with 510 additions and 669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
libc/cmake/modules/LLVMLibCTestRules.cmake
View File

@@ -498,6 +498,9 @@ function(add_integration_test test_name)
libc.src.string.memcpy
libc.src.string.memmove
libc.src.string.memset
# __stack_chk_fail should always be included to allow building libc with
# stack protector.
libc.src.compiler.__stack_chk_fail
)
list(REMOVE_DUPLICATES fq_deps_list)
@@ -665,6 +668,9 @@ function(add_libc_hermetic_test test_name)
libc.src.string.memmove
libc.src.string.memset
libc.src.__support.StringUtil.error_to_string
# __stack_chk_fail should always be included to allow building libc with
# stack protector.
libc.src.compiler.__stack_chk_fail
)
if(TARGET libc.src.time.clock)

3
libc/config/linux/app.h
View File

@@ -119,6 +119,9 @@ void init_tls(TLSDescriptor &tls);
// Cleanup the TLS area as described in |tls_descriptor|.
void cleanup_tls(uintptr_t tls_addr, uintptr_t tls_size);
// Set the thread pointer for the current thread.
bool set_thread_ptr(uintptr_t val);
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_CONFIG_LINUX_APP_H

25
libc/startup/linux/CMakeLists.txt
View File

@@ -84,10 +84,33 @@ endif()
add_subdirectory(${LIBC_TARGET_ARCHITECTURE})
add_object_library(
do_start
SRCS
do_start.cpp
HDRS
do_start.h
DEPENDS
libc.config.linux.app_h
libc.include.sys_mman
libc.include.sys_syscall
libc.src.__support.threads.thread
libc.src.__support.OSUtil.osutil
libc.src.stdlib.exit
libc.src.stdlib.atexit
libc.src.unistd.environ
COMPILE_OPTIONS
-ffreestanding # To avoid compiler warnings about calling the main function.
-fno-builtin # avoid emit unexpected calls
-fno-stack-protector # stack protect canary is not available yet.
)
# TODO: factor out crt1 into multiple objects
merge_relocatable_object(
crt1
.${LIBC_TARGET_ARCHITECTURE}.crt1
.${LIBC_TARGET_ARCHITECTURE}.start
.${LIBC_TARGET_ARCHITECTURE}.tls
.do_start
)
add_startup_object(

19
libc/startup/linux/aarch64/CMakeLists.txt
View File

@@ -1,17 +1,24 @@
add_startup_object(
crt1
tls
SRC
start.cpp
tls.cpp
DEPENDS
libc.config.linux.app_h
libc.include.sys_mman
libc.include.sys_syscall
libc.src.__support.threads.thread
libc.src.__support.OSUtil.osutil
libc.src.stdlib.exit
libc.src.stdlib.atexit
libc.src.string.memory_utils.inline_memcpy
libc.src.unistd.environ
COMPILE_OPTIONS
-fno-omit-frame-pointer
-ffreestanding # To avoid compiler warnings about calling the main function.
)
add_startup_object(
start
SRC
start.cpp
DEPENDS
libc.config.linux.app_h
COMPILE_OPTIONS
-fno-omit-frame-pointer
-ffreestanding # To avoid compiler warnings about calling the main function.

215
libc/startup/linux/aarch64/start.cpp
View File

@@ -1,4 +1,4 @@
//===-- Implementation of crt for aarch64 ---------------------------------===//
//===-- Implementation of _start for aarch64 ------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,213 +6,8 @@
//
//===----------------------------------------------------------------------===//
#include "config/linux/app.h"
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/stdlib/atexit.h"
#include "src/stdlib/exit.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include <arm_acle.h>
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
extern "C" int main(int, char **, char **);
// Source documentation:
// https://github.com/ARM-software/abi-aa/tree/main/sysvabi64
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
AppProperties app;
static ThreadAttributes main_thread_attrib;
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// aarch64 follows the variant 1 TLS layout:
//
// 1. First entry is the dynamic thread vector pointer
// 2. Second entry is a 8-byte reserved word.
// 3. Padding for alignment.
// 4. The TLS data from the ELF image.
//
// The thread pointer points to the first entry.
const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
uintptr_t padding = 0;
const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
if (diff != 0)
padding += (ALIGNMENT_MASK - diff) + 1;
uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_ret_val = LIBC_NAMESPACE::syscall_impl<long>(
MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.page_size)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
LIBC_NAMESPACE::inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
tls_descriptor.size = alloc_size;
tls_descriptor.addr = thread_ptr;
tls_descriptor.tp = thread_ptr;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
LIBC_NAMESPACE::syscall_impl<long>(SYS_munmap, addr, size);
}
static void set_thread_ptr(uintptr_t val) { __arm_wsr64("tpidr_el0", val); }
using InitCallback = void(int, char **, char **);
using FiniCallback = void(void);
extern "C" {
// These arrays are present in the .init_array and .fini_array sections.
// The symbols are inserted by linker when it sees references to them.
extern uintptr_t __preinit_array_start[];
extern uintptr_t __preinit_array_end[];
extern uintptr_t __init_array_start[];
extern uintptr_t __init_array_end[];
extern uintptr_t __fini_array_start[];
extern uintptr_t __fini_array_end[];
}
static void call_init_array_callbacks(int argc, char **argv, char **env) {
size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
for (size_t i = 0; i < preinit_array_size; ++i)
reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
size_t init_array_size = __init_array_end - __init_array_start;
for (size_t i = 0; i < init_array_size; ++i)
reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
}
static void call_fini_array_callbacks() {
size_t fini_array_size = __fini_array_end - __fini_array_start;
for (size_t i = fini_array_size; i > 0; --i)
reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
}
} // namespace LIBC_NAMESPACE
using LIBC_NAMESPACE::app;
using LIBC_NAMESPACE::AuxEntry;
__attribute__((noinline)) static void do_start() {
auto tid = LIBC_NAMESPACE::syscall_impl<long>(SYS_gettid);
if (tid <= 0)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
LIBC_NAMESPACE::main_thread_attrib.tid = static_cast<int>(tid);
// After the argv array, is a 8-byte long NULL value before the array of env
// values. The end of the env values is marked by another 8-byte long NULL
// value. We step over it (the "+ 1" below) to get to the env values.
uint64_t *env_ptr = app.args->argv + app.args->argc + 1;
uint64_t *env_end_marker = env_ptr;
app.env_ptr = env_ptr;
while (*env_end_marker)
++env_end_marker;
// Initialize the POSIX global declared in unistd.h
environ = reinterpret_cast<char **>(env_ptr);
// After the env array, is the aux-vector. The end of the aux-vector is
// denoted by an AT_NULL entry.
Elf64_Phdr *program_hdr_table = nullptr;
uintptr_t program_hdr_count;
app.auxv_ptr = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) {
switch (aux_entry->id) {
case AT_PHDR:
program_hdr_table = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
break;
case AT_PHNUM:
program_hdr_count = aux_entry->value;
break;
case AT_PAGESZ:
app.page_size = aux_entry->value;
break;
default:
break; // TODO: Read other useful entries from the aux vector.
}
}
app.tls.size = 0;
for (uintptr_t i = 0; i < program_hdr_count; ++i) {
Elf64_Phdr *phdr = program_hdr_table + i;
if (phdr->p_type != PT_TLS)
continue;
// TODO: p_vaddr value has to be adjusted for static-pie executables.
app.tls.address = phdr->p_vaddr;
app.tls.size = phdr->p_memsz;
app.tls.init_size = phdr->p_filesz;
app.tls.align = phdr->p_align;
}
// This descriptor has to be static since its cleanup function cannot
// capture the context.
static LIBC_NAMESPACE::TLSDescriptor tls;
LIBC_NAMESPACE::init_tls(tls);
if (tls.size != 0)
LIBC_NAMESPACE::set_thread_ptr(tls.tp);
LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib;
LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr =
LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr();
// We register the cleanup_tls function to be the last atexit callback to be
// invoked. It will tear down the TLS. Other callbacks may depend on TLS (such
// as the stack protector canary).
LIBC_NAMESPACE::atexit(
[]() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); });
// We want the fini array callbacks to be run after other atexit
// callbacks are run. So, we register them before running the init
// array callbacks as they can potentially register their own atexit
// callbacks.
LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks);
LIBC_NAMESPACE::call_init_array_callbacks(
static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
int retval = main(static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
LIBC_NAMESPACE::exit(retval);
}
extern "C" void _start() {
#include "startup/linux/do_start.h"
extern "C" [[noreturn]] void _start() {
// Skip the Frame Pointer and the Link Register
// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst
// Section 6.2.3. Note that this only works if the current function
@@ -223,7 +18,7 @@ extern "C" void _start() {
// will take us to the previous stack pointer. That is the reason why the
// actual business logic of the startup code is pushed into a non-inline
// function do_start so that this function is free of any stack usage.
app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
LIBC_NAMESPACE::app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
reinterpret_cast<uintptr_t *>(__builtin_frame_address(0)) + 2);
do_start();
LIBC_NAMESPACE::do_start();
}

86
libc/startup/linux/aarch64/tls.cpp Normal file
View File

@@ -0,0 +1,86 @@
//===-- Implementation of tls for aarch64 ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "startup/linux/do_start.h"
#include <arm_acle.h>
#include <sys/mman.h>
#include <sys/syscall.h>
// Source documentation:
// https://github.com/ARM-software/abi-aa/tree/main/sysvabi64
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// aarch64 follows the variant 1 TLS layout:
//
// 1. First entry is the dynamic thread vector pointer
// 2. Second entry is a 8-byte reserved word.
// 3. Padding for alignment.
// 4. The TLS data from the ELF image.
//
// The thread pointer points to the first entry.
const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
uintptr_t padding = 0;
const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
if (diff != 0)
padding += (ALIGNMENT_MASK - diff) + 1;
uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_ret_val = syscall_impl<long>(MMAP_SYSCALL_NUMBER, nullptr,
alloc_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.page_size)
syscall_impl<long>(SYS_exit, 1);
uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
tls_descriptor.size = alloc_size;
tls_descriptor.addr = thread_ptr;
tls_descriptor.tp = thread_ptr;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
syscall_impl<long>(SYS_munmap, addr, size);
}
bool set_thread_ptr(uintptr_t val) {
__arm_wsr64("tpidr_el0", val);
return true;
}
} // namespace LIBC_NAMESPACE

140
libc/startup/linux/do_start.cpp Normal file
View File

@@ -0,0 +1,140 @@
//===-- Implementation file of do_start -----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "startup/linux/do_start.h"
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/stdlib/atexit.h"
#include "src/stdlib/exit.h"
#include "src/unistd/environ.h"
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/syscall.h>
extern "C" int main(int argc, char **argv, char **envp);
extern "C" {
// These arrays are present in the .init_array and .fini_array sections.
// The symbols are inserted by linker when it sees references to them.
extern uintptr_t __preinit_array_start[];
extern uintptr_t __preinit_array_end[];
extern uintptr_t __init_array_start[];
extern uintptr_t __init_array_end[];
extern uintptr_t __fini_array_start[];
extern uintptr_t __fini_array_end[];
}
namespace LIBC_NAMESPACE {
// TODO: this symbol will be moved to config.linux.app
AppProperties app;
using InitCallback = void(int, char **, char **);
using FiniCallback = void(void);
static void call_init_array_callbacks(int argc, char **argv, char **env) {
size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
for (size_t i = 0; i < preinit_array_size; ++i)
reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
size_t init_array_size = __init_array_end - __init_array_start;
for (size_t i = 0; i < init_array_size; ++i)
reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
}
static void call_fini_array_callbacks() {
size_t fini_array_size = __fini_array_end - __fini_array_start;
for (size_t i = fini_array_size; i > 0; --i)
reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
}
static ThreadAttributes main_thread_attrib;
[[noreturn]] void do_start() {
auto tid = syscall_impl<long>(SYS_gettid);
if (tid <= 0)
syscall_impl<long>(SYS_exit, 1);
main_thread_attrib.tid = static_cast<int>(tid);
// After the argv array, is a 8-byte long NULL value before the array of env
// values. The end of the env values is marked by another 8-byte long NULL
// value. We step over it (the "+ 1" below) to get to the env values.
ArgVEntryType *env_ptr = app.args->argv + app.args->argc + 1;
ArgVEntryType *env_end_marker = env_ptr;
app.env_ptr = env_ptr;
while (*env_end_marker)
++env_end_marker;
// Initialize the POSIX global declared in unistd.h
environ = reinterpret_cast<char **>(env_ptr);
// After the env array, is the aux-vector. The end of the aux-vector is
// denoted by an AT_NULL entry.
Elf64_Phdr *program_hdr_table = nullptr;
uintptr_t program_hdr_count = 0;
app.auxv_ptr = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) {
switch (aux_entry->id) {
case AT_PHDR:
program_hdr_table = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
break;
case AT_PHNUM:
program_hdr_count = aux_entry->value;
break;
case AT_PAGESZ:
app.page_size = aux_entry->value;
break;
default:
break; // TODO: Read other useful entries from the aux vector.
}
}
app.tls.size = 0;
for (uintptr_t i = 0; i < program_hdr_count; ++i) {
Elf64_Phdr *phdr = program_hdr_table + i;
if (phdr->p_type != PT_TLS)
continue;
// TODO: p_vaddr value has to be adjusted for static-pie executables.
app.tls.address = phdr->p_vaddr;
app.tls.size = phdr->p_memsz;
app.tls.init_size = phdr->p_filesz;
app.tls.align = phdr->p_align;
}
// This descriptor has to be static since its cleanup function cannot
// capture the context.
static TLSDescriptor tls;
init_tls(tls);
if (tls.size != 0 && !set_thread_ptr(tls.tp))
syscall_impl<long>(SYS_exit, 1);
self.attrib = &main_thread_attrib;
main_thread_attrib.atexit_callback_mgr =
internal::get_thread_atexit_callback_mgr();
// We register the cleanup_tls function to be the last atexit callback to be
// invoked. It will tear down the TLS. Other callbacks may depend on TLS (such
// as the stack protector canary).
atexit([]() { cleanup_tls(tls.tp, tls.size); });
// We want the fini array callbacks to be run after other atexit
// callbacks are run. So, we register them before running the init
// array callbacks as they can potentially register their own atexit
// callbacks.
atexit(&call_fini_array_callbacks);
call_init_array_callbacks(static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
int retval = main(static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
exit(retval);
}
} // namespace LIBC_NAMESPACE

14
libc/startup/linux/do_start.h Normal file
View File

@@ -0,0 +1,14 @@
//===-- Header file of do_start -------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "config/linux/app.h"
namespace LIBC_NAMESPACE {
// setup the libc runtime and invoke the main routine.
[[noreturn]] void do_start();
} // namespace LIBC_NAMESPACE

20
libc/startup/linux/riscv/CMakeLists.txt
View File

@@ -1,17 +1,25 @@
add_startup_object(
crt1
tls
SRC
start.cpp
tls.cpp
DEPENDS
libc.config.linux.app_h
libc.include.sys_mman
libc.include.sys_syscall
libc.src.__support.threads.thread
libc.src.__support.OSUtil.osutil
libc.src.stdlib.exit
libc.src.stdlib.atexit
libc.src.string.memory_utils.inline_memcpy
libc.src.unistd.environ
COMPILE_OPTIONS
-fno-omit-frame-pointer
-ffreestanding # To avoid compiler warnings about calling the main function.
)
add_startup_object(
start
SRC
start.cpp
DEPENDS
libc.config.linux.app_h
libc.src.__support.macros.attributes
COMPILE_OPTIONS
-fno-omit-frame-pointer
-ffreestanding # To avoid compiler warnings about calling the main function.

223
libc/startup/linux/riscv/start.cpp
View File

@@ -1,223 +1,20 @@
//===-- Implementation of crt for riscv64 ---------------------------------===//
//===-- Implementation of _start for riscv --------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/__support/macros/attributes.h"
#include "startup/linux/do_start.h"
#include "config/linux/app.h"
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/stdlib/atexit.h"
#include "src/stdlib/exit.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
extern "C" int main(int, char **, char **);
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
AppProperties app;
static ThreadAttributes main_thread_attrib;
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// riscv64 follows the variant 1 TLS layout:
const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
uintptr_t padding = 0;
const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
if (diff != 0)
padding += (ALIGNMENT_MASK - diff) + 1;
uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_ret_val = LIBC_NAMESPACE::syscall_impl<long>(
MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.page_size)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
LIBC_NAMESPACE::inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
tls_descriptor.size = alloc_size;
tls_descriptor.addr = thread_ptr;
tls_descriptor.tp = tls_addr;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
LIBC_NAMESPACE::syscall_impl<long>(SYS_munmap, addr, size);
}
static void set_thread_ptr(uintptr_t val) {
LIBC_INLINE_ASM("mv tp, %0\n\t" : : "r"(val));
}
using InitCallback = void(int, char **, char **);
using FiniCallback = void(void);
extern "C" {
// These arrays are present in the .init_array and .fini_array sections.
// The symbols are inserted by linker when it sees references to them.
extern uintptr_t __preinit_array_start[];
extern uintptr_t __preinit_array_end[];
extern uintptr_t __init_array_start[];
extern uintptr_t __init_array_end[];
extern uintptr_t __fini_array_start[];
extern uintptr_t __fini_array_end[];
}
static void call_init_array_callbacks(int argc, char **argv, char **env) {
size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
for (size_t i = 0; i < preinit_array_size; ++i)
reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
size_t init_array_size = __init_array_end - __init_array_start;
for (size_t i = 0; i < init_array_size; ++i)
reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
}
static void call_fini_array_callbacks() {
size_t fini_array_size = __fini_array_end - __fini_array_start;
for (size_t i = fini_array_size; i > 0; --i)
reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
}
} // namespace LIBC_NAMESPACE
using LIBC_NAMESPACE::app;
using LIBC_NAMESPACE::AuxEntry;
#if defined(LIBC_TARGET_ARCH_IS_X86_64) || \
defined(LIBC_TARGET_ARCH_IS_AARCH64) || \
defined(LIBC_TARGET_ARCH_IS_RISCV64)
typedef Elf64_Phdr PgrHdrTableType;
#elif defined(LIBC_TARGET_ARCH_IS_RISCV32)
typedef Elf32_Phdr PgrHdrTableType;
#else
#error "Program header table type is not defined for the target platform."
#endif
__attribute__((noinline)) static void do_start() {
LIBC_INLINE_ASM(".option push\n\t"
".option norelax\n\t"
"lla gp, __global_pointer$\n\t"
".option pop\n\t");
auto tid = LIBC_NAMESPACE::syscall_impl<long>(SYS_gettid);
if (tid <= 0)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
LIBC_NAMESPACE::main_thread_attrib.tid = static_cast<int>(tid);
// After the argv array, is a 8-byte long NULL value before the array of env
// values. The end of the env values is marked by another 8-byte long NULL
// value. We step over it (the "+ 1" below) to get to the env values.
LIBC_NAMESPACE::ArgVEntryType *env_ptr = app.args->argv + app.args->argc + 1;
LIBC_NAMESPACE::ArgVEntryType *env_end_marker = env_ptr;
app.env_ptr = env_ptr;
while (*env_end_marker)
++env_end_marker;
// Initialize the POSIX global declared in unistd.h
environ = reinterpret_cast<char **>(env_ptr);
// After the env array, is the aux-vector. The end of the aux-vector is
// denoted by an AT_NULL entry.
PgrHdrTableType *program_hdr_table = nullptr;
uintptr_t program_hdr_count;
app.auxv_ptr = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) {
switch (aux_entry->id) {
case AT_PHDR:
program_hdr_table = reinterpret_cast<PgrHdrTableType *>(aux_entry->value);
break;
case AT_PHNUM:
program_hdr_count = aux_entry->value;
break;
case AT_PAGESZ:
app.page_size = aux_entry->value;
break;
default:
break; // TODO: Read other useful entries from the aux vector.
}
}
app.tls.size = 0;
for (uintptr_t i = 0; i < program_hdr_count; ++i) {
PgrHdrTableType *phdr = program_hdr_table + i;
if (phdr->p_type != PT_TLS)
continue;
// TODO: p_vaddr value has to be adjusted for static-pie executables.
app.tls.address = phdr->p_vaddr;
app.tls.size = phdr->p_memsz;
app.tls.init_size = phdr->p_filesz;
app.tls.align = phdr->p_align;
}
// This descriptor has to be static since its cleanup function cannot
// capture the context.
static LIBC_NAMESPACE::TLSDescriptor tls;
LIBC_NAMESPACE::init_tls(tls);
if (tls.size != 0)
LIBC_NAMESPACE::set_thread_ptr(tls.tp);
LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib;
LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr =
LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr();
// We register the cleanup_tls function to be the last atexit callback to be
// invoked. It will tear down the TLS. Other callbacks may depend on TLS (such
// as the stack protector canary).
LIBC_NAMESPACE::atexit(
[]() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); });
// We want the fini array callbacks to be run after other atexit
// callbacks are run. So, we register them before running the init
// array callbacks as they can potentially register their own atexit
// callbacks.
LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks);
LIBC_NAMESPACE::call_init_array_callbacks(
static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
int retval = main(static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
LIBC_NAMESPACE::exit(retval);
}
extern "C" void _start() {
extern "C" [[noreturn]] void _start() {
asm volatile(".option push\n\t"
".option norelax\n\t"
"lla gp, __global_pointer$\n\t"
".option pop\n\t");
// Fetch the args using the frame pointer.
app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
LIBC_NAMESPACE::app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
reinterpret_cast<uintptr_t *>(__builtin_frame_address(0)));
do_start();
LIBC_NAMESPACE::do_start();
}

74
libc/startup/linux/riscv/tls.cpp Normal file
View File

@@ -0,0 +1,74 @@
//===-- Implementation of tls for riscv -----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "startup/linux/do_start.h"
#include <sys/mman.h>
#include <sys/syscall.h>
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// riscv64 follows the variant 1 TLS layout:
const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
uintptr_t padding = 0;
const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
if (diff != 0)
padding += (ALIGNMENT_MASK - diff) + 1;
uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_ret_val = syscall_impl<long>(MMAP_SYSCALL_NUMBER, nullptr,
alloc_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.page_size)
syscall_impl<long>(SYS_exit, 1);
uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
tls_descriptor.size = alloc_size;
tls_descriptor.addr = thread_ptr;
tls_descriptor.tp = tls_addr;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
syscall_impl<long>(SYS_munmap, addr, size);
}
bool set_thread_ptr(uintptr_t val) {
LIBC_INLINE_ASM("mv tp, %0\n\t" : : "r"(val));
return true;
}
} // namespace LIBC_NAMESPACE

26
libc/startup/linux/x86_64/CMakeLists.txt
View File

@@ -1,22 +1,30 @@
add_startup_object(
crt1
tls
SRC
start.cpp
tls.cpp
DEPENDS
libc.config.linux.app_h
libc.include.sys_mman
libc.include.sys_syscall
libc.include.unistd
libc.src.__support.threads.thread
libc.src.__support.OSUtil.osutil
libc.src.stdlib.exit
libc.src.stdlib.abort
libc.src.stdlib.atexit
libc.src.string.memory_utils.inline_memcpy
libc.src.unistd.environ
COMPILE_OPTIONS
-fno-stack-protector
-fno-omit-frame-pointer
-ffreestanding # To avoid compiler warnings about calling the main function.
-ffreestanding
-fno-builtin
)
add_startup_object(
start
SRC
start.cpp
DEPENDS
libc.config.linux.app_h
libc.src.__support.macros.attributes
COMPILE_OPTIONS
-fno-stack-protector
-fno-omit-frame-pointer
-ffreestanding
-fno-builtin
)

235
libc/startup/linux/x86_64/start.cpp
View File

@@ -1,151 +1,18 @@
//===-- Implementation of crt for x86_64 ----------------------------------===//
//===-- Implementation of _start for x86_64 -------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/__support/macros/attributes.h"
#include "startup/linux/do_start.h"
#include "config/linux/app.h"
#include "src/__support/OSUtil/io.h"
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/thread.h"
#include "src/stdlib/abort.h"
#include "src/stdlib/atexit.h"
#include "src/stdlib/exit.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include <asm/prctl.h>
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
extern "C" int main(int, char **, char **);
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
AppProperties app;
static ThreadAttributes main_thread_attrib;
// TODO: The function is x86_64 specific. Move it to config/linux/app.h
// and generalize it. Also, dynamic loading is not handled currently.
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// We will assume the alignment is always a power of two.
uintptr_t tls_size = app.tls.size & -app.tls.align;
if (tls_size != app.tls.size)
tls_size += app.tls.align;
// Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the
// address of the TLS block. So, we add more size to accomodate this address
// entry.
// We also need to include space for the stack canary. The canary is at
// offset 0x28 (40) and is of size uintptr_t.
uintptr_t tls_size_with_addr = tls_size + sizeof(uintptr_t) + 40;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_retval = LIBC_NAMESPACE::syscall_impl<long>(
MMAP_SYSCALL_NUMBER, nullptr, tls_size_with_addr, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_retval < 0 && static_cast<uintptr_t>(mmap_retval) > -app.page_size)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
uintptr_t *tls_addr = reinterpret_cast<uintptr_t *>(mmap_retval);
// x86_64 TLS faces down from the thread pointer with the first entry
// pointing to the address of the first real TLS byte.
uintptr_t end_ptr = reinterpret_cast<uintptr_t>(tls_addr) + tls_size;
*reinterpret_cast<uintptr_t *>(end_ptr) = end_ptr;
LIBC_NAMESPACE::inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
uintptr_t *stack_guard_addr = reinterpret_cast<uintptr_t *>(end_ptr + 40);
// Setting the stack guard to a random value.
// We cannot call the get_random function here as the function sets errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
ssize_t stack_guard_retval = LIBC_NAMESPACE::syscall_impl<ssize_t>(
SYS_getrandom, reinterpret_cast<long>(stack_guard_addr), sizeof(uint64_t),
0);
if (stack_guard_retval < 0)
LIBC_NAMESPACE::syscall_impl(SYS_exit, 1);
tls_descriptor = {tls_size_with_addr, reinterpret_cast<uintptr_t>(tls_addr),
end_ptr};
return;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
LIBC_NAMESPACE::syscall_impl<long>(SYS_munmap, addr, size);
}
// Sets the thread pointer to |val|. Returns true on success, false on failure.
static bool set_thread_ptr(uintptr_t val) {
return LIBC_NAMESPACE::syscall_impl(SYS_arch_prctl, ARCH_SET_FS, val) != -1;
}
using InitCallback = void(int, char **, char **);
using FiniCallback = void(void);
extern "C" {
// These arrays are present in the .init_array and .fini_array sections.
// The symbols are inserted by linker when it sees references to them.
extern uintptr_t __preinit_array_start[];
extern uintptr_t __preinit_array_end[];
extern uintptr_t __init_array_start[];
extern uintptr_t __init_array_end[];
extern uintptr_t __fini_array_start[];
extern uintptr_t __fini_array_end[];
}
static void call_init_array_callbacks(int argc, char **argv, char **env) {
size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
for (size_t i = 0; i < preinit_array_size; ++i)
reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
size_t init_array_size = __init_array_end - __init_array_start;
for (size_t i = 0; i < init_array_size; ++i)
reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
}
static void call_fini_array_callbacks() {
size_t fini_array_size = __fini_array_end - __fini_array_start;
for (size_t i = fini_array_size; i > 0; --i)
reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
}
} // namespace LIBC_NAMESPACE
using LIBC_NAMESPACE::app;
using LIBC_NAMESPACE::AuxEntry;
extern "C" void _start() {
// This TU is compiled with -fno-omit-frame-pointer. Hence, the previous value
// of the base pointer is pushed on to the stack. So, we step over it (the
// "+ 1" below) to get to the args.
app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
extern "C" [[noreturn]] void _start() {
// This TU is compiled with -fno-omit-frame-pointer. Hence, the previous
// value of the base pointer is pushed on to the stack. So, we step over
// it (the "+ 1" below) to get to the args.
LIBC_NAMESPACE::app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
reinterpret_cast<uintptr_t *>(__builtin_frame_address(0)) + 1);
// The x86_64 ABI requires that the stack pointer is aligned to a 16-byte
@@ -159,88 +26,8 @@ extern "C" void _start() {
// compilers can generate code assuming the alignment as required by the ABI.
// If the stack pointers as setup by the OS are already aligned, then the
// following code is a NOP.
__asm__ __volatile__("andq $0xfffffffffffffff0, %rsp\n\t");
__asm__ __volatile__("andq $0xfffffffffffffff0, %rbp\n\t");
asm volatile("andq $0xfffffffffffffff0, %rsp\n\t");
asm volatile("andq $0xfffffffffffffff0, %rbp\n\t");
auto tid = LIBC_NAMESPACE::syscall_impl<long>(SYS_gettid);
if (tid <= 0)
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
LIBC_NAMESPACE::main_thread_attrib.tid = static_cast<int>(tid);
// After the argv array, is a 8-byte long NULL value before the array of env
// values. The end of the env values is marked by another 8-byte long NULL
// value. We step over it (the "+ 1" below) to get to the env values.
uint64_t *env_ptr = app.args->argv + app.args->argc + 1;
uint64_t *env_end_marker = env_ptr;
app.env_ptr = env_ptr;
while (*env_end_marker)
++env_end_marker;
// Initialize the POSIX global declared in unistd.h
environ = reinterpret_cast<char **>(env_ptr);
// After the env array, is the aux-vector. The end of the aux-vector is
// denoted by an AT_NULL entry.
Elf64_Phdr *program_hdr_table = nullptr;
uintptr_t program_hdr_count = 0;
app.auxv_ptr = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) {
switch (aux_entry->id) {
case AT_PHDR:
program_hdr_table = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
break;
case AT_PHNUM:
program_hdr_count = aux_entry->value;
break;
case AT_PAGESZ:
app.page_size = aux_entry->value;
break;
default:
break; // TODO: Read other useful entries from the aux vector.
}
}
app.tls.size = 0;
for (uintptr_t i = 0; i < program_hdr_count; ++i) {
Elf64_Phdr *phdr = program_hdr_table + i;
if (phdr->p_type != PT_TLS)
continue;
// TODO: p_vaddr value has to be adjusted for static-pie executables.
app.tls.address = phdr->p_vaddr;
app.tls.size = phdr->p_memsz;
app.tls.init_size = phdr->p_filesz;
app.tls.align = phdr->p_align;
}
// This descriptor has to be static since its cleanup function cannot
// capture the context.
static LIBC_NAMESPACE::TLSDescriptor tls;
LIBC_NAMESPACE::init_tls(tls);
if (tls.size != 0 && !LIBC_NAMESPACE::set_thread_ptr(tls.tp))
LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib;
LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr =
LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr();
// We register the cleanup_tls function to be the last atexit callback to be
// invoked. It will tear down the TLS. Other callbacks may depend on TLS (such
// as the stack protector canary).
LIBC_NAMESPACE::atexit(
[]() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); });
// We want the fini array callbacks to be run after other atexit
// callbacks are run. So, we register them before running the init
// array callbacks as they can potentially register their own atexit
// callbacks.
LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks);
LIBC_NAMESPACE::call_init_array_callbacks(
static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
int retval = main(static_cast<int>(app.args->argc),
reinterpret_cast<char **>(app.args->argv),
reinterpret_cast<char **>(env_ptr));
LIBC_NAMESPACE::exit(retval);
LIBC_NAMESPACE::do_start();
}

93
libc/startup/linux/x86_64/tls.cpp Normal file
View File

@@ -0,0 +1,93 @@
//===-- Implementation of tls for x86_64 ----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/__support/OSUtil/syscall.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "startup/linux/do_start.h"
#include <asm/prctl.h>
#include <sys/mman.h>
#include <sys/syscall.h>
namespace LIBC_NAMESPACE {
#ifdef SYS_mmap2
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
#elif SYS_mmap
static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
#else
#error "mmap and mmap2 syscalls not available."
#endif
// TODO: Also generalize this routine and handle dynamic loading properly.
void init_tls(TLSDescriptor &tls_descriptor) {
if (app.tls.size == 0) {
tls_descriptor.size = 0;
tls_descriptor.tp = 0;
return;
}
// We will assume the alignment is always a power of two.
uintptr_t tls_size = app.tls.size & -app.tls.align;
if (tls_size != app.tls.size)
tls_size += app.tls.align;
// Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the
// address of the TLS block. So, we add more size to accomodate this address
// entry.
// We also need to include space for the stack canary. The canary is at
// offset 0x28 (40) and is of size uintptr_t.
uintptr_t tls_size_with_addr = tls_size + sizeof(uintptr_t) + 40;
// We cannot call the mmap function here as the functions set errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long mmap_retval = syscall_impl<long>(
MMAP_SYSCALL_NUMBER, nullptr, tls_size_with_addr, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
// We cannot check the return value with MAP_FAILED as that is the return
// of the mmap function and not the mmap syscall.
if (mmap_retval < 0 && static_cast<uintptr_t>(mmap_retval) > -app.page_size)
syscall_impl<long>(SYS_exit, 1);
uintptr_t *tls_addr = reinterpret_cast<uintptr_t *>(mmap_retval);
// x86_64 TLS faces down from the thread pointer with the first entry
// pointing to the address of the first real TLS byte.
uintptr_t end_ptr = reinterpret_cast<uintptr_t>(tls_addr) + tls_size;
*reinterpret_cast<uintptr_t *>(end_ptr) = end_ptr;
inline_memcpy(reinterpret_cast<char *>(tls_addr),
reinterpret_cast<const char *>(app.tls.address),
app.tls.init_size);
uintptr_t *stack_guard_addr = reinterpret_cast<uintptr_t *>(end_ptr + 40);
// Setting the stack guard to a random value.
// We cannot call the get_random function here as the function sets errno on
// failure. Since errno is implemented via a thread local variable, we cannot
// use errno before TLS is setup.
long stack_guard_retval =
syscall_impl(SYS_getrandom, reinterpret_cast<long>(stack_guard_addr),
sizeof(uint64_t), 0);
if (stack_guard_retval < 0)
syscall_impl(SYS_exit, 1);
tls_descriptor = {tls_size_with_addr, reinterpret_cast<uintptr_t>(tls_addr),
end_ptr};
return;
}
void cleanup_tls(uintptr_t addr, uintptr_t size) {
if (size == 0)
return;
syscall_impl<long>(SYS_munmap, addr, size);
}
// Sets the thread pointer to |val|. Returns true on success, false on failure.
bool set_thread_ptr(uintptr_t val) {
return syscall_impl(SYS_arch_prctl, ARCH_SET_FS, val) != -1;
}
} // namespace LIBC_NAMESPACE