From cf0613e2b71a63794c7d6b8c03d8232cc668accb Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 6 Jun 2024 14:41:02 +0200 Subject: [PATCH 1/6] bootstrap: Fix runner tags See https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher --- .gitlab-ci-main.yml | 2 +- bootstrap/.gitlab-ci.yml | 4 +--- bootstrap/sha1sum.txt | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml index 26cf07d6fce..3da27e86e2f 100644 --- a/.gitlab-ci-main.yml +++ b/.gitlab-ci-main.yml @@ -47,7 +47,7 @@ variables: # Set this to the contents of bootstrap/sha1sum.txt # which is generated by bootstrap/template.py --render # - SAMBA_CI_CONTAINER_TAG: 9a406973474a7903fe7fd6215226660911ed73c0 + SAMBA_CI_CONTAINER_TAG: 2d7eba00f6919d62a37ccc31daab18d07c761c6a # # We use the ubuntu2204 image as default as # it matches what we have on atb-devel-224 diff --git a/bootstrap/.gitlab-ci.yml b/bootstrap/.gitlab-ci.yml index ba82cdc1251..4de8c23afb4 100644 --- a/bootstrap/.gitlab-ci.yml +++ b/bootstrap/.gitlab-ci.yml @@ -6,9 +6,7 @@ # We need to make sure we only use gitlab.com # runners and not our own runners, as our current runners # don't allow 'docker build ...' to run. - - docker - - gce - - shared + - saas-linux-small-amd64 variables: SAMBA_CI_IS_BROKEN_IMAGE: "no" SAMBA_CI_TEST_JOB: "samba-o3" diff --git a/bootstrap/sha1sum.txt b/bootstrap/sha1sum.txt index 61ecaf0ccf6..ef035a8b771 100644 --- a/bootstrap/sha1sum.txt +++ b/bootstrap/sha1sum.txt @@ -1 +1 @@ -9a406973474a7903fe7fd6215226660911ed73c0 +2d7eba00f6919d62a37ccc31daab18d07c761c6a -- 2.45.2 From 0eca5185fd43dde9720535b5040f19858cf869f4 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 6 Jun 2024 16:10:14 +0200 Subject: [PATCH 2/6] bootstrap: Set git safe.directory BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher --- .gitlab-ci-main.yml | 2 +- bootstrap/.gitlab-ci.yml | 2 +- bootstrap/sha1sum.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml index 3da27e86e2f..3c71143a39a 100644 --- a/.gitlab-ci-main.yml +++ b/.gitlab-ci-main.yml @@ -47,7 +47,7 @@ variables: # Set this to the contents of bootstrap/sha1sum.txt # which is generated by bootstrap/template.py --render # - SAMBA_CI_CONTAINER_TAG: 2d7eba00f6919d62a37ccc31daab18d07c761c6a + SAMBA_CI_CONTAINER_TAG: 431701dccb250e5138e9f8cd4aade863f979193d # # We use the ubuntu2204 image as default as # it matches what we have on atb-devel-224 diff --git a/bootstrap/.gitlab-ci.yml b/bootstrap/.gitlab-ci.yml index 4de8c23afb4..77b4e4fe290 100644 --- a/bootstrap/.gitlab-ci.yml +++ b/bootstrap/.gitlab-ci.yml @@ -45,7 +45,7 @@ diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt # run smoke test with samba-o3 or samba-fuzz podman run --volume $(pwd):/src:ro ${ci_image_name} \ - /bin/bash -c "git clone /src samba && cd samba && export PKG_CONFIG_PATH=/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig && script/autobuild.py ${SAMBA_CI_TEST_JOB} --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase" + /bin/bash -c "git config --global --add safe.directory /src/.git && git clone /src samba && cd samba && export PKG_CONFIG_PATH=/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig && script/autobuild.py ${SAMBA_CI_TEST_JOB} --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase" podman tag ${ci_image_name} ${ci_image_path}:${SAMBA_CI_CONTAINER_TAG} podman tag ${ci_image_name} ${ci_image_path}:${timestamp_tag} # We build all images, but only upload is it's not marked as broken diff --git a/bootstrap/sha1sum.txt b/bootstrap/sha1sum.txt index ef035a8b771..cfc515b96b4 100644 --- a/bootstrap/sha1sum.txt +++ b/bootstrap/sha1sum.txt @@ -1 +1 @@ -2d7eba00f6919d62a37ccc31daab18d07c761c6a +431701dccb250e5138e9f8cd4aade863f979193d -- 2.45.2 From e626611605b95a98fa0707cb222cdf52d0636a37 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Mon, 10 Jun 2024 15:28:30 +0200 Subject: [PATCH 3/6] bootstrap: Fix building CentOS 8 Stream container images BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher --- .gitlab-ci-main.yml | 2 +- bootstrap/config.py | 3 +++ bootstrap/generated-dists/centos8s/bootstrap.sh | 3 +++ bootstrap/sha1sum.txt | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml index 3c71143a39a..c1c95c608d3 100644 --- a/.gitlab-ci-main.yml +++ b/.gitlab-ci-main.yml @@ -47,7 +47,7 @@ variables: # Set this to the contents of bootstrap/sha1sum.txt # which is generated by bootstrap/template.py --render # - SAMBA_CI_CONTAINER_TAG: 431701dccb250e5138e9f8cd4aade863f979193d + SAMBA_CI_CONTAINER_TAG: b078783e082ead539940faaa644567bf4ed67f67 # # We use the ubuntu2204 image as default as # it matches what we have on atb-devel-224 diff --git a/bootstrap/config.py b/bootstrap/config.py index 11d8314aefc..a5a7366c7fa 100644 --- a/bootstrap/config.py +++ b/bootstrap/config.py @@ -241,6 +241,9 @@ CENTOS8S_YUM_BOOTSTRAP = r""" {GENERATED_MARKER} set -xueo pipefail +sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + yum update -y yum install -y dnf-plugins-core yum install -y epel-release diff --git a/bootstrap/generated-dists/centos8s/bootstrap.sh b/bootstrap/generated-dists/centos8s/bootstrap.sh index 4b2c62c66d9..9e0aabbac28 100755 --- a/bootstrap/generated-dists/centos8s/bootstrap.sh +++ b/bootstrap/generated-dists/centos8s/bootstrap.sh @@ -7,6 +7,9 @@ set -xueo pipefail +sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + yum update -y yum install -y dnf-plugins-core yum install -y epel-release diff --git a/bootstrap/sha1sum.txt b/bootstrap/sha1sum.txt index cfc515b96b4..1bb5e922d9b 100644 --- a/bootstrap/sha1sum.txt +++ b/bootstrap/sha1sum.txt @@ -1 +1 @@ -431701dccb250e5138e9f8cd4aade863f979193d +b078783e082ead539940faaa644567bf4ed67f67 -- 2.45.2 From 6b4f2733bc7c633e48445476fb0418f0405cca18 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Fri, 7 Jun 2024 16:20:10 +0200 Subject: [PATCH 4/6] gitlab-ci: Set git safe.directory for devel repo BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher --- .gitlab-ci-main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml index c1c95c608d3..face2103327 100644 --- a/.gitlab-ci-main.yml +++ b/.gitlab-ci-main.yml @@ -147,6 +147,7 @@ include: - ccache -s # We are already running .gitlab-ci directives from this repo, remove additional checks that break our CI - git config --global --add safe.directory `pwd` + - git config --global --add safe.directory /builds/samba-team/devel/samba/.git after_script: - mount - df -h -- 2.45.2 From e6cf18a5ad035f9459e30a73d25d67c2cec1c66f Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 13 Jun 2024 07:41:41 +0200 Subject: [PATCH 5/6] third_party: Update uid_wrapper to version 1.3.1 This fixes issues with bind compiled with jemalloc. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher --- buildtools/wafsamba/samba_third_party.py | 2 +- third_party/uid_wrapper/uid_wrapper.c | 58 +++++++++++++++++++++++- third_party/uid_wrapper/wscript | 4 +- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/buildtools/wafsamba/samba_third_party.py b/buildtools/wafsamba/samba_third_party.py index 52898486fd9..8fa1756b03b 100644 --- a/buildtools/wafsamba/samba_third_party.py +++ b/buildtools/wafsamba/samba_third_party.py @@ -39,7 +39,7 @@ Build.BuildContext.CHECK_RESOLV_WRAPPER = CHECK_RESOLV_WRAPPER @conf def CHECK_UID_WRAPPER(conf): - return conf.CHECK_BUNDLED_SYSTEM_PKG('uid_wrapper', minversion='1.3.0') + return conf.CHECK_BUNDLED_SYSTEM_PKG('uid_wrapper', minversion='1.3.1') Build.BuildContext.CHECK_UID_WRAPPER = CHECK_UID_WRAPPER @conf diff --git a/third_party/uid_wrapper/uid_wrapper.c b/third_party/uid_wrapper/uid_wrapper.c index 5b6a82b8fef..ca578e61f2e 100644 --- a/third_party/uid_wrapper/uid_wrapper.c +++ b/third_party/uid_wrapper/uid_wrapper.c @@ -38,6 +38,10 @@ #include +#ifdef HAVE_GNU_LIB_NAMES_H +#include +#endif + #ifdef HAVE_GCC_THREAD_LOCAL_STORAGE # define UWRAP_THREAD __thread #else @@ -558,6 +562,13 @@ static void *uwrap_load_lib_handle(enum uwrap_lib lib) switch (lib) { case UWRAP_LIBC: handle = uwrap.libc.handle; +#ifdef LIBC_SO + if (handle == NULL) { + handle = dlopen(LIBC_SO, flags); + + uwrap.libc.handle = handle; + } +#endif if (handle == NULL) { for (i = 10; i >= 0; i--) { char soname[256] = {0}; @@ -656,6 +667,9 @@ static void *_uwrap_bind_symbol(enum uwrap_lib lib, const char *fn_name) dlsym(RTLD_DEFAULT, #sym_name); \ } +/* JEMALLOC: This tells uid_wrapper if it should handle syscall() */ +static bool uwrap_handle_syscall; + /* DO NOT call this function during library initialization! */ static void __uwrap_bind_symbol_all_once(void) { @@ -699,6 +713,8 @@ static void __uwrap_bind_symbol_all_once(void) #endif uwrap_bind_symbol_libpthread(pthread_create); uwrap_bind_symbol_libpthread(pthread_exit); + + uwrap_handle_syscall = true; } static void uwrap_bind_symbol_all(void) @@ -863,7 +879,27 @@ static long int libc_vsyscall(long int sysno, va_list va) long int rc; int i; - uwrap_bind_symbol_all(); + /* + * JEMALLOC: + * + * This is a workaround to prevent a deadlock in jemalloc calling + * malloc_init() twice. The first allocation call will trigger a + * malloc_init() of jemalloc. The functions calls syscall(SYS_open, ...) + * so it goes to socket or uid wrapper. In this code path we need to + * avoid any allocation calls. This will prevent the deadlock. + * + * We also need to avoid dlopen() as that would trigger the recursion + * into malloc_init(), so we use dlsym(RTLD_NEXT), until we reached + * swrap_constructor() or any real socket call at that time + * swrap_bind_symbol_all() will replace the function pointer again after + * dlopen of libc. + */ + if (uwrap_handle_syscall) { + uwrap_bind_symbol_all(); + } else if (uwrap.libc.symbols._libc_syscall.obj == NULL) { + uwrap.libc.symbols._libc_syscall.obj = dlsym(RTLD_NEXT, + "syscall"); + } for (i = 0; i < 8; i++) { args[i] = va_arg(va, long int); @@ -1375,7 +1411,7 @@ static void uwrap_init_env(struct uwrap_thread *id) exit(-1); } - UWRAP_LOG(UWRAP_LOG_DEBUG, "Initalize groups with %s", env); + UWRAP_LOG(UWRAP_LOG_DEBUG, "Initialize groups with %s", env); id->ngroups = ngroups; } } @@ -2708,6 +2744,21 @@ long int syscall (long int sysno, ...) va_start(va, sysno); + /* + * JEMALLOC: + * + * This is a workaround to prevent a deadlock in jemalloc calling + * malloc_init() twice. The first allocation call will trigger a + * malloc_init() of jemalloc. The functions calls syscall(SYS_open, ...) + * so it goes to socket or uid wrapper. In this code path we need to + * avoid any allocation calls. This will prevent the deadlock. + */ + if (!uwrap_handle_syscall) { + rc = libc_vsyscall(sysno, va); + va_end(va); + return rc; + } + /* * We need to check for uwrap related syscall numbers before calling * uid_wrapper_enabled() otherwise we'd deadlock during the freebsd libc @@ -2821,6 +2872,9 @@ void uwrap_constructor(void) * for main process. */ uwrap_init(); + + /* Let socket_wrapper handle syscall() */ + uwrap_handle_syscall = true; } /**************************** diff --git a/third_party/uid_wrapper/wscript b/third_party/uid_wrapper/wscript index 7b65d930640..5af76903fb9 100644 --- a/third_party/uid_wrapper/wscript +++ b/third_party/uid_wrapper/wscript @@ -3,13 +3,15 @@ from waflib import Options import os, sys -VERSION="1.3.0" +VERSION="1.3.1" def configure(conf): if conf.CHECK_UID_WRAPPER(): conf.DEFINE('USING_SYSTEM_UID_WRAPPER', 1) libuid_wrapper_so_path = 'libuid_wrapper.so' else: + conf.CHECK_HEADERS('gnu/lib-names.h') + # check HAVE_GCC_ATOMIC_BUILTINS conf.CHECK_CODE(''' #include -- 2.45.2 From 8b3251e2ae290297f800eb8a5be89890e1b3144d Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 13 Jun 2024 07:47:26 +0200 Subject: [PATCH 6/6] third_party: Update socket_wrapper to version 1.4.3 This fixes issues with bind compiled with jemalloc. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15660 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher Autobuild-User(master): Andreas Schneider Autobuild-Date(master): Thu Jun 13 08:41:39 UTC 2024 on atb-devel-224 --- buildtools/wafsamba/samba_third_party.py | 2 +- third_party/socket_wrapper/socket_wrapper.c | 45 ++++++++++++++++++++- third_party/socket_wrapper/wscript | 3 +- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/buildtools/wafsamba/samba_third_party.py b/buildtools/wafsamba/samba_third_party.py index 8fa1756b03b..a42bb2ddc90 100644 --- a/buildtools/wafsamba/samba_third_party.py +++ b/buildtools/wafsamba/samba_third_party.py @@ -24,7 +24,7 @@ Build.BuildContext.CHECK_CMOCKA = CHECK_CMOCKA @conf def CHECK_SOCKET_WRAPPER(conf): - return conf.CHECK_BUNDLED_SYSTEM_PKG('socket_wrapper', minversion='1.4.2') + return conf.CHECK_BUNDLED_SYSTEM_PKG('socket_wrapper', minversion='1.4.3') Build.BuildContext.CHECK_SOCKET_WRAPPER = CHECK_SOCKET_WRAPPER @conf diff --git a/third_party/socket_wrapper/socket_wrapper.c b/third_party/socket_wrapper/socket_wrapper.c index c759d350fb1..37799c82419 100644 --- a/third_party/socket_wrapper/socket_wrapper.c +++ b/third_party/socket_wrapper/socket_wrapper.c @@ -1388,6 +1388,9 @@ static ssize_t libc_writev(int fd, const struct iovec *iov, int iovcnt) return swrap.libc.symbols._libc_writev.f(fd, iov, iovcnt); } +/* JEMALLOC: This tells socket_wrapper if it should handle syscall() */ +static bool swrap_handle_syscall; + #ifdef HAVE_SYSCALL DO_NOT_SANITIZE_ADDRESS_ATTRIBUTE static long int libc_vsyscall(long int sysno, va_list va) @@ -1396,7 +1399,27 @@ static long int libc_vsyscall(long int sysno, va_list va) long int rc; int i; - swrap_bind_symbol_all(); + /* + * JEMALLOC: + * + * This is a workaround to prevent a deadlock in jemalloc calling + * malloc_init() twice. The first allocation call will trigger a + * malloc_init() of jemalloc. The functions calls syscall(SYS_open, ...) + * so it goes to socket or uid wrapper. In this code path we need to + * avoid any allocation calls. This will prevent the deadlock. + * + * We also need to avoid dlopen() as that would trigger the recursion + * into malloc_init(), so we use dlsym(RTLD_NEXT), until we reached + * swrap_constructor() or any real socket call at that time + * swrap_bind_symbol_all() will replace the function pointer again after + * dlopen of libc. + */ + if (swrap_handle_syscall) { + swrap_bind_symbol_all(); + } else if (swrap.libc.symbols._libc_syscall.obj == NULL) { + swrap.libc.symbols._libc_syscall.obj = dlsym(RTLD_NEXT, + "syscall"); + } for (i = 0; i < 8; i++) { args[i] = va_arg(va, long int); @@ -1517,6 +1540,8 @@ static void __swrap_bind_symbol_all_once(void) swrap_bind_symbol_rtld_default_optional(uid_wrapper_syscall_valid); swrap_bind_symbol_rtld_default_optional(uid_wrapper_syscall_va); #endif + + swrap_handle_syscall = true; } static void swrap_bind_symbol_all(void) @@ -8744,6 +8769,21 @@ long int syscall(long int sysno, ...) va_start(va, sysno); + /* + * JEMALLOC: + * + * This is a workaround to prevent a deadlock in jemalloc calling + * malloc_init() twice. The first allocation call will trigger a + * malloc_init() of jemalloc. The functions calls syscall(SYS_open, ...) + * so it goes to socket or uid wrapper. In this code path we need to + * avoid any allocation calls. This will prevent the deadlock. + */ + if (!swrap_handle_syscall) { + rc = libc_vsyscall(sysno, va); + va_end(va); + return rc; + } + /* * We should only handle the syscall numbers * we care about... @@ -8860,6 +8900,9 @@ void swrap_constructor(void) pthread_atfork(&swrap_thread_prepare, &swrap_thread_parent, &swrap_thread_child); + + /* Let socket_wrapper handle syscall() */ + swrap_handle_syscall = true; } /**************************** diff --git a/third_party/socket_wrapper/wscript b/third_party/socket_wrapper/wscript index 9d2210d9c35..cdd34938ba3 100644 --- a/third_party/socket_wrapper/wscript +++ b/third_party/socket_wrapper/wscript @@ -2,7 +2,7 @@ import os -VERSION = "1.4.2" +VERSION = "1.4.3" def configure(conf): @@ -10,6 +10,7 @@ def configure(conf): conf.DEFINE('USING_SYSTEM_SOCKET_WRAPPER', 1) libsocket_wrapper_so_path = 'libsocket_wrapper.so' else: + conf.CHECK_HEADERS('gnu/lib-names.h') if conf.CONFIG_SET("HAVE___THREAD"): conf.DEFINE("HAVE_GCC_THREAD_LOCAL_STORAGE", 1) -- 2.45.2