From e2344320f99e5b4e4cb323458607a99ca793b0a5 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 7 Feb 2020 11:02:18 +0100 Subject: [PATCH 1/3] lib/replace: remove unused check for aio.h Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison (cherry picked from commit c1b9bf993402f7d74f992c609ede94653e87172f) --- lib/replace/wscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/replace/wscript b/lib/replace/wscript index a2bc60478ad..ab2b3c043af 100644 --- a/lib/replace/wscript +++ b/lib/replace/wscript @@ -67,7 +67,7 @@ def configure(conf): headers='sys/inotify.h') conf.CHECK_HEADERS('security/pam_appl.h zlib.h asm/unistd.h') - conf.CHECK_HEADERS('aio.h sys/unistd.h alloca.h float.h') + conf.CHECK_HEADERS('sys/unistd.h alloca.h float.h') conf.SET_TARGET_TYPE('tirpc', 'EMPTY') -- 2.17.1 From bd519f3d901e0f0b89aa5091a416d94964e514cf Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 7 Feb 2020 11:00:21 +0100 Subject: [PATCH 2/3] bootstrap: replace libaio-dev/libaio-devel with liburing-dev/liburing-devel We'll never use libaio, but will soon add a io_uring vfs module. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison (cherry picked from commit fb5a99fa0200785ef1b5a8c9bc4a645d09fbeb5e) --- .gitlab-ci.yml | 2 +- bootstrap/config.py | 18 +++++++++++++++++- bootstrap/generated-dists/centos6/bootstrap.sh | 1 - bootstrap/generated-dists/centos6/packages.yml | 1 - bootstrap/generated-dists/centos7/bootstrap.sh | 1 - bootstrap/generated-dists/centos7/packages.yml | 1 - bootstrap/generated-dists/centos8/bootstrap.sh | 1 - bootstrap/generated-dists/centos8/packages.yml | 1 - .../generated-dists/debian10/bootstrap.sh | 1 - .../generated-dists/debian10/packages.yml | 1 - bootstrap/generated-dists/debian7/bootstrap.sh | 1 - bootstrap/generated-dists/debian7/packages.yml | 1 - bootstrap/generated-dists/debian8/bootstrap.sh | 1 - bootstrap/generated-dists/debian8/packages.yml | 1 - bootstrap/generated-dists/debian9/bootstrap.sh | 1 - bootstrap/generated-dists/debian9/packages.yml | 1 - .../generated-dists/fedora29/bootstrap.sh | 1 - .../generated-dists/fedora29/packages.yml | 1 - .../generated-dists/fedora30/bootstrap.sh | 1 - .../generated-dists/fedora30/packages.yml | 1 - .../generated-dists/fedora31/bootstrap.sh | 2 +- .../generated-dists/fedora31/packages.yml | 2 +- .../generated-dists/opensuse150/bootstrap.sh | 1 - .../generated-dists/opensuse150/packages.yml | 1 - .../generated-dists/opensuse151/bootstrap.sh | 1 - .../generated-dists/opensuse151/packages.yml | 1 - .../generated-dists/ubuntu1404/bootstrap.sh | 1 - .../generated-dists/ubuntu1404/packages.yml | 1 - .../generated-dists/ubuntu1604/bootstrap.sh | 1 - .../generated-dists/ubuntu1604/packages.yml | 1 - .../generated-dists/ubuntu1804/bootstrap.sh | 1 - .../generated-dists/ubuntu1804/packages.yml | 1 - bootstrap/sha1sum.txt | 2 +- 33 files changed, 21 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df91d7dadb1..56adf10c7be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ variables: # Set this to the contents of bootstrap/sha1sum.txt # which is generated by bootstrap/template.py --render # - SAMBA_CI_CONTAINER_TAG: cffd3e4d9ad505cbf2b6732d2e3021d60e4159ad + SAMBA_CI_CONTAINER_TAG: 41319f2580c026f66b2750604a0eb15d6b6f7b50 # # We use the ubuntu1804 image as default as # it matches what we have on sn-devel-184. diff --git a/bootstrap/config.py b/bootstrap/config.py index ecead21da29..bcada1dc628 100644 --- a/bootstrap/config.py +++ b/bootstrap/config.py @@ -69,7 +69,7 @@ PKGS = [ ('mingw-w64', 'mingw64-gcc'), ('zlib1g-dev', 'zlib-devel'), ('libbsd-dev', 'libbsd-devel'), - ('libaio-dev', 'libaio-devel'), + ('liburing-dev', 'liburing-devel'), ('libarchive-dev', 'libarchive-devel'), ('libblkid-dev', 'libblkid-devel'), ('libcap-dev', 'libcap-devel'), @@ -385,6 +385,7 @@ DEB_DISTS = { 'python-gpg': 'python-gpgme', 'python3-gpg': '', # no python3 gpg pkg available, remove 'language-pack-en': '', # included in locales + 'liburing-dev': '', # not available } }, 'debian8': { @@ -394,6 +395,7 @@ DEB_DISTS = { 'python-gpg': 'python-gpgme', 'python3-gpg': 'python3-gpgme', 'language-pack-en': '', # included in locales + 'liburing-dev': '', # not available } }, 'debian9': { @@ -401,6 +403,7 @@ DEB_DISTS = { 'vagrant_box': 'debian/stretch64', 'replace': { 'language-pack-en': '', # included in locales + 'liburing-dev': '', # not available } }, 'debian10': { @@ -408,6 +411,7 @@ DEB_DISTS = { 'vagrant_box': 'debian/buster64', 'replace': { 'language-pack-en': '', # included in locales + 'liburing-dev': '', # not available } }, 'ubuntu1404': { @@ -423,6 +427,7 @@ DEB_DISTS = { 'libunwind-dev': 'libunwind8-dev', 'glusterfs-common': '', 'libcephfs-dev': '', + 'liburing-dev': '', # not available } }, 'ubuntu1604': { @@ -433,11 +438,15 @@ DEB_DISTS = { 'python3-gpg': 'python3-gpgme', 'glusterfs-common': '', 'libcephfs-dev': '', + 'liburing-dev': '', # not available } }, 'ubuntu1804': { 'docker_image': 'ubuntu:18.04', 'vagrant_box': 'ubuntu/bionic64', + 'replace': { + 'liburing-dev': '', # not available + } }, } @@ -464,6 +473,7 @@ RPM_DISTS = { 'glusterfs-api-devel': '', 'glusterfs-devel': '', 'libcephfs-devel': '', + 'liburing-devel': '', # not available } }, 'centos7': { @@ -495,6 +505,7 @@ RPM_DISTS = { 'glusterfs-devel': '', 'libcephfs-devel': '', 'gnutls-devel': 'compat-gnutls34-devel', + 'liburing-devel': '', # not available } }, 'centos8': { @@ -511,6 +522,7 @@ RPM_DISTS = { 'policycoreutils-python': 'python3-policycoreutils', 'python3-crypto': '', 'quota-devel': '', # FIXME: Add me back, once available! + 'liburing-devel': '', # not available yet, Add me back, once available! } }, 'fedora29': { @@ -519,6 +531,7 @@ RPM_DISTS = { 'bootstrap': DNF_BOOTSTRAP, 'replace': { 'lsb-release': 'redhat-lsb', + 'liburing-devel': '', # not available } }, 'fedora30': { @@ -527,6 +540,7 @@ RPM_DISTS = { 'bootstrap': DNF_BOOTSTRAP, 'replace': { 'lsb-release': 'redhat-lsb', + 'liburing-devel': '', # not available } }, 'fedora31': { @@ -569,6 +583,7 @@ RPM_DISTS = { 'glusterfs-api-devel': '', 'libtasn1-tools': '', # asn1Parser is part of libtasn1 'mingw64-gcc': '', # doesn't exist + 'liburing-devel': '', # not available } }, 'opensuse151': { @@ -601,6 +616,7 @@ RPM_DISTS = { 'glusterfs-api-devel': '', 'libtasn1-tools': '', # asn1Parser is part of libtasn1 'mingw64-gcc': '', # doesn't exist + 'liburing-devel': '', # not available, will be added in 15.2 } } } diff --git a/bootstrap/generated-dists/centos6/bootstrap.sh b/bootstrap/generated-dists/centos6/bootstrap.sh index b1fb2536fa9..ee6fcc33799 100755 --- a/bootstrap/generated-dists/centos6/bootstrap.sh +++ b/bootstrap/generated-dists/centos6/bootstrap.sh @@ -46,7 +46,6 @@ yum install -y \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/centos6/packages.yml b/bootstrap/generated-dists/centos6/packages.yml index cba40f3b93d..1e2b5a98fa6 100644 --- a/bootstrap/generated-dists/centos6/packages.yml +++ b/bootstrap/generated-dists/centos6/packages.yml @@ -32,7 +32,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/centos7/bootstrap.sh b/bootstrap/generated-dists/centos7/bootstrap.sh index f97c77d2a6b..2f0bb1bcc28 100755 --- a/bootstrap/generated-dists/centos7/bootstrap.sh +++ b/bootstrap/generated-dists/centos7/bootstrap.sh @@ -46,7 +46,6 @@ yum install -y \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/centos7/packages.yml b/bootstrap/generated-dists/centos7/packages.yml index 87efa4330d1..475326b6db7 100644 --- a/bootstrap/generated-dists/centos7/packages.yml +++ b/bootstrap/generated-dists/centos7/packages.yml @@ -32,7 +32,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/centos8/bootstrap.sh b/bootstrap/generated-dists/centos8/bootstrap.sh index e19a8b169c7..22484b3f6ad 100755 --- a/bootstrap/generated-dists/centos8/bootstrap.sh +++ b/bootstrap/generated-dists/centos8/bootstrap.sh @@ -49,7 +49,6 @@ yum install -y \ krb5-devel \ krb5-server \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/centos8/packages.yml b/bootstrap/generated-dists/centos8/packages.yml index a86bbbdc620..07be0deeceb 100644 --- a/bootstrap/generated-dists/centos8/packages.yml +++ b/bootstrap/generated-dists/centos8/packages.yml @@ -34,7 +34,6 @@ packages: - krb5-devel - krb5-server - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/debian10/bootstrap.sh b/bootstrap/generated-dists/debian10/bootstrap.sh index eb610c5ee79..f0847eb3c20 100755 --- a/bootstrap/generated-dists/debian10/bootstrap.sh +++ b/bootstrap/generated-dists/debian10/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ krb5-user \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/debian10/packages.yml b/bootstrap/generated-dists/debian10/packages.yml index 8440e143f62..a242cd8b362 100644 --- a/bootstrap/generated-dists/debian10/packages.yml +++ b/bootstrap/generated-dists/debian10/packages.yml @@ -28,7 +28,6 @@ packages: - krb5-user - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/debian7/bootstrap.sh b/bootstrap/generated-dists/debian7/bootstrap.sh index ea8a9ad8afc..daedce84fd9 100755 --- a/bootstrap/generated-dists/debian7/bootstrap.sh +++ b/bootstrap/generated-dists/debian7/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ krb5-user \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/debian7/packages.yml b/bootstrap/generated-dists/debian7/packages.yml index e531b5822ed..3cac6870b3e 100644 --- a/bootstrap/generated-dists/debian7/packages.yml +++ b/bootstrap/generated-dists/debian7/packages.yml @@ -28,7 +28,6 @@ packages: - krb5-user - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/debian8/bootstrap.sh b/bootstrap/generated-dists/debian8/bootstrap.sh index 6c720adcd1a..010508360ed 100755 --- a/bootstrap/generated-dists/debian8/bootstrap.sh +++ b/bootstrap/generated-dists/debian8/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ krb5-user \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/debian8/packages.yml b/bootstrap/generated-dists/debian8/packages.yml index 53052141cf5..1c9552b85f3 100644 --- a/bootstrap/generated-dists/debian8/packages.yml +++ b/bootstrap/generated-dists/debian8/packages.yml @@ -28,7 +28,6 @@ packages: - krb5-user - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/debian9/bootstrap.sh b/bootstrap/generated-dists/debian9/bootstrap.sh index eb610c5ee79..f0847eb3c20 100755 --- a/bootstrap/generated-dists/debian9/bootstrap.sh +++ b/bootstrap/generated-dists/debian9/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ krb5-user \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/debian9/packages.yml b/bootstrap/generated-dists/debian9/packages.yml index 8440e143f62..a242cd8b362 100644 --- a/bootstrap/generated-dists/debian9/packages.yml +++ b/bootstrap/generated-dists/debian9/packages.yml @@ -28,7 +28,6 @@ packages: - krb5-user - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/fedora29/bootstrap.sh b/bootstrap/generated-dists/fedora29/bootstrap.sh index c82c86166e5..effe2a9d214 100755 --- a/bootstrap/generated-dists/fedora29/bootstrap.sh +++ b/bootstrap/generated-dists/fedora29/bootstrap.sh @@ -46,7 +46,6 @@ dnf install -y \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/fedora29/packages.yml b/bootstrap/generated-dists/fedora29/packages.yml index 35aaa53c117..3b767f8ba08 100644 --- a/bootstrap/generated-dists/fedora29/packages.yml +++ b/bootstrap/generated-dists/fedora29/packages.yml @@ -35,7 +35,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/fedora30/bootstrap.sh b/bootstrap/generated-dists/fedora30/bootstrap.sh index c82c86166e5..effe2a9d214 100755 --- a/bootstrap/generated-dists/fedora30/bootstrap.sh +++ b/bootstrap/generated-dists/fedora30/bootstrap.sh @@ -46,7 +46,6 @@ dnf install -y \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/fedora30/packages.yml b/bootstrap/generated-dists/fedora30/packages.yml index 35aaa53c117..3b767f8ba08 100644 --- a/bootstrap/generated-dists/fedora30/packages.yml +++ b/bootstrap/generated-dists/fedora30/packages.yml @@ -35,7 +35,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/fedora31/bootstrap.sh b/bootstrap/generated-dists/fedora31/bootstrap.sh index fd6eecd1ab7..18c58092eff 100755 --- a/bootstrap/generated-dists/fedora31/bootstrap.sh +++ b/bootstrap/generated-dists/fedora31/bootstrap.sh @@ -46,7 +46,6 @@ dnf install -y \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ @@ -60,6 +59,7 @@ dnf install -y \ libtasn1-tools \ libtirpc-devel \ libunwind-devel \ + liburing-devel \ libuuid-devel \ libxslt \ lmdb \ diff --git a/bootstrap/generated-dists/fedora31/packages.yml b/bootstrap/generated-dists/fedora31/packages.yml index 8a2e2c785ad..3165af8dd82 100644 --- a/bootstrap/generated-dists/fedora31/packages.yml +++ b/bootstrap/generated-dists/fedora31/packages.yml @@ -35,7 +35,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel @@ -49,6 +48,7 @@ packages: - libtasn1-tools - libtirpc-devel - libunwind-devel + - liburing-devel - libuuid-devel - libxslt - lmdb diff --git a/bootstrap/generated-dists/opensuse150/bootstrap.sh b/bootstrap/generated-dists/opensuse150/bootstrap.sh index 0e229caacb5..3fbcaacb24f 100755 --- a/bootstrap/generated-dists/opensuse150/bootstrap.sh +++ b/bootstrap/generated-dists/opensuse150/bootstrap.sh @@ -43,7 +43,6 @@ zypper --non-interactive install \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/opensuse150/packages.yml b/bootstrap/generated-dists/opensuse150/packages.yml index 910f95cf55f..0eb19244677 100644 --- a/bootstrap/generated-dists/opensuse150/packages.yml +++ b/bootstrap/generated-dists/opensuse150/packages.yml @@ -31,7 +31,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/opensuse151/bootstrap.sh b/bootstrap/generated-dists/opensuse151/bootstrap.sh index 0e229caacb5..3fbcaacb24f 100755 --- a/bootstrap/generated-dists/opensuse151/bootstrap.sh +++ b/bootstrap/generated-dists/opensuse151/bootstrap.sh @@ -43,7 +43,6 @@ zypper --non-interactive install \ krb5-server \ lcov \ libacl-devel \ - libaio-devel \ libarchive-devel \ libattr-devel \ libblkid-devel \ diff --git a/bootstrap/generated-dists/opensuse151/packages.yml b/bootstrap/generated-dists/opensuse151/packages.yml index 910f95cf55f..0eb19244677 100644 --- a/bootstrap/generated-dists/opensuse151/packages.yml +++ b/bootstrap/generated-dists/opensuse151/packages.yml @@ -31,7 +31,6 @@ packages: - krb5-server - lcov - libacl-devel - - libaio-devel - libarchive-devel - libattr-devel - libblkid-devel diff --git a/bootstrap/generated-dists/ubuntu1404/bootstrap.sh b/bootstrap/generated-dists/ubuntu1404/bootstrap.sh index 9c707de566d..78c8969ac69 100755 --- a/bootstrap/generated-dists/ubuntu1404/bootstrap.sh +++ b/bootstrap/generated-dists/ubuntu1404/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ language-pack-en \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/ubuntu1404/packages.yml b/bootstrap/generated-dists/ubuntu1404/packages.yml index f8d2ab8fc4c..f6c8a0aaa00 100644 --- a/bootstrap/generated-dists/ubuntu1404/packages.yml +++ b/bootstrap/generated-dists/ubuntu1404/packages.yml @@ -28,7 +28,6 @@ packages: - language-pack-en - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/ubuntu1604/bootstrap.sh b/bootstrap/generated-dists/ubuntu1604/bootstrap.sh index d7540d31a38..a8f47762ded 100755 --- a/bootstrap/generated-dists/ubuntu1604/bootstrap.sh +++ b/bootstrap/generated-dists/ubuntu1604/bootstrap.sh @@ -39,7 +39,6 @@ apt-get -y install \ language-pack-en \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/ubuntu1604/packages.yml b/bootstrap/generated-dists/ubuntu1604/packages.yml index da1c540bf3f..c3cd9af9c3e 100644 --- a/bootstrap/generated-dists/ubuntu1604/packages.yml +++ b/bootstrap/generated-dists/ubuntu1604/packages.yml @@ -28,7 +28,6 @@ packages: - language-pack-en - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/generated-dists/ubuntu1804/bootstrap.sh b/bootstrap/generated-dists/ubuntu1804/bootstrap.sh index f0b3e3120d0..97d32815d72 100755 --- a/bootstrap/generated-dists/ubuntu1804/bootstrap.sh +++ b/bootstrap/generated-dists/ubuntu1804/bootstrap.sh @@ -40,7 +40,6 @@ apt-get -y install \ language-pack-en \ lcov \ libacl1-dev \ - libaio-dev \ libarchive-dev \ libattr1-dev \ libavahi-common-dev \ diff --git a/bootstrap/generated-dists/ubuntu1804/packages.yml b/bootstrap/generated-dists/ubuntu1804/packages.yml index eda09ec5c7c..f45deb2c808 100644 --- a/bootstrap/generated-dists/ubuntu1804/packages.yml +++ b/bootstrap/generated-dists/ubuntu1804/packages.yml @@ -29,7 +29,6 @@ packages: - language-pack-en - lcov - libacl1-dev - - libaio-dev - libarchive-dev - libattr1-dev - libavahi-common-dev diff --git a/bootstrap/sha1sum.txt b/bootstrap/sha1sum.txt index 5f1e8108f71..62c2245564e 100644 --- a/bootstrap/sha1sum.txt +++ b/bootstrap/sha1sum.txt @@ -1 +1 @@ -cffd3e4d9ad505cbf2b6732d2e3021d60e4159ad +41319f2580c026f66b2750604a0eb15d6b6f7b50 -- 2.17.1 From 255a73767360c45ffeeeca958452370474461bb4 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 5 Jun 2019 17:01:49 +0200 Subject: [PATCH 3/3] s3:modules: add vfs_io_uring module The module makes use of the new io_uring infrastructure (intruduced in linux 5.1), see https://lwn.net/Articles/778411/ and http://git.kernel.dk/cgit/liburing/ Currently this only implements SMB_VFS_{PREAD,PWRITE,FSYNC}_SEND/RECV and avoids the overhead of our userspace threadpool. In future we'll hopefully make more use of more advanced io_uring features. For now we don't have automated tests as our test infrastructure doesn't use a recent kernel. At least we're able to do compile tests on fedora31. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Sat Feb 15 11:37:45 UTC 2020 on sn-devel-184 (cherry picked from commit 195e88cea34c983afc02cfb521a60cdd1edbe9e1) --- docs-xml/manpages/vfs_io_uring.8.xml | 107 ++++++ docs-xml/wscript_build | 1 + source3/modules/vfs_io_uring.c | 543 +++++++++++++++++++++++++++ source3/modules/wscript_build | 8 + source3/wscript | 11 + 5 files changed, 670 insertions(+) create mode 100644 docs-xml/manpages/vfs_io_uring.8.xml create mode 100644 source3/modules/vfs_io_uring.c diff --git a/docs-xml/manpages/vfs_io_uring.8.xml b/docs-xml/manpages/vfs_io_uring.8.xml new file mode 100644 index 00000000000..f47a5041ec0 --- /dev/null +++ b/docs-xml/manpages/vfs_io_uring.8.xml @@ -0,0 +1,107 @@ + + + + + + vfs_io_uring + 8 + Samba + System Administration tools + &doc.version; + + + + vfs_io_uring + Implement async io in Samba vfs using io_uring of Linux (>= 5.1). + + + + + vfs objects = io_uring + + + + + DESCRIPTION + + This VFS module is part of the + samba + 7 suite. + + The io_uring VFS module enables asynchronous + pread, pwrite and fsync using the io_uring infrastructure of Linux (>= 5.1). + This provides much less overhead compared to the usage of the pthreadpool for + async io. + + This module SHOULD be listed last in any module stack as + it requires real kernel file descriptors. + + + + + + EXAMPLES + + Straight forward use: + + + + /data/ice + io_uring + + + + + + OPTIONS + + + + + io_uring:num_entries = NUMBER_OF_QUEUE_ENTRIES + + The number of entries in the submission queue. + The maximum allowed value depends on the kernel version + and the kernel will roundup the value to a power of 2. + + The default is '128'. + + + + + io_uring:sqpoll = BOOL + + Use the IORING_SETUP_SQPOLL feature. + + The default is 'no'. + + + + + + + + SEE ALSO + + io_uring_setup2. + + + + + VERSION + + This man page is part of version &doc.version; of the Samba suite. + + + + + AUTHOR + + The original Samba software and related utilities + were created by Andrew Tridgell. Samba is now developed + by the Samba Team as an Open Source project similar + to the way the Linux kernel is developed. + + + + diff --git a/docs-xml/wscript_build b/docs-xml/wscript_build index 70f5b43dd33..21158e11fe7 100644 --- a/docs-xml/wscript_build +++ b/docs-xml/wscript_build @@ -68,6 +68,7 @@ vfs_module_manpages = ['vfs_acl_tdb', 'vfs_acl_xattr', 'vfs_aio_fork', 'vfs_aio_pthread', + 'vfs_io_uring', 'vfs_audit', 'vfs_btrfs', 'vfs_cacheprime', diff --git a/source3/modules/vfs_io_uring.c b/source3/modules/vfs_io_uring.c new file mode 100644 index 00000000000..378e48d112f --- /dev/null +++ b/source3/modules/vfs_io_uring.c @@ -0,0 +1,543 @@ +/* + * Use the io_uring of Linux (>= 5.1) + * + * Copyright (C) Volker Lendecke 2008 + * Copyright (C) Jeremy Allison 2010 + * Copyright (C) Stefan Metzmacher 2019 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "includes.h" +#include "system/filesys.h" +#include "smbd/smbd.h" +#include "smbd/globals.h" +#include "lib/util/tevent_unix.h" +#include "smbprofile.h" +#include + +struct vfs_io_uring_request; + +struct vfs_io_uring_config { + struct io_uring uring; + struct tevent_fd *fde; + struct vfs_io_uring_request *queue; + struct vfs_io_uring_request *pending; +}; + +struct vfs_io_uring_request { + struct vfs_io_uring_request *prev, *next; + struct vfs_io_uring_request **list_head; + struct vfs_io_uring_config *config; + struct tevent_req *req; + void *state; + struct io_uring_sqe sqe; + struct io_uring_cqe cqe; + struct timespec start_time; + struct timespec end_time; + SMBPROFILE_BYTES_ASYNC_STATE(profile_bytes); +}; + +static void vfs_io_uring_finish_req(struct vfs_io_uring_request *cur, + const struct io_uring_cqe *cqe, + struct timespec end_time, + const char *location) +{ + struct tevent_req *req = + talloc_get_type_abort(cur->req, + struct tevent_req); + + talloc_set_destructor(cur->state, NULL); + if (cur->list_head != NULL) { + DLIST_REMOVE((*cur->list_head), cur); + cur->list_head = NULL; + } + cur->cqe = *cqe; + + SMBPROFILE_BYTES_ASYNC_SET_IDLE(cur->profile_bytes); + cur->end_time = end_time; + + /* + * We rely on being inside the _send() function + * or tevent_req_defer_callback() being called + * already. + */ + _tevent_req_done(req, location); +} + +static void vfs_io_uring_config_destroy(struct vfs_io_uring_config *config, + int ret, + const char *location) +{ + struct vfs_io_uring_request *cur = NULL, *next = NULL; + struct timespec start_time; + struct timespec end_time; + struct io_uring_cqe err_cqe = { + .res = ret, + }; + + PROFILE_TIMESTAMP(&start_time); + + if (config->uring.ring_fd != -1) { + /* TODO: cancel queued and pending requests */ + TALLOC_FREE(config->fde); + io_uring_queue_exit(&config->uring); + config->uring.ring_fd = -1; + } + + PROFILE_TIMESTAMP(&end_time); + + for (cur = config->pending; cur != NULL; cur = next) { + next = cur->next; + err_cqe.user_data = (uintptr_t)(void *)cur; + vfs_io_uring_finish_req(cur, &err_cqe, end_time, location); + } + + for (cur = config->queue; cur != NULL; cur = next) { + next = cur->next; + err_cqe.user_data = (uintptr_t)(void *)cur; + cur->start_time = start_time; + vfs_io_uring_finish_req(cur, &err_cqe, end_time, location); + } +} + +static int vfs_io_uring_config_destructor(struct vfs_io_uring_config *config) +{ + vfs_io_uring_config_destroy(config, -EUCLEAN, __location__); + return 0; +} + +static int vfs_io_uring_request_state_deny_destructor(void *_state) +{ + struct __vfs_io_uring_generic_state { + struct vfs_io_uring_request ur; + } *state = (struct __vfs_io_uring_generic_state *)_state; + struct vfs_io_uring_request *cur = &state->ur; + + /* our parent is gone */ + cur->req = NULL; + + /* remove ourself from any list */ + DLIST_REMOVE((*cur->list_head), cur); + cur->list_head = NULL; + + /* + * Our state is about to go away, + * all we can do is shutting down the whole uring. + * But that's ok as we're most likely called from exit_server() + */ + vfs_io_uring_config_destroy(cur->config, -ESHUTDOWN, __location__); + return 0; +} + +static void vfs_io_uring_fd_handler(struct tevent_context *ev, + struct tevent_fd *fde, + uint16_t flags, + void *private_data); + +static int vfs_io_uring_connect(vfs_handle_struct *handle, const char *service, + const char *user) +{ + int ret; + struct vfs_io_uring_config *config; + unsigned num_entries; + bool sqpoll; + unsigned flags = 0; + + config = talloc_zero(handle->conn, struct vfs_io_uring_config); + if (config == NULL) { + DEBUG(0, ("talloc_zero() failed\n")); + return -1; + } + + SMB_VFS_HANDLE_SET_DATA(handle, config, + NULL, struct vfs_io_uring_config, + return -1); + + ret = SMB_VFS_NEXT_CONNECT(handle, service, user); + if (ret < 0) { + return ret; + } + + num_entries = lp_parm_ulong(SNUM(handle->conn), + "vfs_io_uring", + "num_entries", + 128); + num_entries = MAX(num_entries, 1); + + sqpoll = lp_parm_bool(SNUM(handle->conn), + "vfs_io_uring", + "sqpoll", + false); + if (sqpoll) { + flags |= IORING_SETUP_SQPOLL; + } + + ret = io_uring_queue_init(num_entries, &config->uring, flags); + if (ret < 0) { + SMB_VFS_NEXT_DISCONNECT(handle); + errno = -ret; + return -1; + } + + talloc_set_destructor(config, vfs_io_uring_config_destructor); + +#ifdef HAVE_IO_URING_RING_DONTFORK + ret = io_uring_ring_dontfork(&config->uring); + if (ret < 0) { + SMB_VFS_NEXT_DISCONNECT(handle); + errno = -ret; + return -1; + } +#endif /* HAVE_IO_URING_RING_DONTFORK */ + + config->fde = tevent_add_fd(handle->conn->sconn->ev_ctx, + config, + config->uring.ring_fd, + TEVENT_FD_READ, + vfs_io_uring_fd_handler, + handle); + if (config->fde == NULL) { + ret = errno; + SMB_VFS_NEXT_DISCONNECT(handle); + errno = ret; + return -1; + } + + return 0; +} + +static void vfs_io_uring_queue_run(struct vfs_io_uring_config *config) +{ + struct vfs_io_uring_request *cur = NULL, *next = NULL; + struct io_uring_cqe *cqe = NULL; + unsigned cqhead; + unsigned nr = 0; + struct timespec start_time; + struct timespec end_time; + int ret; + + PROFILE_TIMESTAMP(&start_time); + + if (config->uring.ring_fd == -1) { + vfs_io_uring_config_destroy(config, -ESTALE, __location__); + return; + } + + for (cur = config->queue; cur != NULL; cur = next) { + struct io_uring_sqe *sqe = NULL; + + next = cur->next; + + sqe = io_uring_get_sqe(&config->uring); + if (sqe == NULL) { + break; + } + + talloc_set_destructor(cur->state, + vfs_io_uring_request_state_deny_destructor); + DLIST_REMOVE(config->queue, cur); + *sqe = cur->sqe; + DLIST_ADD_END(config->pending, cur); + cur->list_head = &config->pending; + SMBPROFILE_BYTES_ASYNC_SET_BUSY(cur->profile_bytes); + + cur->start_time = start_time; + } + + ret = io_uring_submit(&config->uring); + if (ret == -EAGAIN || ret == -EBUSY) { + /* We just retry later */ + } else if (ret < 0) { + vfs_io_uring_config_destroy(config, ret, __location__); + return; + } + + PROFILE_TIMESTAMP(&end_time); + + io_uring_for_each_cqe(&config->uring, cqhead, cqe) { + cur = (struct vfs_io_uring_request *)io_uring_cqe_get_data(cqe); + vfs_io_uring_finish_req(cur, cqe, end_time, __location__); + nr++; + } + + io_uring_cq_advance(&config->uring, nr); +} + +static void vfs_io_uring_fd_handler(struct tevent_context *ev, + struct tevent_fd *fde, + uint16_t flags, + void *private_data) +{ + vfs_handle_struct *handle = (vfs_handle_struct *)private_data; + struct vfs_io_uring_config *config = NULL; + + SMB_VFS_HANDLE_GET_DATA(handle, config, + struct vfs_io_uring_config, + smb_panic(__location__)); + + vfs_io_uring_queue_run(config); +} + +struct vfs_io_uring_pread_state { + struct vfs_io_uring_request ur; + struct iovec iov; +}; + +static struct tevent_req *vfs_io_uring_pread_send(struct vfs_handle_struct *handle, + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct files_struct *fsp, + void *data, + size_t n, off_t offset) +{ + struct tevent_req *req = NULL; + struct vfs_io_uring_pread_state *state = NULL; + struct vfs_io_uring_config *config = NULL; + + SMB_VFS_HANDLE_GET_DATA(handle, config, + struct vfs_io_uring_config, + smb_panic(__location__)); + + req = tevent_req_create(mem_ctx, &state, + struct vfs_io_uring_pread_state); + if (req == NULL) { + return NULL; + } + state->ur.config = config; + state->ur.req = req; + state->ur.state = state; + + SMBPROFILE_BYTES_ASYNC_START(syscall_asys_pread, profile_p, + state->ur.profile_bytes, n); + SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes); + + state->iov.iov_base = (void *)data; + state->iov.iov_len = n; + io_uring_prep_readv(&state->ur.sqe, + fsp->fh->fd, + &state->iov, 1, + offset); + io_uring_sqe_set_data(&state->ur.sqe, &state->ur); + DLIST_ADD_END(config->queue, &state->ur); + state->ur.list_head = &config->queue; + + vfs_io_uring_queue_run(config); + + if (!tevent_req_is_in_progress(req)) { + return tevent_req_post(req, ev); + } + + tevent_req_defer_callback(req, ev); + return req; +} + +static ssize_t vfs_io_uring_pread_recv(struct tevent_req *req, + struct vfs_aio_state *vfs_aio_state) +{ + struct vfs_io_uring_pread_state *state = tevent_req_data( + req, struct vfs_io_uring_pread_state); + int ret; + + SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes); + vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time, + &state->ur.start_time); + + if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) { + return -1; + } + + if (state->ur.cqe.res < 0) { + vfs_aio_state->error = -state->ur.cqe.res; + ret = -1; + } else { + vfs_aio_state->error = 0; + ret = state->ur.cqe.res; + } + + tevent_req_received(req); + return ret; +} + +struct vfs_io_uring_pwrite_state { + struct vfs_io_uring_request ur; + struct iovec iov; +}; + +static struct tevent_req *vfs_io_uring_pwrite_send(struct vfs_handle_struct *handle, + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct files_struct *fsp, + const void *data, + size_t n, off_t offset) +{ + struct tevent_req *req = NULL; + struct vfs_io_uring_pwrite_state *state = NULL; + struct vfs_io_uring_config *config = NULL; + + SMB_VFS_HANDLE_GET_DATA(handle, config, + struct vfs_io_uring_config, + smb_panic(__location__)); + + req = tevent_req_create(mem_ctx, &state, + struct vfs_io_uring_pwrite_state); + if (req == NULL) { + return NULL; + } + state->ur.config = config; + state->ur.req = req; + state->ur.state = state; + + SMBPROFILE_BYTES_ASYNC_START(syscall_asys_pwrite, profile_p, + state->ur.profile_bytes, n); + SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes); + + state->iov.iov_base = discard_const(data); + state->iov.iov_len = n; + io_uring_prep_writev(&state->ur.sqe, + fsp->fh->fd, + &state->iov, 1, + offset); + io_uring_sqe_set_data(&state->ur.sqe, &state->ur); + DLIST_ADD_END(config->queue, &state->ur); + state->ur.list_head = &config->queue; + + vfs_io_uring_queue_run(config); + + if (!tevent_req_is_in_progress(req)) { + return tevent_req_post(req, ev); + } + + tevent_req_defer_callback(req, ev); + return req; +} + +static ssize_t vfs_io_uring_pwrite_recv(struct tevent_req *req, + struct vfs_aio_state *vfs_aio_state) +{ + struct vfs_io_uring_pwrite_state *state = tevent_req_data( + req, struct vfs_io_uring_pwrite_state); + int ret; + + SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes); + vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time, + &state->ur.start_time); + + if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) { + return -1; + } + + if (state->ur.cqe.res < 0) { + vfs_aio_state->error = -state->ur.cqe.res; + ret = -1; + } else { + vfs_aio_state->error = 0; + ret = state->ur.cqe.res; + } + + tevent_req_received(req); + return ret; +} + +struct vfs_io_uring_fsync_state { + struct vfs_io_uring_request ur; +}; + +static struct tevent_req *vfs_io_uring_fsync_send(struct vfs_handle_struct *handle, + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct files_struct *fsp) +{ + struct tevent_req *req = NULL; + struct vfs_io_uring_fsync_state *state = NULL; + struct vfs_io_uring_config *config = NULL; + + SMB_VFS_HANDLE_GET_DATA(handle, config, + struct vfs_io_uring_config, + smb_panic(__location__)); + + req = tevent_req_create(mem_ctx, &state, + struct vfs_io_uring_fsync_state); + if (req == NULL) { + return NULL; + } + state->ur.config = config; + state->ur.req = req; + state->ur.state = state; + + SMBPROFILE_BYTES_ASYNC_START(syscall_asys_fsync, profile_p, + state->ur.profile_bytes, 0); + SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes); + + io_uring_prep_fsync(&state->ur.sqe, + fsp->fh->fd, + 0); /* fsync_flags */ + io_uring_sqe_set_data(&state->ur.sqe, &state->ur); + DLIST_ADD_END(config->queue, &state->ur); + state->ur.list_head = &config->queue; + + vfs_io_uring_queue_run(config); + + if (!tevent_req_is_in_progress(req)) { + return tevent_req_post(req, ev); + } + + tevent_req_defer_callback(req, ev); + return req; +} + +static int vfs_io_uring_fsync_recv(struct tevent_req *req, + struct vfs_aio_state *vfs_aio_state) +{ + struct vfs_io_uring_fsync_state *state = tevent_req_data( + req, struct vfs_io_uring_fsync_state); + int ret; + + SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes); + vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time, + &state->ur.start_time); + + if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) { + return -1; + } + + if (state->ur.cqe.res < 0) { + vfs_aio_state->error = -state->ur.cqe.res; + ret = -1; + } else { + vfs_aio_state->error = 0; + ret = state->ur.cqe.res; + } + + tevent_req_received(req); + return ret; +} + +static struct vfs_fn_pointers vfs_io_uring_fns = { + .connect_fn = vfs_io_uring_connect, + .pread_send_fn = vfs_io_uring_pread_send, + .pread_recv_fn = vfs_io_uring_pread_recv, + .pwrite_send_fn = vfs_io_uring_pwrite_send, + .pwrite_recv_fn = vfs_io_uring_pwrite_recv, + .fsync_send_fn = vfs_io_uring_fsync_send, + .fsync_recv_fn = vfs_io_uring_fsync_recv, +}; + +static_decl_vfs; +NTSTATUS vfs_io_uring_init(TALLOC_CTX *ctx) +{ + return smb_register_vfs(SMB_VFS_INTERFACE_VERSION, + "io_uring", &vfs_io_uring_fns); +} diff --git a/source3/modules/wscript_build b/source3/modules/wscript_build index cb0c083fcc8..41d8568e43a 100644 --- a/source3/modules/wscript_build +++ b/source3/modules/wscript_build @@ -396,6 +396,14 @@ bld.SAMBA3_MODULE('vfs_aio_pthread', internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_aio_pthread'), enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_aio_pthread')) +bld.SAMBA3_MODULE('vfs_io_uring', + subsystem='vfs', + source='vfs_io_uring.c', + deps='samba-util tevent uring', + init_function='', + internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_io_uring'), + enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_io_uring')) + bld.SAMBA3_MODULE('vfs_preopen', subsystem='vfs', source='vfs_preopen.c', diff --git a/source3/wscript b/source3/wscript index 62fe8ea7d59..85466b493fa 100644 --- a/source3/wscript +++ b/source3/wscript @@ -1754,6 +1754,14 @@ main() { and conf.CHECK_LIB('dbus-1', shlib=True)): conf.DEFINE('HAVE_DBUS', '1') + if conf.CHECK_CFG(package='liburing', args='--cflags --libs', + msg='Checking for liburing package', uselib_store="URING"): + if (conf.CHECK_HEADERS('liburing.h', lib='uring') + and conf.CHECK_LIB('uring', shlib=True)): + conf.CHECK_FUNCS_IN('io_uring_ring_dontfork', 'uring', + headers='liburing.h') + conf.DEFINE('HAVE_LIBURING', '1') + conf.env.build_regedit = False if not Options.options.with_regedit == False: conf.PROCESS_SEPARATE_RULE('system_ncurses') @@ -1932,6 +1940,9 @@ main() { if (conf.CONFIG_SET('HAVE_STRUCT_MSGHDR_MSG_CONTROL') or conf.CONFIG_SET('HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTS')): default_shared_modules.extend(TO_LIST('vfs_aio_fork')) + if conf.CONFIG_SET('HAVE_LIBURING'): + default_shared_modules.extend(TO_LIST('vfs_io_uring')) + if Options.options.with_pthreadpool: default_shared_modules.extend(TO_LIST('vfs_aio_pthread')) -- 2.17.1