From c5359004b3198d0c98222481e6726a4278cd4b24 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 27 Feb 2020 23:58:42 +0100 Subject: [PATCH 1/5] script/autobuild.py: add support git worktree .git is not always a directory, with 'git worktree' it's a file. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit f9374d0a4ecc11acc46884feec28d138accc6dab) --- script/autobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/autobuild.py b/script/autobuild.py index f5cf2a85ca09..890059fe3424 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -37,7 +37,7 @@ def find_git_root(): '''get to the top of the git repo''' p = os.getcwd() while p != '/': - if os.path.isdir(os.path.join(p, ".git")): + if os.path.exists(os.path.join(p, ".git")): return p p = os.path.abspath(os.path.join(p, '..')) return None -- 2.25.1 From 230ffee26931c8ca35cdb5e16c80b567a816d611 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 27 Feb 2020 23:59:00 +0100 Subject: [PATCH 2/5] script/autobuild.py: use more portable 'cp -R -a -l' On platforms like FreeBSD 12 cp doesn't accept the long options, using the one letter options works there and keeps working on Linux as well. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 0312a10e09d8dc1295a4a80493761e91031e88e7) --- script/autobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/autobuild.py b/script/autobuild.py index 890059fe3424..f71e0266503c 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -848,7 +848,7 @@ class builder(object): run_cmd("rm -rf %s" % self.test_source_dir) run_cmd("rm -rf %s" % self.prefix) if cp: - run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) + run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) else: run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) self.start_next() -- 2.25.1 From 9f84115d111c3173fa77599f2f01354f122550af Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 28 Feb 2020 00:00:08 +0100 Subject: [PATCH 3/5] script/autobuild.py: allow write_system_info commands to fail These commands are just there as hints to debug possible problems. In order to support autobuild.py on non-linux platforms we should just ignore errors here. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Thu Apr 2 07:36:07 UTC 2020 on sn-devel-184 (cherry picked from commit 9b1e96197e0983a16e73ce351eac7775801736d8) --- script/autobuild.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/autobuild.py b/script/autobuild.py index f71e0266503c..e2778609cceb 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -979,7 +979,10 @@ class buildlist(object): 'cc --version', 'df -m .', 'df -m %s' % testbase]: - out = run_cmd(cmd, output=True, checkfail=False) + try: + out = run_cmd(cmd, output=True, checkfail=False) + except subprocess.CalledProcessError as e: + out = "" % str(e) print('### %s' % cmd, file=f) print(out, file=f) print(file=f) -- 2.25.1 From 5b9e2d314fe0a7322ae3bf1438467c31c2cea4f0 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 4/5] script/autobuild.py: split out a rmdir_force() helper function That also tries to re-add write permissions before removing. In future we'll have jobs changing there directory to read-only. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 7a5df2deaaf62a7edd7c64251f75ab15abe94c07) (cherry picked from commit c933135969be29072971f96481b05f499fd48b57) --- script/autobuild.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index e2778609cceb..6436a55b7d8c 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -4,7 +4,7 @@ # released under GNU GPL v3 or later from __future__ import print_function -from subprocess import call, check_call, check_output, Popen, PIPE +from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError import os import tarfile import sys @@ -823,6 +823,17 @@ def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): else: return call(cmd, shell=True, cwd=dir) +def rmdir_force(dirname, re_raise=True): + try: + run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % ( + dirname, dirname, dirname), output=True, show=True) + except CalledProcessError as e: + do_print("Failed: '%s'" % (str(e))) + run_cmd("tree %s" % dirname, output=True, show=True) + if re_raise: + raise + return False + return True class builder(object): '''handle build of one directory''' @@ -845,8 +856,8 @@ class builder(object): self.test_source_dir = "%s/%s" % (testbase, self.tag) self.cwd = "%s/%s" % (self.test_source_dir, self.dir) self.prefix = "%s/%s" % (test_prefix, self.tag) - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) if cp: run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) else: @@ -856,8 +867,8 @@ class builder(object): def start_next(self): if self.next == len(self.sequence): if not options.nocleanup: - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) do_print('%s: Completed OK' % self.name) self.done = True return @@ -981,7 +992,7 @@ class buildlist(object): 'df -m %s' % testbase]: try: out = run_cmd(cmd, output=True, checkfail=False) - except subprocess.CalledProcessError as e: + except CalledProcessError as e: out = "" % str(e) print('### %s' % cmd, file=f) print(out, file=f) @@ -1018,7 +1029,7 @@ def cleanup(): run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - run_cmd("rm -rf %s" % d) + rmdir_force(d) def daemonize(logfile): -- 2.25.1 From 3a63375a87209f2ae39b541cb744311900c0e259 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 5/5] script/autobuild.py: let cleanup() ignore errors from rmdir_force() by default It's not useful to generate a python backtrace from within the cleanup code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 9883ac45939f253a63f3ff312fc3912c5f02cdac) Autobuild-User(v4-14-test): Karolin Seeger Autobuild-Date(v4-14-test): Tue Feb 2 10:29:44 UTC 2021 on sn-devel-184 (cherry picked from commit cc1568be4d4250390a9ad03c84f5e260fc7acffd) --- script/autobuild.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index 6436a55b7d8c..b1a1b4800d1e 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -1022,14 +1022,23 @@ class buildlist(object): self.tail_proc = Popen(cmd, close_fds=True) -def cleanup(): +def cleanup(do_raise=False): if options.nocleanup: return run_cmd("stat %s || true" % test_tmpdir, show=True) run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - rmdir_force(d) + ok = rmdir_force(d, re_raise=False) + if ok: + continue + if os.path.isdir(d): + do_print("Killing, waiting and retry") + run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False) + else: + do_print("Waiting and retry") + time.sleep(1) + rmdir_force(d, re_raise=do_raise) def daemonize(logfile): @@ -1295,7 +1304,7 @@ while True: (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() if status != 0 or errstr != "retry": break - cleanup() + cleanup(do_raise=True) except Exception: cleanup() raise -- 2.25.1