From 155dacc34bc8f089a068511ef69245c969c43559 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Wed, 29 May 2024 12:14:04 +1200 Subject: [PATCH 1/4] pidl:Typelist: resolveType(): don't mistake a reference for a name This function is only used by Python.pm, and was assuming any argument unrecognised by hasType is a name. It sometimes isn't, resulting in structures like this: { 'DATA' => { 'TYPE' => 'STRUCT' }, 'NAME' => { 'TYPE' => 'STRUCT', 'ALIGN' => undef, 'SURROUNDING_ELEMENT' => undef, 'ORIGINAL' => { 'TYPE' => 'STRUCT', 'FILE' => 'source3/librpc/idl/smbXsrv.idl', 'LINE' => 101, 'NAME' => 'tevent_context' }, 'ELEMENTS' => undef, 'NAME' => 'tevent_context', 'PROPERTIES' => undef }, 'TYPE' => 'TYPEDEF' }; The problem with that is we end up with the HASH reference as a name in Python bindings, like this PyErr_SetString(PyExc_TypeError, "Can not convert C Type struct HASH(0x5e2dfe5ee278) from Python"); which makes the build nondeterministic (as well as making the message a little mysterious). I think all the structures for which this happens are marked '[ignore]' in IDL, meaning they are not transmitted on the wire. They should perhaps also not have useless Python getsetters, but let's call that a different problem. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit f3433f60b8ee83fc785a6e6838513de31bff5a6a) --- pidl/lib/Parse/Pidl/Typelist.pm | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pidl/lib/Parse/Pidl/Typelist.pm b/pidl/lib/Parse/Pidl/Typelist.pm index 2b850ffee4b..c37d6114bd1 100644 --- a/pidl/lib/Parse/Pidl/Typelist.pm +++ b/pidl/lib/Parse/Pidl/Typelist.pm @@ -120,8 +120,18 @@ sub resolveType($) my ($ctype) = @_; if (not hasType($ctype)) { - # assume struct typedef - return { TYPE => "TYPEDEF", NAME => $ctype, DATA => { TYPE => "STRUCT" } }; + if (! ref $ctype) { + # it looks like a name. + # assume struct typedef + return { TYPE => "TYPEDEF", NAME => $ctype, DATA => { TYPE => "STRUCT" } }; + } + if ($ctype->{NAME} && ($ctype->{TYPE} eq "STRUCT")) { + return { + TYPE => "TYPEDEF", + NAME => $ctype->{NAME}, + DATA => $ctype + }; + } } else { return getType($ctype); } -- 2.34.1 From 23e312ff284fef9f243cce5dacb7b36d26f7e4c8 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 12:17:06 +1200 Subject: [PATCH 2/4] buildtools:pidl: avoid hash randomisation in pidl Like many languages, Perl uses has randomisation to prevent nasty users using crafted values that hash to the same number to effect a denial of service. This means the traversal order of perl HASH tables is different every time. The IDL handed to pidl is trusted, so we don't really need randomisation, but we do want to be sure the build is the same every time. I am not aware of hash randomisation causing problems, but it seems prudent to avoid it. We do a similar thing with PYTHONHASHSEED for the entire build. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit 068b366709d005275727a0e8929d272c04cb7bd8) --- buildtools/wafsamba/samba_pidl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildtools/wafsamba/samba_pidl.py b/buildtools/wafsamba/samba_pidl.py index 0ce860ca6ae..2b079392870 100644 --- a/buildtools/wafsamba/samba_pidl.py +++ b/buildtools/wafsamba/samba_pidl.py @@ -81,7 +81,9 @@ def SAMBA_PIDL(bld, pname, source, else: cc = 'CC="%s"' % bld.CONFIG_GET("CC") - t = bld(rule='cd ${PIDL_LAUNCH_DIR} && %s%s %s ${PERL} ${PIDL} --quiet ${OPTIONS} --outputdir ${OUTPUTDIR} -- "${IDLSRC}"' % (pidl_dev, cpp, cc), + t = bld(rule=('cd ${PIDL_LAUNCH_DIR} && PERL_HASH_SEED=0 %s%s %s ${PERL} ' + '${PIDL} --quiet ${OPTIONS} --outputdir ${OUTPUTDIR} -- "${IDLSRC}"' % + (pidl_dev, cpp, cc)), ext_out = '.c', before = 'c', update_outputs = True, -- 2.34.1 From ec2b3e5515f259d1cef4f0897685a0f0d414abfc Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 11:34:51 +1200 Subject: [PATCH 3/4] examples:winexe: reproducible builds with zero timestamp Windows Portable Executable files have a timestamp field and a checksum field. By default the timestamp field is updated to the current time, which consequently changes the checksum. This makes the build nondeterministic. It looks like this: --- a/tmp/winexe-1/winexesvc64_exe_binary.c +++ b/tmp/winexe-2/winexesvc64_exe_binary.c @@ -23,7 +23,7 @@ const DATA_BLOB *winexesvc64_exe_binary(void) 0x6D, 0x6F, 0x64, 0x65, 0x2E, 0x0D, 0x0D, 0x0A, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x45, 0x00, 0x00, 0x64, 0x86, 0x0A, 0x00, - 0xB2, 0x16, 0x55, 0x66, 0x00, 0x00, 0x00, 0x00, + 0xD3, 0x3B, 0x55, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x00, 0x2E, 0x02, 0x0B, 0x02, 0x02, 0x26, 0x00, 0x86, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, @@ -33,7 +33,7 @@ const DATA_BLOB *winexesvc64_exe_binary(void) 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, - 0x73, 0xD7, 0x00, 0x00, 0x03, 0x00, 0x60, 0x01, + 0x94, 0xFC, 0x00, 0x00, 0x03, 0x00, 0x60, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, https://learn.microsoft.com/en-us/windows/win32/debug/pe-format says that a timestamp of zero can be used to represent a time that is not "real or meaningful", so we do that. As far as I can tell, the timestamp and checksum are only used in DLLs, not directly executed .exe files. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (backported from commit e604f7575167d3572e1b67c6e77ab7273508533d) [dbagnall@samba.org: adapted for wscript_build differences] --- examples/winexe/wscript | 14 ++++++++++++++ examples/winexe/wscript_build | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/winexe/wscript b/examples/winexe/wscript index 6b311b1da41..a202282e765 100644 --- a/examples/winexe/wscript +++ b/examples/winexe/wscript @@ -1,4 +1,6 @@ #!/usr/bin/env python +import os + def configure(conf): AR32 = ['i386', 'i586', 'i686'] @@ -27,5 +29,17 @@ def configure(conf): conf.DEFINE('HAVE_WINEXE_CC_WIN64', 1); break + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + if source_date_epoch is None: + # We set the timestamp to be 0, which makes the winexe build + # reproducible. According to + # https://learn.microsoft.com/en-us/windows/win32/debug/pe-format + # + # > If the stamp value is 0 or 0xFFFFFFFF, it does not + # > represent a real or meaningful date/time stamp. + + source_date_epoch = '0' + + conf.env.SOURCE_DATE_EPOCH = source_date_epoch conf.DEFINE("WINEXE_LDFLAGS", "-s -Wall -Wl,-Bstatic -Wl,-Bdynamic -luserenv") diff --git a/examples/winexe/wscript_build b/examples/winexe/wscript_build index 143739f3de0..bb454159e08 100644 --- a/examples/winexe/wscript_build +++ b/examples/winexe/wscript_build @@ -70,7 +70,7 @@ if bld.env.WINEXE_CC_WIN32: 'winexesvc32_exe', source='winexesvc.c', target='winexesvc32.exe', - rule='${WINEXE_CC_WIN32} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}') + rule='SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} ${WINEXE_CC_WIN32} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}') vars = {"WINEXE_FN": "winexesvc32_exe_binary"} bld.SAMBA_GENERATOR( 'winexesvc32_exe_binary', @@ -86,7 +86,7 @@ if bld.env.WINEXE_CC_WIN64: 'winexesvc64_exe', source='winexesvc.c', target='winexesvc64.exe', - rule='${WINEXE_CC_WIN64} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}') + rule='SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} ${WINEXE_CC_WIN64} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}') vars = {"WINEXE_FN": "winexesvc64_exe_binary"} bld.SAMBA_GENERATOR( 'winexesvc64_exe_binary', -- 2.34.1 From 86d249275e18224f7d3367f7fc0d2bd93c659ec2 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Thu, 30 May 2024 10:44:24 +1200 Subject: [PATCH 4/4] examples:winexe: embed Samba version as exe timestamp It turns out the timestamp doesn't need to be real, and it isn't used, but it might as well tell you something. So let's make it tell you what version of Samba it came from, which could be useful for people who have lots of old winexes lying around, the poor souls. 00000040 0e 1f ba 0e 00 b4 09 cd 21 b8 01 4c cd 21 54 68 |........!..L.!Th| 00000050 69 73 20 70 72 6f 67 72 61 6d 20 63 61 6e 6e 6f |is program canno| 00000060 74 20 62 65 20 72 75 6e 20 69 6e 20 44 4f 53 20 |t be run in DOS | 00000070 6d 6f 64 65 2e 0d 0d 0a 24 00 00 00 00 00 00 00 |mode....$.......| 00000080 50 45 00 00 64 86 0a 00 00 15 04 00 00 00 00 00 |PE..d...........| | | | | | major 4. | minor 21. release 0 BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett Autobuild-User(master): Douglas Bagnall Autobuild-Date(master): Fri May 31 01:28:06 UTC 2024 on atb-devel-224 (cherry picked from commit 3a7dbf8b77b2a9e7cdc55bc5b339b9f501d037aa) --- examples/winexe/wscript | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/examples/winexe/wscript b/examples/winexe/wscript index a202282e765..c4f13b89f01 100644 --- a/examples/winexe/wscript +++ b/examples/winexe/wscript @@ -31,14 +31,21 @@ def configure(conf): source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') if source_date_epoch is None: - # We set the timestamp to be 0, which makes the winexe build - # reproducible. According to - # https://learn.microsoft.com/en-us/windows/win32/debug/pe-format + # We use the version to make up the timestamp that will be + # embedded in winexe.exe, to keep the build reproducible. # - # > If the stamp value is 0 or 0xFFFFFFFF, it does not - # > represent a real or meaningful date/time stamp. - - source_date_epoch = '0' + # This is less evil than it sounds. According to Raymond Chen in + # https://devblogs.microsoft.com/oldnewthing/20180103-00/?p=97705 + # since Windows 10 the timestamp has been randomised. + # + # The purpose of the timestamp in Windows PE files seems to be + # to make spotting ABI changes in DLLs quicker, for which a + # random number is just as good as a real time. The timestamp + # in .exe files is not used. + import samba_version + v = samba_version.load_version(env=conf.env) + version = (v.MAJOR << 16) | (v.MINOR << 8) | v.RELEASE + source_date_epoch = str(version) conf.env.SOURCE_DATE_EPOCH = source_date_epoch conf.DEFINE("WINEXE_LDFLAGS", -- 2.34.1