From b668b46df99c7768cb357d60201431dfd5560973 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Wed, 29 May 2024 12:14:04 +1200 Subject: [PATCH 1/7] pidl:Typelist: resolveType(): don't mistake a reference for a name This function is only used by Python.pm, and was assuming any argument unrecognised by hasType is a name. It sometimes isn't, resulting in structures like this: { 'DATA' => { 'TYPE' => 'STRUCT' }, 'NAME' => { 'TYPE' => 'STRUCT', 'ALIGN' => undef, 'SURROUNDING_ELEMENT' => undef, 'ORIGINAL' => { 'TYPE' => 'STRUCT', 'FILE' => 'source3/librpc/idl/smbXsrv.idl', 'LINE' => 101, 'NAME' => 'tevent_context' }, 'ELEMENTS' => undef, 'NAME' => 'tevent_context', 'PROPERTIES' => undef }, 'TYPE' => 'TYPEDEF' }; The problem with that is we end up with the HASH reference as a name in Python bindings, like this PyErr_SetString(PyExc_TypeError, "Can not convert C Type struct HASH(0x5e2dfe5ee278) from Python"); which makes the build nondeterministic (as well as making the message a little mysterious). I think all the structures for which this happens are marked '[ignore]' in IDL, meaning they are not transmitted on the wire. They should perhaps also not have useless Python getsetters, but let's call that a different problem. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit f3433f60b8ee83fc785a6e6838513de31bff5a6a) --- pidl/lib/Parse/Pidl/Typelist.pm | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pidl/lib/Parse/Pidl/Typelist.pm b/pidl/lib/Parse/Pidl/Typelist.pm index 31ea19e357c..2a98a16b2b5 100644 --- a/pidl/lib/Parse/Pidl/Typelist.pm +++ b/pidl/lib/Parse/Pidl/Typelist.pm @@ -138,8 +138,18 @@ sub resolveType($) my ($ctype) = @_; if (not hasType($ctype)) { - # assume struct typedef - return { TYPE => "TYPEDEF", NAME => $ctype, DATA => { TYPE => "STRUCT" } }; + if (! ref $ctype) { + # it looks like a name. + # assume struct typedef + return { TYPE => "TYPEDEF", NAME => $ctype, DATA => { TYPE => "STRUCT" } }; + } + if ($ctype->{NAME} && ($ctype->{TYPE} eq "STRUCT")) { + return { + TYPE => "TYPEDEF", + NAME => $ctype->{NAME}, + DATA => $ctype + }; + } } else { return getType($ctype); } -- 2.25.1 From 6c3cad3bee9798396fe9ad682c9bb21373ea7a4e Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 16:32:14 +1200 Subject: [PATCH 2/7] pidl:python: properly raise exception in ConvertObjectFromPythonData Without the `$self->pidl("$fail");`, the exception is not raised. We also try to improve the Python message. Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit f893c100c0743e52b578d2108222c26321805132) --- pidl/lib/Parse/Pidl/Samba4/Python.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pidl/lib/Parse/Pidl/Samba4/Python.pm b/pidl/lib/Parse/Pidl/Samba4/Python.pm index c09ee4b784d..056abcfb130 100644 --- a/pidl/lib/Parse/Pidl/Samba4/Python.pm +++ b/pidl/lib/Parse/Pidl/Samba4/Python.pm @@ -1829,7 +1829,8 @@ sub ConvertObjectFromPythonData($$$$$$;$$) my $ctype_name = $self->use_type_variable($ctype); unless (defined ($ctype_name)) { error($location, "Unable to determine origin of type `" . mapTypeName($ctype) . "'"); - $self->pidl("PyErr_SetString(PyExc_TypeError, \"Can not convert C Type " . mapTypeName($ctype) . " from Python\");"); + $self->pidl("PyErr_SetString(PyExc_TypeError, \"Cannot convert Python object to NDR $target\");"); + $self->pidl("$fail"); return; } $self->pidl("PY_CHECK_TYPE($ctype_name, $cvar, $fail);"); -- 2.25.1 From 2f27a030814375ee5c1e516b5912d1a0738ffe85 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Thu, 30 May 2024 10:53:24 +1200 Subject: [PATCH 3/7] pidl:python: Exception if unconvertable in ConvertObjectToPythonLevel Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit ea78a5ce8301ecea3c8901d96944ad06744535e4) --- pidl/lib/Parse/Pidl/Samba4/Python.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pidl/lib/Parse/Pidl/Samba4/Python.pm b/pidl/lib/Parse/Pidl/Samba4/Python.pm index 056abcfb130..103b5ba2ae6 100644 --- a/pidl/lib/Parse/Pidl/Samba4/Python.pm +++ b/pidl/lib/Parse/Pidl/Samba4/Python.pm @@ -2296,6 +2296,10 @@ sub ConvertObjectToPythonLevel($$$$$$$) } my $conv = $self->ConvertObjectToPythonData($mem_ctx, $l->{DATA_TYPE}, $var_name, $e->{ORIGINAL}); $self->pidl("$py_var = $conv;"); + if ($conv eq "NULL") { + $self->pidl("PyErr_SetString(PyExc_NotImplementedError, \"Cannot convert NDR $var_name to Python\");"); + $self->pidl("$fail"); + } } elsif ($l->{TYPE} eq "SUBCONTEXT") { $self->ConvertObjectToPythonLevel($mem_ctx, $env, $e, $nl, $var_name, $py_var, $fail, $recurse); } else { -- 2.25.1 From a6138588ae7af50b26ed495ac605533cc063b310 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 12:17:06 +1200 Subject: [PATCH 4/7] buildtools:pidl: avoid hash randomisation in pidl Like many languages, Perl uses has randomisation to prevent nasty users using crafted values that hash to the same number to effect a denial of service. This means the traversal order of perl HASH tables is different every time. The IDL handed to pidl is trusted, so we don't really need randomisation, but we do want to be sure the build is the same every time. I am not aware of hash randomisation causing problems, but it seems prudent to avoid it. We do a similar thing with PYTHONHASHSEED for the entire build. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit 068b366709d005275727a0e8929d272c04cb7bd8) --- buildtools/wafsamba/samba_pidl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildtools/wafsamba/samba_pidl.py b/buildtools/wafsamba/samba_pidl.py index 72997c8bf84..e1010869cdd 100644 --- a/buildtools/wafsamba/samba_pidl.py +++ b/buildtools/wafsamba/samba_pidl.py @@ -81,7 +81,9 @@ def SAMBA_PIDL(bld, pname, source, else: cc = 'CC="%s"' % bld.CONFIG_GET("CC") - t = bld(rule='cd ${PIDL_LAUNCH_DIR} && %s%s %s ${PERL} ${PIDL} --quiet ${OPTIONS} --outputdir ${OUTPUTDIR} -- "${IDLSRC}"' % (pidl_dev, cpp, cc), + t = bld(rule=('cd ${PIDL_LAUNCH_DIR} && PERL_HASH_SEED=0 %s%s %s ${PERL} ' + '${PIDL} --quiet ${OPTIONS} --outputdir ${OUTPUTDIR} -- "${IDLSRC}"' % + (pidl_dev, cpp, cc)), ext_out = '.c', before = 'c', update_outputs = True, -- 2.25.1 From 6acf69f99df2bbdb9fa77999888fc440331bc2f3 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 11:16:23 +1200 Subject: [PATCH 5/7] examples:winexe: more efficient C array generation, no py2 We don't need to recreate the src array every time, and we don't need to worry about Python 2 strings. Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit 3c72f733f459088c8d506973610f3d8530592bc2) --- examples/winexe/wscript_build | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/examples/winexe/wscript_build b/examples/winexe/wscript_build index 364683405c2..a9de3473948 100644 --- a/examples/winexe/wscript_build +++ b/examples/winexe/wscript_build @@ -17,23 +17,12 @@ def generate_winexesvc_c_from_exe(t): return -1 def c_array(src): - N = 0 - result = '' - while src: - l = src[:8] - src = src[8:] - # Even files opened in binary mode are read as type "str" in - # Python 2, so we need to get the integer ordinal of each - # character in the string before we try to convert it to hex. - if isinstance(l, str): - h = ' '.join(["0x%02X," % ord(x) for x in l]) - # Files opened in binary mode are read as type "bytes" in - # Python 3, so we can convert each individual integer in the - # array of bytes to hex directly. - else: - h = ' '.join(["0x%02X," % x for x in l]) - result += "\t\t%s\n" % (h) - return result + result = [] + for i in range(0, len(src), 8): + l = src[i:i+8] + h = ' '.join(["0x%02X," % x for x in l]) + result.append(h) + return "\n\t\t".join(result) src_array = c_array(src_blob) if len(src_array) <= 0: @@ -48,7 +37,7 @@ const DATA_BLOB *%s(void); const DATA_BLOB *%s(void) { \tstatic const uint8_t array[] = { -%s +\t\t%s \t}; \tstatic const DATA_BLOB blob = { \t\t.data = discard_const_p(uint8_t, array), -- 2.25.1 From f14f89b366200d41833219387d8c74569760659f Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Tue, 28 May 2024 11:34:51 +1200 Subject: [PATCH 6/7] examples:winexe: reproducible builds with zero timestamp Windows Portable Executable files have a timestamp field and a checksum field. By default the timestamp field is updated to the current time, which consequently changes the checksum. This makes the build nondeterministic. It looks like this: --- a/tmp/winexe-1/winexesvc64_exe_binary.c +++ b/tmp/winexe-2/winexesvc64_exe_binary.c @@ -23,7 +23,7 @@ const DATA_BLOB *winexesvc64_exe_binary(void) 0x6D, 0x6F, 0x64, 0x65, 0x2E, 0x0D, 0x0D, 0x0A, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x45, 0x00, 0x00, 0x64, 0x86, 0x0A, 0x00, - 0xB2, 0x16, 0x55, 0x66, 0x00, 0x00, 0x00, 0x00, + 0xD3, 0x3B, 0x55, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x00, 0x2E, 0x02, 0x0B, 0x02, 0x02, 0x26, 0x00, 0x86, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, @@ -33,7 +33,7 @@ const DATA_BLOB *winexesvc64_exe_binary(void) 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, - 0x73, 0xD7, 0x00, 0x00, 0x03, 0x00, 0x60, 0x01, + 0x94, 0xFC, 0x00, 0x00, 0x03, 0x00, 0x60, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, https://learn.microsoft.com/en-us/windows/win32/debug/pe-format says that a timestamp of zero can be used to represent a time that is not "real or meaningful", so we do that. As far as I can tell, the timestamp and checksum are only used in DLLs, not directly executed .exe files. Thanks to Freexian and the Debian LTS project for sponsoring this work. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett (cherry picked from commit e604f7575167d3572e1b67c6e77ab7273508533d) --- examples/winexe/wscript | 14 ++++++++++++++ examples/winexe/wscript_build | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/winexe/wscript b/examples/winexe/wscript index 6b311b1da41..a202282e765 100644 --- a/examples/winexe/wscript +++ b/examples/winexe/wscript @@ -1,4 +1,6 @@ #!/usr/bin/env python +import os + def configure(conf): AR32 = ['i386', 'i586', 'i686'] @@ -27,5 +29,17 @@ def configure(conf): conf.DEFINE('HAVE_WINEXE_CC_WIN64', 1); break + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + if source_date_epoch is None: + # We set the timestamp to be 0, which makes the winexe build + # reproducible. According to + # https://learn.microsoft.com/en-us/windows/win32/debug/pe-format + # + # > If the stamp value is 0 or 0xFFFFFFFF, it does not + # > represent a real or meaningful date/time stamp. + + source_date_epoch = '0' + + conf.env.SOURCE_DATE_EPOCH = source_date_epoch conf.DEFINE("WINEXE_LDFLAGS", "-s -Wall -Wl,-Bstatic -Wl,-Bdynamic -luserenv") diff --git a/examples/winexe/wscript_build b/examples/winexe/wscript_build index a9de3473948..faec560bc88 100644 --- a/examples/winexe/wscript_build +++ b/examples/winexe/wscript_build @@ -58,7 +58,7 @@ bld.SAMBA_GENERATOR( 'winexesvc32_exe', source='winexesvc.c', target='winexesvc32.exe', - rule='${WINEXE_CC_WIN32} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}', + rule='SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} ${WINEXE_CC_WIN32} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}', enabled=bld.env.build_winexe and bld.env.WINEXE_CC_WIN32) vars = {"WINEXE_FN": "winexesvc32_exe_binary"} @@ -78,7 +78,7 @@ bld.SAMBA_GENERATOR( 'winexesvc64_exe', source='winexesvc.c', target='winexesvc64.exe', - rule='${WINEXE_CC_WIN64} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}', + rule='SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} ${WINEXE_CC_WIN64} ${SRC} -o ${TGT} ${WINEXE_LDFLAGS}', enabled=bld.env.build_winexe and bld.env.WINEXE_CC_WIN64) vars = {"WINEXE_FN": "winexesvc64_exe_binary"} -- 2.25.1 From 49485df4aab456cabd4a2002d408a62f9cb6ebc0 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Thu, 30 May 2024 10:44:24 +1200 Subject: [PATCH 7/7] examples:winexe: embed Samba version as exe timestamp It turns out the timestamp doesn't need to be real, and it isn't used, but it might as well tell you something. So let's make it tell you what version of Samba it came from, which could be useful for people who have lots of old winexes lying around, the poor souls. 00000040 0e 1f ba 0e 00 b4 09 cd 21 b8 01 4c cd 21 54 68 |........!..L.!Th| 00000050 69 73 20 70 72 6f 67 72 61 6d 20 63 61 6e 6e 6f |is program canno| 00000060 74 20 62 65 20 72 75 6e 20 69 6e 20 44 4f 53 20 |t be run in DOS | 00000070 6d 6f 64 65 2e 0d 0d 0a 24 00 00 00 00 00 00 00 |mode....$.......| 00000080 50 45 00 00 64 86 0a 00 00 15 04 00 00 00 00 00 |PE..d...........| | | | | | major 4. | minor 21. release 0 BUG: https://bugzilla.samba.org/show_bug.cgi?id=13213 Signed-off-by: Douglas Bagnall Reviewed-by: Andrew Bartlett Autobuild-User(master): Douglas Bagnall Autobuild-Date(master): Fri May 31 01:28:06 UTC 2024 on atb-devel-224 (cherry picked from commit 3a7dbf8b77b2a9e7cdc55bc5b339b9f501d037aa) --- examples/winexe/wscript | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/examples/winexe/wscript b/examples/winexe/wscript index a202282e765..c4f13b89f01 100644 --- a/examples/winexe/wscript +++ b/examples/winexe/wscript @@ -31,14 +31,21 @@ def configure(conf): source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') if source_date_epoch is None: - # We set the timestamp to be 0, which makes the winexe build - # reproducible. According to - # https://learn.microsoft.com/en-us/windows/win32/debug/pe-format + # We use the version to make up the timestamp that will be + # embedded in winexe.exe, to keep the build reproducible. # - # > If the stamp value is 0 or 0xFFFFFFFF, it does not - # > represent a real or meaningful date/time stamp. - - source_date_epoch = '0' + # This is less evil than it sounds. According to Raymond Chen in + # https://devblogs.microsoft.com/oldnewthing/20180103-00/?p=97705 + # since Windows 10 the timestamp has been randomised. + # + # The purpose of the timestamp in Windows PE files seems to be + # to make spotting ABI changes in DLLs quicker, for which a + # random number is just as good as a real time. The timestamp + # in .exe files is not used. + import samba_version + v = samba_version.load_version(env=conf.env) + version = (v.MAJOR << 16) | (v.MINOR << 8) | v.RELEASE + source_date_epoch = str(version) conf.env.SOURCE_DATE_EPOCH = source_date_epoch conf.DEFINE("WINEXE_LDFLAGS", -- 2.25.1