/* * Copyright 2022-2024 by Marc Aurèle La France, tsi@tuyoix.net * * Permission to use, copy, modify, distribute, and sell this software and its * documentation for any purpose is hereby granted without fee, provided that * the above copyright notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting documentation, and * that the name of Marc Aurèle La France not be used in advertising or * publicity pertaining to distribution of the software without specific, * written prior permission. Marc Aurèle La France makes no representations * about the suitability of this software for any purpose. It is provided * "as-is" without express or implied warranty. * * MARC AURÈLE LA FRANCE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO * EVENT SHALL MARC AURÈLE LA FRANCE BE LIABLE FOR ANY SPECIAL, INDIRECT OR * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE * OF THIS SOFTWARE. */ /* $TSI$ */ /* * Post-process `rsync --out-format='-%i %n%L'` stdout to report file * transactions in ascending pathname order, except that directory deletions * (or replacements) are reported after the deletion of their contents. This * also strips out interim progress lines given deletions may be reported among * them. The file counts at the end of progress lines are ignored as they no * longer make sense after reordering transactions. * * This does not handle total transfer progress lines (i.e. --info=progress2, * specified or implied), nor interim progress lines generated by sending a * signal to rsync. Probably other things too. * * The hyphen in front of the out-format makes locating %i output simpler and * more reliable. * * This should be statically linked to maximise heap space. Out-of-memory * handling here almost always results in slightly incorrect ordering instead * of giving up. * * This is a reaction to https://bugzilla.samba.org/show_bug.cgi?id=6741 which * will never be addressed in rsync itself and probably shouldn't be. */ /* * Revision history (from rsync's perspective): * - Clone stripcr, a filter that omits interim progress lines. * - Reorder deletions. * - Handle escaping of unusual filenames. * - Compensate for sort of directory contents that places non-directories * ahead of subdirectories. * - Improve hardlink and symlink handling. * - Report deletion of directory contents before the directory itself, even if * the latter is replaced with another file type. */ /* * Known issues: * - Non-deletion of hardlink names containing the " => " string. * - Non-deletion of symlink names or target names containing the " -> " * string. */ #undef _GNU_SOURCE #define _GNU_SOURCE 1 /* For reallocarray() & mempcpy() */ #include #include #include #include #include #include #undef DELETING #define DELETING "-*deleting " #undef FN_OFF #define FN_OFF 13 /* Where %n starts in the out-format */ #undef MASK #define MASK ((sizeof(long double) << 1) - 1) /* malloc() alignment */ #undef UNAVAILABLE #define UNAVAILABLE ((((size_t)(-1LL)) / sizeof(transaction_t)) + 1) #undef UNLIMITED #define UNLIMITED ((const char *)(-1LL)) /* * Until gcc quits whining about casts and assignments to void * pointers that * discard qualifiers. If it ever does... After all, void data cannot be * modified, are therefore implicitly const and no other qualifiers matter. */ #ifndef voidconst #define voidconst /* const */ #endif typedef struct { voidconst char *encoded; voidconst char *decoded; double rate; unsigned int hours; unsigned char minutes; unsigned char seconds; char notime; /* Enough for 64-bit values in decimal with commas and trailing NUL */ char filesize[27]; char percent; /* Deletions are more than 100% */ char scale; } transaction_t; static transaction_t *Transactions; /* Maintained in descending order */ static size_t iTransaction; static size_t nTransaction, nTransactions; static size_t sTransaction = UNAVAILABLE; static voidconst char *Transaction, *transaction; static const char *line, *p, *prefix = ""; static char *buffer; static size_t start, len; static char percent; static bool decoded; static size_t notices; static void report(void) { printf("\n%zu notice%s issued!\n", notices, (notices == 1) ? "" : "s"); } /* No sense in second-guessing malloc() internals. So-o-o-o... */ static void die(void) { report(); puts("\n!!Premature termination.\n"); fflush(stdout); /* Just in case */ /* Take down rsync as well */ kill(0, SIGTERM); /* No return ... */ exit(SIGTERM + 128); /* ... supposedly ... */ } static void printdeletion(const char *pathname) { fputs(DELETING, stdout); fputs(pathname, stdout); } static void printtransaction(void) { voidconst char *Encoded = Transactions[nTransaction].encoded; if (*prefix != '\0') { fputs(prefix, stdout); prefix = ""; } if (Transactions[nTransaction].percent > 100) { if (Encoded != Transactions[nTransaction].decoded) free(Transactions[nTransaction].decoded); printdeletion(Encoded); } else { if (sTransaction == nTransaction) sTransaction = UNAVAILABLE; if ((Encoded + FN_OFF) != Transactions[nTransaction].decoded) free(Transactions[nTransaction].decoded); fputs(Encoded, stdout); if (Transactions[nTransaction].percent == 100) { printf("%15s 100%% %7.2f%cB/s ", Transactions[nTransaction].filesize, Transactions[nTransaction].rate, Transactions[nTransaction].scale); if (Transactions[nTransaction].notime == '?') puts(" ??:??:??"); else printf("%4u:%02hhu:%02hhu\n", Transactions[nTransaction].hours, Transactions[nTransaction].minutes, Transactions[nTransaction].seconds); } } free(Encoded); } static void flushtransactions(void) { while (nTransaction > 0) { nTransaction--; printtransaction(); } } static bool outofmemory(size_t size) { transaction_t *newTransactions; /* Last chance before creating more chaos */ if (((nTransactions - nTransaction) * sizeof(transaction_t)) >= (size + (MASK + 1 + sizeof(transaction_t)))) { Transactions = reallocarray(Transactions, nTransaction + 1, sizeof(transaction_t)); nTransactions = nTransaction + 1; prefix = ""; return false; } if (nTransaction > 0) { nTransaction--; printtransaction(); return false; } prefix = ""; if (nTransactions > 1) { Transactions = realloc(Transactions, sizeof(transaction_t)); nTransactions = 1; return false; } if ((newTransactions = malloc(sizeof(transaction_t))) == NULL) return true; if ((nTransactions == 0) || (newTransactions < Transactions)) { free(Transactions); Transactions = newTransactions; nTransactions = 1; return false; } free(newTransactions); return true; } static void inserttransaction(void) { if (nTransaction >= nTransactions) { transaction_t *saveTransactions = Transactions; if ((Transactions = reallocarray(Transactions, ++nTransactions, sizeof(transaction_t))) == NULL) { if (--nTransactions == 0) die(); notices++; prefix = "!I"; Transactions = saveTransactions; nTransaction--; printtransaction(); } } /* Binary search candidate */ for (iTransaction = nTransaction; iTransaction-- > 0; ) { int res = strncmp(transaction, Transactions[iTransaction].decoded, len); if ((res < 0) || ((res == 0) && (p == NULL) && /* Non-deletion */ (Transactions[iTransaction].decoded[len] != '/'))) break; Transactions[iTransaction + 1] = Transactions[iTransaction]; } iTransaction++; Transactions[iTransaction].encoded = Transaction; Transactions[iTransaction].decoded = transaction; nTransaction++; } static voidconst char *decode(const char *enil, voidconst char *pathname) { const char *src, *limit = UNLIMITED; char *dst, *filename; decoded = true; if (enil[1] == 'h') /* Hard link */ { if (((limit = strstr(pathname, " => ")) == NULL) || (strstr(limit + 4, " => ") != NULL)) { limit = UNLIMITED; notices++; printf("!line=%s", enil); } } else if (enil[2] == 'L') /* Symbolic link */ { if (((limit = strstr(pathname, " -> ")) == NULL) || (strstr(limit + 4, " -> ") != NULL)) { limit = UNLIMITED; notices++; printf("!line=%s", enil); } } for (src = pathname; (src = strstr(src, "\\#")) != NULL; src += 2) { if (src >= limit) break; if (((src[2] & 0xfc) != '0') || ((src[3] & 0xf8) != '0') || ((src[4] & 0xf8) != '0')) continue; len = strlen(pathname + ((5 - 1) - 1)); while ((filename = malloc(len)) == NULL) { notices++; prefix = "!P"; if (outofmemory(len)) { decoded = false; return pathname; } } dst = mempcpy(filename, pathname, src - pathname); while (true) { *dst++ = (src[2] << 6) | ((src[3] & 7) << 3) | (src[4] & 7); src += 5; while ((src[0] != '\\') || (src[1] != '#') || ((src[2] & 0xfc) != '0') || ((src[3] & 0xf8) != '0') || ((src[4] & 0xf8) != '0')) { if ((*dst++ = *src++) != '\0') { if (src < limit) continue; *dst++ = '\n'; *dst++ = '\0'; } len = (len & ~MASK) + !!(len & MASK); if ((filename + len) <= dst) return filename; return realloc(filename, dst - filename); } } } if (limit == UNLIMITED) return pathname; len = limit - pathname; while ((filename = malloc(len + 2)) == NULL) { notices++; prefix = "!Q"; if (outofmemory(len + 2)) { decoded = false; return pathname; } } dst = mempcpy(filename, pathname, len); *dst++ = '\n'; *dst++ = '\0'; return filename; } static bool isdeletion(void) { transaction_t temp; if ((p = strstr(line, DELETING)) == NULL) return false; if (p == line) return true; /* Validate the single interim progress line that precedes this deletion */ if (sscanf(line, " %26[0-9,.kKMGTPE]%hhd%%%lf%cB/s%*[ ]%zn%c", temp.filesize, &temp.percent, &temp.rate, &temp.scale, &len, &temp.notime) == 5) { if (temp.notime == '?') { if (!strncmp(line + len, "??:??:?? " DELETING, 10 + strlen(DELETING))) return true; } else { if ((sscanf(line + len, "%u:%2hhu:%2hhu%*[ ]" DELETING "%c", &temp.hours, &temp.minutes, &temp.seconds, &temp.notime) == 4) && (temp.minutes < 60) && (temp.seconds < 60)) return true; } } p = NULL; return false; } static bool printabletransaction(void) { if (nTransaction == 0) return false; if (line[2] == 'd') /* Directory */ return true; if (sTransaction >= UNAVAILABLE) return false; if (strcmp(transaction, Transactions[sTransaction].decoded) <= 0) return true; return false; } static bool invalidprogress(void) { if (Transactions[sTransaction].notime != '?') { size_t length; if (sscanf(line + len, "%u:%2hhu:%2hhu%zn", &Transactions[sTransaction].hours, &Transactions[sTransaction].minutes, &Transactions[sTransaction].seconds, &length) != 3) return true; length += len; while (line[length] == ' ') length++; if ((line[length] != '\n') && (line[length] != '(')) return true; if ((Transactions[sTransaction].minutes >= 60) || (Transactions[sTransaction].seconds >= 60)) return true; } Transactions[sTransaction].percent = percent; return false; } static void print(size_t end) { char savec = buffer[end]; buffer[end] = '\0'; line = buffer + start; /* * Deletions may insert themselves ahead of other transfers in the same * directory or between iterations of progress lines. Buffer them so they * can be reinserted in the desired order. */ if (isdeletion()) { while ((Transaction = strdup(p + FN_OFF)) == NULL) { notices++; prefix = "!D"; if (outofmemory(strlen(p + (FN_OFF - 1)))) { fputs("!d", stdout); printdeletion(p + FN_OFF); goto done; } } transaction = decode(p, Transaction); len = strlen(transaction); if ((len > 2) && (transaction[len - 2] == '/')) len--; /* Directory; don't compare last newline */ inserttransaction(); Transactions[iTransaction].percent = SCHAR_MAX; if (iTransaction <= sTransaction) sTransaction++; } else if (*line == '-') /* -%i */ { transaction = decode(line, line + FN_OFF); if (printabletransaction()) { while (nTransaction-- > 0) { if (strcmp(transaction, Transactions[nTransaction].decoded) < 0) break; printtransaction(); } nTransaction++; } while ((Transaction = strdup(line)) == NULL) { notices++; prefix = "!N"; if (outofmemory(strlen(line) + 1)) die(); } if (transaction == (line + FN_OFF)) { if (decoded) transaction = Transaction + FN_OFF; else transaction = decode(line, Transaction + FN_OFF); } len = strlen(transaction); if ((len > 1) && (line[2] != 'd')) len--; /* Non-directory; don't compare last newline */ inserttransaction(); Transactions[iTransaction].percent = 0; sTransaction = iTransaction; } else if ((sTransaction < UNAVAILABLE) && (sscanf(line, " %26[0-9,.kKMGTPE]%hhd%%%lf%cB/s%*[ ]%zn%c", Transactions[sTransaction].filesize, &percent, &Transactions[sTransaction].rate, &Transactions[sTransaction].scale, &len, &Transactions[sTransaction].notime) == 5)) { if (invalidprogress()) { notices++; printf("!line=%s", line); } } else { flushtransactions(); fputs(line, stdout); } done: buffer[end] = savec; } int main(void) { static size_t length; /* Make `tail -f` sane */ (void) setlinebuf(stdout); #ifdef M_ARENA_MAX /* Use a single arena */ (void) mallopt(M_ARENA_MAX, 1); #endif #ifdef M_MMAP_MAX /* Disable use of mmap() to avoid wasteful pagesize alignments */ (void) mallopt(M_MMAP_MAX, 0); #endif #ifdef M_MXFAST /* Don't use "fastbins" */ (void) mallopt(M_MXFAST, 0); #endif while (true) { static size_t next, size; if (next >= length) { static int eof; size_t requested, completed; if (eof) break; if (start > 0) { if ((next -= start) > 0) /* Need to special-case */ (void) memmove(buffer, buffer + start, next); start = 0; } else if (size == length) { static size_t sz; char *savebuffer = buffer; while ((buffer = realloc(buffer, sz += 8192)) == NULL) { size_t need = sz; notices++; prefix = "!B"; sz -= 8192; buffer = savebuffer; if (outofmemory(need)) { if (sz == 0) die(); print(length); start = next = 0; break; } } /* Allow for print()'s temporary NUL-termination */ size = sz - 1; } requested = size - next; completed = fread(buffer + next, 1, requested, stdin); length = next + completed; if (completed < requested) { if (completed <= 0) break; eof = feof(stdin) | ferror(stdin); } } switch (buffer[next++]) { case '\n': print(next); /* Fall through */ case '\r': start = next; /* Fall through */ default: break; } } if (start < length) print(length); flushtransactions(); free(Transactions); free(buffer); report(); return !!notices; }