diff --git a/doc/languages-frameworks/emscripten.section.md b/doc/languages-frameworks/emscripten.section.md index 20d358f2e9e3..9ce48db2c2de 100644 --- a/doc/languages-frameworks/emscripten.section.md +++ b/doc/languages-frameworks/emscripten.section.md @@ -86,9 +86,9 @@ One advantage is that when `pkgs.zlib` is updated, it will automatically update postPatch = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' substituteInPlace configure \ - --replace '/usr/bin/libtool' 'ar' \ - --replace 'AR="libtool"' 'AR="ar"' \ - --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"' + --replace-fail '/usr/bin/libtool' 'ar' \ + --replace-fail 'AR="libtool"' 'AR="ar"' \ + --replace-fail 'ARFLAGS="-o"' 'ARFLAGS="-r"' ''; }) ``` diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index d18b048b911b..6ba2463e150e 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -700,7 +700,7 @@ with import {}; hello = attrs: lib.optionalAttrs (lib.versionAtLeast attrs.version "1.0") { postPatch = '' substituteInPlace lib/zoneinfo.rs \ - --replace "/usr/share/zoneinfo" "${tzdata}/share/zoneinfo" + --replace-fail "/usr/share/zoneinfo" "${tzdata}/share/zoneinfo" ''; }; }; diff --git a/doc/stdenv/platform-notes.chapter.md b/doc/stdenv/platform-notes.chapter.md index b47f5af349b8..409c9f2e7b2e 100644 --- a/doc/stdenv/platform-notes.chapter.md +++ b/doc/stdenv/platform-notes.chapter.md @@ -54,7 +54,7 @@ Some common issues when packaging software for Darwin: # ... prePatch = '' substituteInPlace Makefile \ - --replace '/usr/bin/xcrun clang' clang + --replace-fail '/usr/bin/xcrun clang' clang ''; } ``` diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 03bb8a9ff790..7f62af5997b5 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -230,9 +230,9 @@ stdenv.mkDerivation rec { postInstall = '' substituteInPlace $out/bin/solo5-virtio-mkimage \ - --replace "/usr/lib/syslinux" "${syslinux}/share/syslinux" \ - --replace "/usr/share/syslinux" "${syslinux}/share/syslinux" \ - --replace "cp " "cp --no-preserve=mode " + --replace-fail "/usr/lib/syslinux" "${syslinux}/share/syslinux" \ + --replace-fail "/usr/share/syslinux" "${syslinux}/share/syslinux" \ + --replace-fail "cp " "cp --no-preserve=mode " wrapProgram $out/bin/solo5-virtio-mkimage \ --prefix PATH : ${lib.makeBinPath [ dosfstools mtools parted syslinux ]} @@ -1217,9 +1217,20 @@ postInstall = '' Performs string substitution on the contents of \, writing the result to \. The substitutions in \ are of the following form: -#### `--replace` \ \ {#fun-substitute-replace} +#### `--replace-fail` \ \ {#fun-substitute-replace-fail} Replace every occurrence of the string \ by \. +Will error if no change is made. + +#### `--replace-warn` \ \ {#fun-substitute-replace-warn} + +Replace every occurrence of the string \ by \. +Will print a warning if no change is made. + +#### `--replace-quiet` \ \ {#fun-substitute-replace-quiet} + +Replace every occurrence of the string \ by \. +Will do nothing if no change can be made. #### `--subst-var` \ {#fun-substitute-subst-var} @@ -1233,8 +1244,8 @@ Example: ```shell substitute ./foo.in ./foo.out \ - --replace /usr/bin/bar $bar/bin/bar \ - --replace "a string containing spaces" "some other text" \ + --replace-fail /usr/bin/bar $bar/bin/bar \ + --replace-fail "a string containing spaces" "some other text" \ --subst-var someVar ``` diff --git a/pkgs/applications/editors/emacs/sources.nix b/pkgs/applications/editors/emacs/sources.nix index 622498c9dcb6..c8f80ef9fe6c 100644 --- a/pkgs/applications/editors/emacs/sources.nix +++ b/pkgs/applications/editors/emacs/sources.nix @@ -73,6 +73,27 @@ in variant = "mainline"; rev = "28.2"; hash = "sha256-4oSLcUDR0MOEt53QOiZSVU8kPJ67GwugmBxdX3F15Ag="; + patches = fetchpatch: [ + # CVE-2022-45939 + (fetchpatch { + url = "https://git.savannah.gnu.org/cgit/emacs.git/patch/?id=d48bb4874bc6cd3e69c7a15fc3c91cc141025c51"; + hash = "sha256-TiBQkexn/eb6+IqJNDqR/Rn7S7LVdHmL/21A5tGsyJs="; + }) + + # https://lists.gnu.org/archive/html/emacs-devel/2024-03/msg00611.html + (fetchpatch { + url = "https://gitweb.gentoo.org/proj/emacs-patches.git/plain/emacs/28.2/10_all_org-macro-eval.patch?id=af40e12cb742510e5d40a06ffc6dfca97e340dd6"; + hash = "sha256-OdGt4e9JGjWJPkfJhbYsmQQc6jart4BH5aIKPIbWKFs="; + }) + (fetchpatch { + url = "https://gitweb.gentoo.org/proj/emacs-patches.git/plain/emacs/28.2/11_all_untrusted-content.patch?id=af40e12cb742510e5d40a06ffc6dfca97e340dd6"; + hash = "sha256-wa2bsnCt5yFx0+RAFZGBPI+OoKkbrfkkMer/KBEc/wA="; + }) + (fetchpatch { + url = "https://gitweb.gentoo.org/proj/emacs-patches.git/plain/emacs/28.2/12_all_org-remote-unsafe.patch?id=af40e12cb742510e5d40a06ffc6dfca97e340dd6"; + hash = "sha256-b6WU1o3PfDV/6BTPfPNUFny6oERJCNsDrvflxX3Yvek="; + }) + ]; }); emacs29 = import ./make-emacs.nix (mkArgs { diff --git a/pkgs/development/compilers/go/1.21.nix b/pkgs/development/compilers/go/1.21.nix index 771ab0f0f04c..e793cfebbe41 100644 --- a/pkgs/development/compilers/go/1.21.nix +++ b/pkgs/development/compilers/go/1.21.nix @@ -46,11 +46,11 @@ let in stdenv.mkDerivation (finalAttrs: { pname = "go"; - version = "1.21.8"; + version = "1.21.9"; src = fetchurl { url = "https://go.dev/dl/go${finalAttrs.version}.src.tar.gz"; - hash = "sha256-3IBs91qH4UFLW0w9y53T6cyY9M/M7EK3r2F9WmWKPEM="; + hash = "sha256-WPDFztRaABK84v96nfA+Eoq8yIGOur5QJ7uSuv4g5CE="; }; strictDeps = true; diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index cb6406652d46..a02c5a259398 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -20,10 +20,10 @@ sourceVersion = { major = "3"; minor = "10"; - patch = "13"; + patch = "14"; suffix = ""; }; - hash = "sha256-XIiEhmhkDT4VKzW0U27xwjsspL0slX7x7LsFP1cd0/Y="; + hash = "sha256-nFBIH6qMKDIym6D8iGjQpgamgPxPYOxI0mzo4HZ1H9o="; }; python311 = { diff --git a/pkgs/development/libraries/at-spi2-core/default.nix b/pkgs/development/libraries/at-spi2-core/default.nix index 7af9edd26865..a0f0888bf30c 100644 --- a/pkgs/development/libraries/at-spi2-core/default.nix +++ b/pkgs/development/libraries/at-spi2-core/default.nix @@ -23,13 +23,13 @@ stdenv.mkDerivation rec { pname = "at-spi2-core"; - version = "2.50.0"; + version = "2.50.2"; outputs = [ "out" "dev" ]; src = fetchurl { url = "mirror://gnome/sources/${pname}/${lib.versions.majorMinor version}/${pname}-${version}.tar.xz"; - sha256 = "6fWoyCNcndljshcd6RIDARKcZ33ekzlV4d9hi5ScStw="; + hash = "sha256-W4GxRhpi3Y++0aJ2+p71txEvmuX/huHjKtlkS2VP94w="; }; nativeBuildInputs = [ diff --git a/pkgs/development/libraries/audio/libopenmpt/default.nix b/pkgs/development/libraries/audio/libopenmpt/default.nix index b8b89abc8ea7..79f853ef5532 100644 --- a/pkgs/development/libraries/audio/libopenmpt/default.nix +++ b/pkgs/development/libraries/audio/libopenmpt/default.nix @@ -16,13 +16,13 @@ stdenv.mkDerivation rec { pname = "libopenmpt"; - version = "0.7.3"; + version = "0.7.6"; outputs = [ "out" "dev" "bin" ]; src = fetchurl { url = "https://lib.openmpt.org/files/libopenmpt/src/libopenmpt-${version}+release.autotools.tar.gz"; - hash = "sha256-LPg2m3kWsJJk8/FLn7bO81pum+4DKN7E9J2YIRzP1yI="; + hash = "sha256-Fi1yowa7LhFMJPolJn0NCgrBbzn9laXA38daZm7l5PU="; }; enableParallelBuilding = true; diff --git a/pkgs/development/libraries/avahi/default.nix b/pkgs/development/libraries/avahi/default.nix index 772650bd3a8a..271ea1b47ff8 100644 --- a/pkgs/development/libraries/avahi/default.nix +++ b/pkgs/development/libraries/avahi/default.nix @@ -51,6 +51,74 @@ stdenv.mkDerivation rec { url = "https://github.com/lathiat/avahi/commit/a2696da2f2c50ac43b6c4903f72290d5c3fa9f6f.patch"; sha256 = "sha256-BEYFGCnQngp+OpiKIY/oaKygX7isAnxJpUPCUvg+efc="; }) + # CVE-2023-38470 + # https://github.com/lathiat/avahi/pull/457 merged Sep 19 + (fetchpatch { + name = "CVE-2023-38470.patch"; + url = "https://github.com/lathiat/avahi/commit/94cb6489114636940ac683515417990b55b5d66c.patch"; + sha256 = "sha256-Fanh9bvz+uknr5pAmltqijuUAZIG39JR2Lyq5zGKJ58="; + }) + # https://github.com/avahi/avahi/pull/480 merged Sept 19 + (fetchpatch { + name = "bail-out-unless-escaped-labels-fit.patch"; + url = "https://github.com/avahi/avahi/commit/20dec84b2480821704258bc908e7b2bd2e883b24.patch"; + sha256 = "sha256-p/dOuQ/GInIcUwuFhQR3mGc5YBL5J8ho+1gvzcqEN0c="; + }) + # CVE-2023-38473 + # https://github.com/lathiat/avahi/pull/486 merged Oct 18 + (fetchpatch { + name = "CVE-2023-38473.patch"; + url = "https://github.com/lathiat/avahi/commit/b448c9f771bada14ae8de175695a9729f8646797.patch"; + sha256 = "sha256-/ZVhsBkf70vjDWWG5KXxvGXIpLOZUXdRkn3413iSlnI="; + }) + # CVE-2023-38472 + # https://github.com/lathiat/avahi/pull/490 merged Oct 19 + (fetchpatch { + name = "CVE-2023-38472.patch"; + url = "https://github.com/lathiat/avahi/commit/b024ae5749f4aeba03478e6391687c3c9c8dee40.patch"; + sha256 = "sha256-FjR8fmhevgdxR9JQ5iBLFXK0ILp2OZQ8Oo9IKjefCqk="; + }) + # CVE-2023-38471 + # https://github.com/lathiat/avahi/pull/494 merged Oct 24 + (fetchpatch { + name = "CVE-2023-38471.patch"; + url = "https://github.com/lathiat/avahi/commit/894f085f402e023a98cbb6f5a3d117bd88d93b09.patch"; + sha256 = "sha256-4dG+5ZHDa+A4/CszYS8uXWlpmA89m7/jhbZ7rheMs7U="; + }) + # https://github.com/lathiat/avahi/pull/499 merged Oct 25 + (fetchpatch { + name = "CVE-2023-38471-2.patch"; + url = "https://github.com/avahi/avahi/commit/b675f70739f404342f7f78635d6e2dcd85a13460.patch"; + sha256 = "sha256-uDtMPWuz1lsu7n0Co/Gpyh369miQ6GWGyC0UPQB/yI8="; + }) + # CVE-2023-38469 + # https://github.com/lathiat/avahi/pull/500 merged Oct 25 + (fetchpatch { + name = "CVE-2023-38469.patch"; + url = "https://github.com/avahi/avahi/commit/61b9874ff91dd20a12483db07df29fe7f35db77f.patch"; + sha256 = "sha256-qR7scfQqhRGxg2n4HQsxVxCLkXbwZi+PlYxrOSEPsL0="; + excludes = [ ".github/workflows/smoke-tests.sh" ]; + }) + # https://github.com/avahi/avahi/pull/515 merged Nov 3 + (fetchpatch { + name = "fix-compare-rrs-with-zero-length-rdata.patch"; + url = "https://github.com/avahi/avahi/commit/177d75e8c43be45a8383d794ce4084dd5d600a9e.patch"; + sha256 = "sha256-uwIyruAWgiWt0yakRrvMdYjjhEhUk5cIGKt6twyXbHw="; + }) + # https://github.com/avahi/avahi/pull/519 merged Nov 8 + (fetchpatch { + name = "reject-non-utf-8-service-names.patch"; + url = "https://github.com/avahi/avahi/commit/2b6d3e99579e3b6e9619708fad8ad8e07ada8218.patch"; + sha256 = "sha256-lwSA3eEQgH0g51r0i9/HJMJPRXrhQnTIEDxcYqUuLdI="; + excludes = [ "fuzz/fuzz-domain.c" ]; + }) + # https://github.com/avahi/avahi/pull/523 merged Nov 12 + (fetchpatch { + name = "core-no-longer-supply-bogus-services-to-callbacks.patch"; + url = "https://github.com/avahi/avahi/commit/93b14365c1c1e04efd1a890e8caa01a2a514bfd8.patch"; + sha256 = "sha256-VBm8vsBZkTbbWAK8FI71SL89lZuYd1yFNoB5o+FvlEU="; + excludes = [ ".github/workflows/smoke-tests.sh" "fuzz/fuzz-packet.c" ]; + }) ]; depsBuildBuild = [ diff --git a/pkgs/development/libraries/expat/CVE-2023-52425.patch b/pkgs/development/libraries/expat/CVE-2023-52425.patch new file mode 100644 index 000000000000..741535fd2128 --- /dev/null +++ b/pkgs/development/libraries/expat/CVE-2023-52425.patch @@ -0,0 +1,1450 @@ +commit 678a2f7efcaaa977886e055613f2332615aef82c +Author: Tomas Korbar +Date: Tue Feb 13 13:52:28 2024 +0100 + + Fix CVE-2023-52425 + +diff --git a/doc/reference.html b/doc/reference.html +index 8b0d47d..a10f3cb 100644 +--- a/doc/reference.html ++++ b/doc/reference.html +@@ -151,10 +151,11 @@ interface.

+ + +
  • +- Billion Laughs Attack Protection ++ Attack Protection + +
  • +
  • Miscellaneous Functions +@@ -2096,11 +2097,7 @@ parse position may be before the beginning of the buffer.

    + return NULL.

    + + +-

    Billion Laughs Attack Protection

    +- +-

    The functions in this section configure the built-in +- protection against various forms of +- billion laughs attacks.

    ++

    Attack Protection

    + +

    XML_SetBillionLaughsAttackProtectionMaximumAmplification

    +
    +@@ -2188,6 +2185,27 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold(XML_Parser p,
    +   

    + + ++

    XML_SetReparseDeferralEnabled

    ++
    ++/* Added in Expat 2.6.0. */
    ++XML_Bool XMLCALL
    ++XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled);
    ++
    ++
    ++

    ++ Large tokens may require many parse calls before enough data is available for Expat to parse it in full. ++ If Expat retried parsing the token on every parse call, parsing could take quadratic time. ++ To avoid this, Expat only retries once a significant amount of new data is available. ++ This function allows disabling this behavior. ++

    ++

    ++ The enabled argument should be XML_TRUE or XML_FALSE. ++

    ++

    ++ Returns XML_TRUE on success, and XML_FALSE on error. ++

    ++
    ++ +

    Miscellaneous functions

    + +

    The functions in this section either obtain state information from +diff --git a/doc/xmlwf.xml b/doc/xmlwf.xml +index 9603abf..3d35393 100644 +--- a/doc/xmlwf.xml ++++ b/doc/xmlwf.xml +@@ -313,6 +313,16 @@ supports both. + + + ++ ++ ++ ++ ++ Disable reparse deferral, and allow quadratic parse runtime ++ on large tokens (default: reparse deferral enabled). ++ ++ ++ ++ + + + +diff --git a/lib/expat.h b/lib/expat.h +index 1c83563..842dd70 100644 +--- a/lib/expat.h ++++ b/lib/expat.h +@@ -16,6 +16,7 @@ + Copyright (c) 2016 Thomas Beutlich + Copyright (c) 2017 Rhodri James + Copyright (c) 2022 Thijs Schreijer ++ Copyright (c) 2023 Sony Corporation / Snild Dolkow + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining +@@ -1050,6 +1051,10 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + #endif + ++/* Added in Expat 2.6.0. */ ++XMLPARSEAPI(XML_Bool) ++XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); ++ + /* Expat follows the semantic versioning convention. + See http://semver.org. + */ +diff --git a/lib/internal.h b/lib/internal.h +index e09f533..e2709c8 100644 +--- a/lib/internal.h ++++ b/lib/internal.h +@@ -31,6 +31,7 @@ + Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2018 Yury Gribov + Copyright (c) 2019 David Loffredo ++ Copyright (c) 2023 Sony Corporation / Snild Dolkow + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining +@@ -160,6 +161,9 @@ unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); + const char *unsignedCharToPrintable(unsigned char c); + #endif + ++extern XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c ++extern unsigned int g_parseAttempts; // used for testing only ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/libexpat.def.cmake b/lib/libexpat.def.cmake +index cf434a2..3ff4d55 100644 +--- a/lib/libexpat.def.cmake ++++ b/lib/libexpat.def.cmake +@@ -77,3 +77,4 @@ EXPORTS + ; added with version 2.4.0 + @_EXPAT_COMMENT_DTD@ XML_SetBillionLaughsAttackProtectionActivationThreshold @69 + @_EXPAT_COMMENT_DTD@ XML_SetBillionLaughsAttackProtectionMaximumAmplification @70 ++XML_SetReparseDeferralEnabled @71 +diff --git a/lib/xmlparse.c b/lib/xmlparse.c +index b6c2eca..2ae64e9 100644 +--- a/lib/xmlparse.c ++++ b/lib/xmlparse.c +@@ -73,6 +73,7 @@ + # endif + #endif + ++#include + #include + #include /* memset(), memcpy() */ + #include +@@ -196,6 +197,8 @@ typedef char ICHAR; + /* Do safe (NULL-aware) pointer arithmetic */ + #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) + ++#define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) ++ + #include "internal.h" + #include "xmltok.h" + #include "xmlrole.h" +@@ -602,6 +605,9 @@ static unsigned long getDebugLevel(const char *variableName, + ? 0 \ + : ((*((pool)->ptr)++ = c), 1)) + ++XML_Bool g_reparseDeferralEnabledDefault = XML_TRUE; // write ONLY in runtests.c ++unsigned int g_parseAttempts = 0; // used for testing only ++ + struct XML_ParserStruct { + /* The first member must be m_userData so that the XML_GetUserData + macro works. */ +@@ -617,6 +623,9 @@ struct XML_ParserStruct { + const char *m_bufferLim; + XML_Index m_parseEndByteIndex; + const char *m_parseEndPtr; ++ size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ ++ XML_Bool m_reparseDeferralEnabled; ++ int m_lastBufferRequestSize; + XML_Char *m_dataBuf; + XML_Char *m_dataBufEnd; + XML_StartElementHandler m_startElementHandler; +@@ -948,6 +957,47 @@ get_hash_secret_salt(XML_Parser parser) { + return parser->m_hash_secret_salt; + } + ++static enum XML_Error ++callProcessor(XML_Parser parser, const char *start, const char *end, ++ const char **endPtr) { ++ const size_t have_now = EXPAT_SAFE_PTR_DIFF(end, start); ++ ++ if (parser->m_reparseDeferralEnabled ++ && ! parser->m_parsingStatus.finalBuffer) { ++ // Heuristic: don't try to parse a partial token again until the amount of ++ // available data has increased significantly. ++ const size_t had_before = parser->m_partialTokenBytesBefore; ++ // ...but *do* try anyway if we're close to causing a reallocation. ++ size_t available_buffer ++ = EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); ++#if XML_CONTEXT_BYTES > 0 ++ available_buffer -= EXPAT_MIN(available_buffer, XML_CONTEXT_BYTES); ++#endif ++ available_buffer ++ += EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd); ++ // m_lastBufferRequestSize is never assigned a value < 0, so the cast is ok ++ const bool enough ++ = (have_now >= 2 * had_before) ++ || ((size_t)parser->m_lastBufferRequestSize > available_buffer); ++ ++ if (! enough) { ++ *endPtr = start; // callers may expect this to be set ++ return XML_ERROR_NONE; ++ } ++ } ++ g_parseAttempts += 1; ++ const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); ++ if (ret == XML_ERROR_NONE) { ++ // if we consumed nothing, remember what we had on this parse attempt. ++ if (*endPtr == start) { ++ parser->m_partialTokenBytesBefore = have_now; ++ } else { ++ parser->m_partialTokenBytesBefore = 0; ++ } ++ } ++ return ret; ++} ++ + static XML_Bool /* only valid for root parser */ + startParsing(XML_Parser parser) { + /* hash functions must be initialized before setContext() is called */ +@@ -1129,6 +1179,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { + parser->m_bufferEnd = parser->m_buffer; + parser->m_parseEndByteIndex = 0; + parser->m_parseEndPtr = NULL; ++ parser->m_partialTokenBytesBefore = 0; ++ parser->m_reparseDeferralEnabled = g_reparseDeferralEnabledDefault; ++ parser->m_lastBufferRequestSize = 0; + parser->m_declElementType = NULL; + parser->m_declAttributeId = NULL; + parser->m_declEntity = NULL; +@@ -1298,6 +1351,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, + to worry which hash secrets each table has. + */ + unsigned long oldhash_secret_salt; ++ XML_Bool oldReparseDeferralEnabled; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) +@@ -1342,6 +1396,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, + to worry which hash secrets each table has. + */ + oldhash_secret_salt = parser->m_hash_secret_salt; ++ oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; + + #ifdef XML_DTD + if (! context) +@@ -1394,6 +1449,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, + parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; + parser->m_ns_triplets = oldns_triplets; + parser->m_hash_secret_salt = oldhash_secret_salt; ++ parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; + parser->m_parentParser = oldParser; + #ifdef XML_DTD + parser->m_paramEntityParsing = oldParamEntityParsing; +@@ -1848,55 +1904,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + parser->m_parsingStatus.parsing = XML_PARSING; + } + +- if (len == 0) { +- parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; +- if (! isFinal) +- return XML_STATUS_OK; +- parser->m_positionPtr = parser->m_bufferPtr; +- parser->m_parseEndPtr = parser->m_bufferEnd; +- +- /* If data are left over from last buffer, and we now know that these +- data are the final chunk of input, then we have to check them again +- to detect errors based on that fact. +- */ +- parser->m_errorCode +- = parser->m_processor(parser, parser->m_bufferPtr, +- parser->m_parseEndPtr, &parser->m_bufferPtr); +- +- if (parser->m_errorCode == XML_ERROR_NONE) { +- switch (parser->m_parsingStatus.parsing) { +- case XML_SUSPENDED: +- /* It is hard to be certain, but it seems that this case +- * cannot occur. This code is cleaning up a previous parse +- * with no new data (since len == 0). Changing the parsing +- * state requires getting to execute a handler function, and +- * there doesn't seem to be an opportunity for that while in +- * this circumstance. +- * +- * Given the uncertainty, we retain the code but exclude it +- * from coverage tests. +- * +- * LCOV_EXCL_START +- */ +- XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, +- parser->m_bufferPtr, &parser->m_position); +- parser->m_positionPtr = parser->m_bufferPtr; +- return XML_STATUS_SUSPENDED; +- /* LCOV_EXCL_STOP */ +- case XML_INITIALIZED: +- case XML_PARSING: +- parser->m_parsingStatus.parsing = XML_FINISHED; +- /* fall through */ +- default: +- return XML_STATUS_OK; +- } +- } +- parser->m_eventEndPtr = parser->m_eventPtr; +- parser->m_processor = errorProcessor; +- return XML_STATUS_ERROR; +- } + #ifndef XML_CONTEXT_BYTES +- else if (parser->m_bufferPtr == parser->m_bufferEnd) { ++ if (parser->m_bufferPtr == parser->m_bufferEnd) { + const char *end; + int nLeftOver; + enum XML_Status result; +@@ -1907,12 +1916,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } ++ // though this isn't a buffer request, we assume that `len` is the app's ++ // preferred buffer fill size, and therefore save it here. ++ parser->m_lastBufferRequestSize = len; + parser->m_parseEndByteIndex += len; + parser->m_positionPtr = s; + parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; + + parser->m_errorCode +- = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end); ++ = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); + + if (parser->m_errorCode != XML_ERROR_NONE) { + parser->m_eventEndPtr = parser->m_eventPtr; +@@ -1939,23 +1951,25 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + &parser->m_position); + nLeftOver = s + len - end; + if (nLeftOver) { +- if (parser->m_buffer == NULL +- || nLeftOver > parser->m_bufferLim - parser->m_buffer) { +- /* avoid _signed_ integer overflow */ +- char *temp = NULL; +- const int bytesToAllocate = (int)((unsigned)len * 2U); +- if (bytesToAllocate > 0) { +- temp = (char *)REALLOC(parser, parser->m_buffer, bytesToAllocate); +- } +- if (temp == NULL) { +- parser->m_errorCode = XML_ERROR_NO_MEMORY; +- parser->m_eventPtr = parser->m_eventEndPtr = NULL; +- parser->m_processor = errorProcessor; +- return XML_STATUS_ERROR; +- } +- parser->m_buffer = temp; +- parser->m_bufferLim = parser->m_buffer + bytesToAllocate; ++ // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED ++ // (and XML_ERROR_FINISHED) from XML_GetBuffer. ++ const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; ++ parser->m_parsingStatus.parsing = XML_PARSING; ++ void *const temp = XML_GetBuffer(parser, nLeftOver); ++ parser->m_parsingStatus.parsing = originalStatus; ++ // GetBuffer may have overwritten this, but we want to remember what the ++ // app requested, not how many bytes were left over after parsing. ++ parser->m_lastBufferRequestSize = len; ++ if (temp == NULL) { ++ // NOTE: parser->m_errorCode has already been set by XML_GetBuffer(). ++ parser->m_eventPtr = parser->m_eventEndPtr = NULL; ++ parser->m_processor = errorProcessor; ++ return XML_STATUS_ERROR; + } ++ // Since we know that the buffer was empty and XML_CONTEXT_BYTES is 0, we ++ // don't have any data to preserve, and can copy straight into the start ++ // of the buffer rather than the GetBuffer return pointer (which may be ++ // pointing further into the allocated buffer). + memcpy(parser->m_buffer, end, nLeftOver); + } + parser->m_bufferPtr = parser->m_buffer; +@@ -1967,15 +1981,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + return result; + } + #endif /* not defined XML_CONTEXT_BYTES */ +- else { +- void *buff = XML_GetBuffer(parser, len); +- if (buff == NULL) +- return XML_STATUS_ERROR; +- else { +- memcpy(buff, s, len); +- return XML_ParseBuffer(parser, len, isFinal); +- } ++ void *buff = XML_GetBuffer(parser, len); ++ if (buff == NULL) ++ return XML_STATUS_ERROR; ++ if (len > 0) { ++ assert(s != NULL); // make sure s==NULL && len!=0 was rejected above ++ memcpy(buff, s, len); + } ++ return XML_ParseBuffer(parser, len, isFinal); + } + + enum XML_Status XMLCALL +@@ -2015,8 +2028,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { + parser->m_parseEndByteIndex += len; + parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; + +- parser->m_errorCode = parser->m_processor( +- parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr); ++ parser->m_errorCode = callProcessor(parser, start, parser->m_parseEndPtr, ++ &parser->m_bufferPtr); + + if (parser->m_errorCode != XML_ERROR_NONE) { + parser->m_eventEndPtr = parser->m_eventPtr; +@@ -2061,10 +2074,14 @@ XML_GetBuffer(XML_Parser parser, int len) { + default:; + } + +- if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { +-#ifdef XML_CONTEXT_BYTES ++ // whether or not the request succeeds, `len` seems to be the app's preferred ++ // buffer fill size; remember it. ++ parser->m_lastBufferRequestSize = len; ++ if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) ++ || parser->m_buffer == NULL) { ++#if XML_CONTEXT_BYTES > 0 + int keep; +-#endif /* defined XML_CONTEXT_BYTES */ ++#endif /* XML_CONTEXT_BYTES > 0 */ + /* Do not invoke signed arithmetic overflow: */ + int neededSize = (int)((unsigned)len + + (unsigned)EXPAT_SAFE_PTR_DIFF( +@@ -2073,7 +2090,7 @@ XML_GetBuffer(XML_Parser parser, int len) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return NULL; + } +-#ifdef XML_CONTEXT_BYTES ++#if XML_CONTEXT_BYTES > 0 + keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + if (keep > XML_CONTEXT_BYTES) + keep = XML_CONTEXT_BYTES; +@@ -2083,10 +2100,11 @@ XML_GetBuffer(XML_Parser parser, int len) { + return NULL; + } + neededSize += keep; +-#endif /* defined XML_CONTEXT_BYTES */ +- if (neededSize +- <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { +-#ifdef XML_CONTEXT_BYTES ++#endif /* XML_CONTEXT_BYTES > 0 */ ++ if (parser->m_buffer && parser->m_bufferPtr ++ && neededSize ++ <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { ++#if XML_CONTEXT_BYTES > 0 + if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { + int offset + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) +@@ -2099,19 +2117,17 @@ XML_GetBuffer(XML_Parser parser, int len) { + parser->m_bufferPtr -= offset; + } + #else +- if (parser->m_buffer && parser->m_bufferPtr) { +- memmove(parser->m_buffer, parser->m_bufferPtr, +- EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); +- parser->m_bufferEnd +- = parser->m_buffer +- + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); +- parser->m_bufferPtr = parser->m_buffer; +- } +-#endif /* not defined XML_CONTEXT_BYTES */ ++ memmove(parser->m_buffer, parser->m_bufferPtr, ++ EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); ++ parser->m_bufferEnd ++ = parser->m_buffer ++ + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); ++ parser->m_bufferPtr = parser->m_buffer; ++#endif /* XML_CONTEXT_BYTES > 0 */ + } else { + char *newBuf; + int bufferSize +- = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr); ++ = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); + if (bufferSize == 0) + bufferSize = INIT_BUFFER_SIZE; + do { +@@ -2128,7 +2144,7 @@ XML_GetBuffer(XML_Parser parser, int len) { + return NULL; + } + parser->m_bufferLim = newBuf + bufferSize; +-#ifdef XML_CONTEXT_BYTES ++#if XML_CONTEXT_BYTES > 0 + if (parser->m_bufferPtr) { + memcpy(newBuf, &parser->m_bufferPtr[-keep], + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) +@@ -2158,7 +2174,7 @@ XML_GetBuffer(XML_Parser parser, int len) { + parser->m_bufferEnd = newBuf; + } + parser->m_bufferPtr = parser->m_buffer = newBuf; +-#endif /* not defined XML_CONTEXT_BYTES */ ++#endif /* XML_CONTEXT_BYTES > 0 */ + } + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_positionPtr = NULL; +@@ -2208,7 +2224,7 @@ XML_ResumeParser(XML_Parser parser) { + } + parser->m_parsingStatus.parsing = XML_PARSING; + +- parser->m_errorCode = parser->m_processor( ++ parser->m_errorCode = callProcessor( + parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr); + + if (parser->m_errorCode != XML_ERROR_NONE) { +@@ -2561,6 +2577,15 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( + } + #endif /* XML_DTD */ + ++XML_Bool XMLCALL ++XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { ++ if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { ++ parser->m_reparseDeferralEnabled = enabled; ++ return XML_TRUE; ++ } ++ return XML_FALSE; ++} ++ + /* Initially tag->rawName always points into the parse buffer; + for those TAG instances opened while the current parse buffer was + processed, and not yet closed, we need to store tag->rawName in a more +@@ -4482,15 +4507,15 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, + parser->m_processor = entityValueProcessor; + return entityValueProcessor(parser, next, end, nextPtr); + } +- /* If we are at the end of the buffer, this would cause XmlPrologTok to +- return XML_TOK_NONE on the next call, which would then cause the +- function to exit with *nextPtr set to s - that is what we want for other +- tokens, but not for the BOM - we would rather like to skip it; +- then, when this routine is entered the next time, XmlPrologTok will +- return XML_TOK_INVALID, since the BOM is still in the buffer ++ /* XmlPrologTok has now set the encoding based on the BOM it found, and we ++ must move s and nextPtr forward to consume the BOM. ++ ++ If we didn't, and got XML_TOK_NONE from the next XmlPrologTok call, we ++ would leave the BOM in the buffer and return. On the next call to this ++ function, our XmlPrologTok call would return XML_TOK_INVALID, since it ++ is not valid to have multiple BOMs. + */ +- else if (tok == XML_TOK_BOM && next == end +- && ! parser->m_parsingStatus.finalBuffer) { ++ else if (tok == XML_TOK_BOM) { + # ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { +@@ -4500,7 +4525,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, + # endif + + *nextPtr = next; +- return XML_ERROR_NONE; ++ s = next; + } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with +diff --git a/tests/minicheck.c b/tests/minicheck.c +index 1c65748..f383380 100644 +--- a/tests/minicheck.c ++++ b/tests/minicheck.c +@@ -208,6 +208,21 @@ srunner_run_all(SRunner *runner, int verbosity) { + } + } + ++void ++_fail(const char *file, int line, const char *msg) { ++ /* Always print the error message so it isn't lost. In this case, ++ we have a failure, so there's no reason to be quiet about what ++ it is. ++ */ ++ _check_current_filename = file; ++ _check_current_lineno = line; ++ if (msg != NULL) { ++ const int has_newline = (msg[strlen(msg) - 1] == '\n'); ++ fprintf(stderr, "ERROR: %s%s", msg, has_newline ? "" : "\n"); ++ } ++ longjmp(env, 1); ++} ++ + void + _fail_unless(int condition, const char *file, int line, const char *msg) { + /* Always print the error message so it isn't lost. In this case, +diff --git a/tests/minicheck.h b/tests/minicheck.h +index cc1f835..032b54e 100644 +--- a/tests/minicheck.h ++++ b/tests/minicheck.h +@@ -64,7 +64,14 @@ extern "C" { + } \ + } + +-#define fail(msg) _fail_unless(0, __FILE__, __LINE__, msg) ++ ++# define fail(msg) _fail(__FILE__, __LINE__, msg) ++# define assert_true(cond) \ ++ do { \ ++ if (! (cond)) { \ ++ _fail(__FILE__, __LINE__, "check failed: " #cond); \ ++ } \ ++ } while (0) + + typedef void (*tcase_setup_function)(void); + typedef void (*tcase_teardown_function)(void); +@@ -103,6 +110,11 @@ void _check_set_test_info(char const *function, char const *filename, + * Prototypes for the actual implementation. + */ + ++# if defined(__GNUC__) ++__attribute__((noreturn)) ++# endif ++void ++_fail(const char *file, int line, const char *msg); + void _fail_unless(int condition, const char *file, int line, const char *msg); + Suite *suite_create(const char *name); + TCase *tcase_create(const char *name); +diff --git a/tests/runtests.c b/tests/runtests.c +index 915fa52..941f61d 100644 +--- a/tests/runtests.c ++++ b/tests/runtests.c +@@ -54,6 +54,7 @@ + #include + #include + #include /* intptr_t uint64_t */ ++#include + + #if ! defined(__cplusplus) + # include +@@ -1071,7 +1072,7 @@ START_TEST(test_column_number_after_parse) { + const char *text = ""; + XML_Size colno; + +- if (_XML_Parse_SINGLE_BYTES(g_parser, text, (int)strlen(text), XML_FALSE) ++ if (_XML_Parse_SINGLE_BYTES(g_parser, text, (int)strlen(text), XML_TRUE) + == XML_STATUS_ERROR) + xml_failure(g_parser); + colno = XML_GetCurrentColumnNumber(g_parser); +@@ -2582,7 +2583,7 @@ START_TEST(test_default_current) { + if (_XML_Parse_SINGLE_BYTES(g_parser, text, (int)strlen(text), XML_TRUE) + == XML_STATUS_ERROR) + xml_failure(g_parser); +- CharData_CheckXMLChars(&storage, XCS("DCDCDCDCDCDD")); ++ CharData_CheckXMLChars(&storage, XCS("DCDCDCDD")); + + /* Again, without the defaulting */ + XML_ParserReset(g_parser, NULL); +@@ -2593,7 +2594,7 @@ START_TEST(test_default_current) { + if (_XML_Parse_SINGLE_BYTES(g_parser, text, (int)strlen(text), XML_TRUE) + == XML_STATUS_ERROR) + xml_failure(g_parser); +- CharData_CheckXMLChars(&storage, XCS("DcccccD")); ++ CharData_CheckXMLChars(&storage, XCS("DcccD")); + + /* Now with an internal entity to complicate matters */ + XML_ParserReset(g_parser, NULL); +@@ -3946,6 +3947,19 @@ START_TEST(test_get_buffer_3_overflow) { + END_TEST + #endif // defined(XML_CONTEXT_BYTES) + ++START_TEST(test_getbuffer_allocates_on_zero_len) { ++ for (int first_len = 1; first_len >= 0; first_len--) { ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ assert_true(XML_GetBuffer(parser, first_len) != NULL); ++ assert_true(XML_GetBuffer(parser, 0) != NULL); ++ if (XML_ParseBuffer(parser, 0, XML_FALSE) != XML_STATUS_OK) ++ xml_failure(parser); ++ XML_ParserFree(parser); ++ } ++} ++END_TEST ++ + /* Test position information macros */ + START_TEST(test_byte_info_at_end) { + const char *text = ""; +@@ -6205,6 +6219,12 @@ START_TEST(test_utf8_in_start_tags) { + char doc[1024]; + size_t failCount = 0; + ++ // we need all the bytes to be parsed, but we don't want the errors that can ++ // trigger on isFinal=XML_TRUE, so we skip the test if the heuristic is on. ++ if (g_reparseDeferralEnabledDefault) { ++ return; ++ } ++ + for (; i < sizeof(cases) / sizeof(cases[0]); i++) { + size_t j = 0; + for (; j < sizeof(atNameStart) / sizeof(atNameStart[0]); j++) { +@@ -6830,6 +6850,613 @@ START_TEST(test_nested_entity_suspend) { + } + END_TEST + ++/* Regression test for quadratic parsing on large tokens */ ++START_TEST(test_big_tokens_take_linear_time) { ++ const char *const too_slow_failure_message ++ = "Compared to the baseline runtime of the first test, this test has a " ++ "slowdown of more than . " ++ "Please keep increasing the value by 1 until it reliably passes the " ++ "test on your hardware and open a bug sharing that number with us. " ++ "Thanks in advance!"; ++ const struct { ++ const char *pre; ++ const char *post; ++ } text[] = { ++ {"", ""}, // assumed good, used as baseline ++ {""}, // CDATA, performed OK before patch ++ {""}, // big attribute, used to be O(N²) ++ {""}, // long comment, used to be O(N²) ++ {"<", "/>"}, // big elem name, used to be O(N²) ++ }; ++ const int num_cases = sizeof(text) / sizeof(text[0]); ++ // For the test we need a value that is: ++ // (1) big enough that the test passes reliably (avoiding flaky tests), and ++ // (2) small enough that the test actually catches regressions. ++ const int max_slowdown = 15; ++ char aaaaaa[4096]; ++ const int fillsize = (int)sizeof(aaaaaa); ++ const int fillcount = 100; ++ ++ memset(aaaaaa, 'a', fillsize); ++ ++ if (! g_reparseDeferralEnabledDefault) { ++ return; // heuristic is disabled; we would get O(n^2) and fail. ++ } ++#if defined(_WIN32) ++ if (CLOCKS_PER_SEC < 100000) { ++ // Skip this test if clock() doesn't have reasonably good resolution. ++ // This workaround is only applied to Windows targets, since XSI requires ++ // the value to be 1 000 000 (10x the condition here), and we want to be ++ // very sure that at least one platform in CI can catch regressions. ++ return; ++ } ++#endif ++ ++ clock_t baseline = 0; ++ for (int i = 0; i < num_cases; ++i) { ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ enum XML_Status status; ++ const clock_t start = clock(); ++ ++ // parse the start text ++ status = _XML_Parse_SINGLE_BYTES(parser, text[i].pre, ++ (int)strlen(text[i].pre), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ // parse lots of 'a', failing the test early if it takes too long ++ for (int f = 0; f < fillcount; ++f) { ++ status = _XML_Parse_SINGLE_BYTES(parser, aaaaaa, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ // i == 0 means we're still calculating the baseline value ++ if (i > 0) { ++ const clock_t now = clock(); ++ const clock_t clocks_so_far = now - start; ++ const int slowdown = clocks_so_far / baseline; ++ if (slowdown >= max_slowdown) { ++ fprintf( ++ stderr, ++ "fill#%d: clocks_so_far=%d baseline=%d slowdown=%d max_slowdown=%d\n", ++ f, (int)clocks_so_far, (int)baseline, slowdown, max_slowdown); ++ fail(too_slow_failure_message); ++ } ++ } ++ } ++ // parse the end text ++ status = _XML_Parse_SINGLE_BYTES(parser, text[i].post, ++ (int)strlen(text[i].post), XML_TRUE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ ++ // how long did it take in total? ++ const clock_t end = clock(); ++ const clock_t taken = end - start; ++ if (i == 0) { ++ assert_true(taken > 0); // just to make sure we don't div-by-0 later ++ baseline = taken; ++ } ++ const int slowdown = taken / baseline; ++ if (slowdown >= max_slowdown) { ++ fprintf(stderr, "taken=%d baseline=%d slowdown=%d max_slowdown=%d\n", ++ (int)taken, (int)baseline, slowdown, max_slowdown); ++ fail(too_slow_failure_message); ++ } ++ ++ XML_ParserFree(parser); ++ } ++} ++END_TEST ++ ++START_TEST(test_set_reparse_deferral) { ++ const char *const pre = ""; ++ const char *const start = ""; ++ char eeeeee[100]; ++ const int fillsize = (int)sizeof(eeeeee); ++ memset(eeeeee, 'e', fillsize); ++ ++ for (int enabled = 0; enabled <= 1; enabled += 1) { ++ ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ assert_true(XML_SetReparseDeferralEnabled(parser, enabled)); ++ // pre-grow the buffer to avoid reparsing due to almost-fullness ++ assert_true(XML_GetBuffer(parser, fillsize * 10103) != NULL); ++ ++ CharData storage; ++ CharData_Init(&storage); ++ XML_SetUserData(parser, &storage); ++ XML_SetStartElementHandler(parser, start_element_event_handler); ++ ++ enum XML_Status status; ++ // parse the start text ++ status = XML_Parse(parser, pre, (int)strlen(pre), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // first element should be done ++ ++ // ..and the start of the token ++ status = XML_Parse(parser, start, (int)strlen(start), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // still just the first one ++ ++ // try to parse lots of 'e', but the token isn't finished ++ for (int c = 0; c < 100; ++c) { ++ status = XML_Parse(parser, eeeeee, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // *still* just the first one ++ ++ // end the token. ++ status = XML_Parse(parser, end, (int)strlen(end), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ ++ if (enabled) { ++ // In general, we may need to push more data to trigger a reparse attempt, ++ // but in this test, the data is constructed to always require it. ++ CharData_CheckXMLChars(&storage, XCS("d")); // or the test is incorrect ++ // 2x the token length should suffice; the +1 covers the start and end. ++ for (int c = 0; c < 101; ++c) { ++ status = XML_Parse(parser, eeeeee, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ } ++ } ++ CharData_CheckXMLChars(&storage, XCS("dx")); // the should be done ++ ++ XML_ParserFree(parser); ++ } ++} ++END_TEST ++ ++struct element_decl_data { ++ XML_Parser parser; ++ int count; ++}; ++ ++static void ++element_decl_counter(void *userData, const XML_Char *name, XML_Content *model) { ++ UNUSED_P(name); ++ struct element_decl_data *testdata = (struct element_decl_data *)userData; ++ testdata->count += 1; ++ XML_FreeContentModel(testdata->parser, model); ++} ++ ++static int ++external_inherited_parser(XML_Parser p, const XML_Char *context, ++ const XML_Char *base, const XML_Char *systemId, ++ const XML_Char *publicId) { ++ UNUSED_P(base); ++ UNUSED_P(systemId); ++ UNUSED_P(publicId); ++ const char *const pre = "\n"; ++ const char *const start = "\n"; ++ const char *const post = "\n"; ++ const int enabled = *(int *)XML_GetUserData(p); ++ char eeeeee[100]; ++ char spaces[100]; ++ const int fillsize = (int)sizeof(eeeeee); ++ assert_true(fillsize == (int)sizeof(spaces)); ++ memset(eeeeee, 'e', fillsize); ++ memset(spaces, ' ', fillsize); ++ ++ XML_Parser parser = XML_ExternalEntityParserCreate(p, context, NULL); ++ assert_true(parser != NULL); ++ // pre-grow the buffer to avoid reparsing due to almost-fullness ++ assert_true(XML_GetBuffer(parser, fillsize * 10103) != NULL); ++ ++ struct element_decl_data testdata; ++ testdata.parser = parser; ++ testdata.count = 0; ++ XML_SetUserData(parser, &testdata); ++ XML_SetElementDeclHandler(parser, element_decl_counter); ++ ++ enum XML_Status status; ++ // parse the initial text ++ status = XML_Parse(parser, pre, (int)strlen(pre), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ assert_true(testdata.count == 1); // first element should be done ++ ++ // ..and the start of the big token ++ status = XML_Parse(parser, start, (int)strlen(start), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ assert_true(testdata.count == 1); // still just the first one ++ ++ // try to parse lots of 'e', but the token isn't finished ++ for (int c = 0; c < 100; ++c) { ++ status = XML_Parse(parser, eeeeee, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ } ++ assert_true(testdata.count == 1); // *still* just the first one ++ ++ // end the big token. ++ status = XML_Parse(parser, end, (int)strlen(end), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ ++ if (enabled) { ++ // In general, we may need to push more data to trigger a reparse attempt, ++ // but in this test, the data is constructed to always require it. ++ assert_true(testdata.count == 1); // or the test is incorrect ++ // 2x the token length should suffice; the +1 covers the start and end. ++ for (int c = 0; c < 101; ++c) { ++ status = XML_Parse(parser, spaces, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ } ++ } ++ assert_true(testdata.count == 2); // the big token should be done ++ ++ // parse the final text ++ status = XML_Parse(parser, post, (int)strlen(post), XML_TRUE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ assert_true(testdata.count == 3); // after isFinal=XML_TRUE, all must be done ++ ++ XML_ParserFree(parser); ++ return XML_STATUS_OK; ++} ++ ++START_TEST(test_reparse_deferral_is_inherited) { ++ const char *const text ++ = ""; ++ for (int enabled = 0; enabled <= 1; ++enabled) { ++ ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ XML_SetUserData(parser, (void *)&enabled); ++ XML_SetParamEntityParsing(parser, XML_PARAM_ENTITY_PARSING_ALWAYS); ++ // this handler creates a sub-parser and checks that its deferral behavior ++ // is what we expected, based on the value of `enabled` (in userdata). ++ XML_SetExternalEntityRefHandler(parser, external_inherited_parser); ++ assert_true(XML_SetReparseDeferralEnabled(parser, enabled)); ++ if (XML_Parse(parser, text, (int)strlen(text), XML_TRUE) != XML_STATUS_OK) ++ xml_failure(parser); ++ ++ XML_ParserFree(parser); ++ } ++} ++END_TEST ++ ++START_TEST(test_set_reparse_deferral_on_null_parser) { ++ assert_true(XML_SetReparseDeferralEnabled(NULL, 0) == XML_FALSE); ++ assert_true(XML_SetReparseDeferralEnabled(NULL, 1) == XML_FALSE); ++ assert_true(XML_SetReparseDeferralEnabled(NULL, 10) == XML_FALSE); ++ assert_true(XML_SetReparseDeferralEnabled(NULL, 100) == XML_FALSE); ++ assert_true(XML_SetReparseDeferralEnabled(NULL, (XML_Bool)INT_MIN) ++ == XML_FALSE); ++ assert_true(XML_SetReparseDeferralEnabled(NULL, (XML_Bool)INT_MAX) ++ == XML_FALSE); ++} ++END_TEST ++ ++START_TEST(test_set_reparse_deferral_on_the_fly) { ++ const char *const pre = ""; ++ char iiiiii[100]; ++ const int fillsize = (int)sizeof(iiiiii); ++ memset(iiiiii, 'i', fillsize); ++ ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ assert_true(XML_SetReparseDeferralEnabled(parser, XML_TRUE)); ++ ++ CharData storage; ++ CharData_Init(&storage); ++ XML_SetUserData(parser, &storage); ++ XML_SetStartElementHandler(parser, start_element_event_handler); ++ ++ enum XML_Status status; ++ // parse the start text ++ status = XML_Parse(parser, pre, (int)strlen(pre), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // first element should be done ++ ++ // try to parse some 'i', but the token isn't finished ++ status = XML_Parse(parser, iiiiii, fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // *still* just the first one ++ ++ // end the token. ++ status = XML_Parse(parser, end, (int)strlen(end), XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("d")); // not yet. ++ ++ // now change the heuristic setting and add *no* data ++ assert_true(XML_SetReparseDeferralEnabled(parser, XML_FALSE)); ++ // we avoid isFinal=XML_TRUE, because that would force-bypass the heuristic. ++ status = XML_Parse(parser, "", 0, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ CharData_CheckXMLChars(&storage, XCS("dx")); ++ ++ XML_ParserFree(parser); ++} ++END_TEST ++ ++START_TEST(test_set_bad_reparse_option) { ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 2)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 3)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 99)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 127)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 128)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 129)); ++ assert_true(XML_FALSE == XML_SetReparseDeferralEnabled(parser, 255)); ++ assert_true(XML_TRUE == XML_SetReparseDeferralEnabled(parser, 0)); ++ assert_true(XML_TRUE == XML_SetReparseDeferralEnabled(parser, 1)); ++ XML_ParserFree(parser); ++} ++END_TEST ++ ++static size_t g_totalAlloc = 0; ++static size_t g_biggestAlloc = 0; ++ ++static void * ++counting_realloc(void *ptr, size_t size) { ++ g_totalAlloc += size; ++ if (size > g_biggestAlloc) { ++ g_biggestAlloc = size; ++ } ++ return realloc(ptr, size); ++} ++ ++static void * ++counting_malloc(size_t size) { ++ return counting_realloc(NULL, size); ++} ++ ++START_TEST(test_bypass_heuristic_when_close_to_bufsize) { ++ if (! g_reparseDeferralEnabledDefault) { ++ return; // this test is irrelevant when the deferral heuristic is disabled. ++ } ++ ++ const int document_length = 65536; ++ char *const document = (char *)malloc(document_length); ++ ++ const XML_Memory_Handling_Suite memfuncs = { ++ counting_malloc, ++ counting_realloc, ++ free, ++ }; ++ ++ const int leading_list[] = {0, 3, 61, 96, 400, 401, 4000, 4010, 4099, -1}; ++ const int bigtoken_list[] = {3000, 4000, 4001, 4096, 4099, 5000, 20000, -1}; ++ const int fillsize_list[] = {131, 256, 399, 400, 401, 1025, 4099, 4321, -1}; ++ ++ for (const int *leading = leading_list; *leading >= 0; leading++) { ++ for (const int *bigtoken = bigtoken_list; *bigtoken >= 0; bigtoken++) { ++ for (const int *fillsize = fillsize_list; *fillsize >= 0; fillsize++) { ++ // start by checking that the test looks reasonably valid ++ assert_true(*leading + *bigtoken <= document_length); ++ ++ // put 'x' everywhere; some will be overwritten by elements. ++ memset(document, 'x', document_length); ++ // maybe add an initial tag ++ if (*leading) { ++ assert_true(*leading >= 3); // or the test case is invalid ++ memcpy(document, "", 3); ++ } ++ // add the large token ++ document[*leading + 0] = '<'; ++ document[*leading + 1] = 'b'; ++ memset(&document[*leading + 2], ' ', *bigtoken - 2); // a spacy token ++ document[*leading + *bigtoken - 1] = '>'; ++ ++ // 1 for 'b', plus 1 or 0 depending on the presence of 'a' ++ const int expected_elem_total = 1 + (*leading ? 1 : 0); ++ ++ XML_Parser parser = XML_ParserCreate_MM(NULL, &memfuncs, NULL); ++ assert_true(parser != NULL); ++ ++ CharData storage; ++ CharData_Init(&storage); ++ XML_SetUserData(parser, &storage); ++ XML_SetStartElementHandler(parser, start_element_event_handler); ++ ++ g_biggestAlloc = 0; ++ g_totalAlloc = 0; ++ int offset = 0; ++ // fill data until the big token is covered (but not necessarily parsed) ++ while (offset < *leading + *bigtoken) { ++ assert_true(offset + *fillsize <= document_length); ++ const enum XML_Status status ++ = XML_Parse(parser, &document[offset], *fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ offset += *fillsize; ++ } ++ // Now, check that we've had a buffer allocation that could fit the ++ // context bytes and our big token. In order to detect a special case, ++ // we need to know how many bytes of our big token were included in the ++ // first push that contained _any_ bytes of the big token: ++ const int bigtok_first_chunk_bytes = *fillsize - (*leading % *fillsize); ++ if (bigtok_first_chunk_bytes >= *bigtoken && XML_CONTEXT_BYTES == 0) { ++ // Special case: we aren't saving any context, and the whole big token ++ // was covered by a single fill, so Expat may have parsed directly ++ // from our input pointer, without allocating an internal buffer. ++ } else if (*leading < XML_CONTEXT_BYTES) { ++ assert_true(g_biggestAlloc >= *leading + (size_t)*bigtoken); ++ } else { ++ assert_true(g_biggestAlloc >= XML_CONTEXT_BYTES + (size_t)*bigtoken); ++ } ++ // fill data until the big token is actually parsed ++ while (storage.count < expected_elem_total) { ++ const size_t alloc_before = g_totalAlloc; ++ assert_true(offset + *fillsize <= document_length); ++ const enum XML_Status status ++ = XML_Parse(parser, &document[offset], *fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ offset += *fillsize; ++ // since all the bytes of the big token are already in the buffer, ++ // the bufsize ceiling should make us finish its parsing without any ++ // further buffer allocations. We assume that there will be no other ++ // large allocations in this test. ++ assert_true(g_totalAlloc - alloc_before < 4096); ++ } ++ // test-the-test: was our alloc even called? ++ assert_true(g_totalAlloc > 0); ++ // test-the-test: there shouldn't be any extra start elements ++ assert_true(storage.count == expected_elem_total); ++ ++ XML_ParserFree(parser); ++ } ++ } ++ } ++ free(document); ++} ++END_TEST ++ ++START_TEST(test_varying_buffer_fills) { ++ const int KiB = 1024; ++ const int MiB = 1024 * KiB; ++ const int document_length = 16 * MiB; ++ const int big = 7654321; // arbitrarily chosen between 4 and 8 MiB ++ ++ char *const document = (char *)malloc(document_length); ++ assert_true(document != NULL); ++ memset(document, 'x', document_length); ++ document[0] = '<'; ++ document[1] = 't'; ++ memset(&document[2], ' ', big - 2); // a very spacy token ++ document[big - 1] = '>'; ++ ++ // Each testcase is a list of buffer fill sizes, terminated by a value < 0. ++ // When reparse deferral is enabled, the final (negated) value is the expected ++ // maximum number of bytes scanned in parse attempts. ++ const int testcases[][30] = { ++ {8 * MiB, -8 * MiB}, ++ {4 * MiB, 4 * MiB, -12 * MiB}, // try at 4MB, then 8MB = 12 MB total ++ // zero-size fills shouldn't trigger the bypass ++ {4 * MiB, 0, 4 * MiB, -12 * MiB}, ++ {4 * MiB, 0, 0, 4 * MiB, -12 * MiB}, ++ {4 * MiB, 0, 1 * MiB, 0, 3 * MiB, -12 * MiB}, ++ // try to hit the buffer ceiling only once (at the end) ++ {4 * MiB, 2 * MiB, 1 * MiB, 512 * KiB, 256 * KiB, 256 * KiB, -12 * MiB}, ++ // try to hit the same buffer ceiling multiple times ++ {4 * MiB + 1, 2 * MiB, 1 * MiB, 512 * KiB, -25 * MiB}, ++ ++ // try to hit every ceiling, by always landing 1K shy of the buffer size ++ {1 * KiB, 2 * KiB, 4 * KiB, 8 * KiB, 16 * KiB, 32 * KiB, 64 * KiB, ++ 128 * KiB, 256 * KiB, 512 * KiB, 1 * MiB, 2 * MiB, 4 * MiB, -16 * MiB}, ++ ++ // try to avoid every ceiling, by always landing 1B past the buffer size ++ // the normal 2x heuristic threshold still forces parse attempts. ++ {2 * KiB + 1, // will attempt 2KiB + 1 ==> total 2KiB + 1 ++ 2 * KiB, 4 * KiB, // will attempt 8KiB + 1 ==> total 10KiB + 2 ++ 8 * KiB, 16 * KiB, // will attempt 32KiB + 1 ==> total 42KiB + 3 ++ 32 * KiB, 64 * KiB, // will attempt 128KiB + 1 ==> total 170KiB + 4 ++ 128 * KiB, 256 * KiB, // will attempt 512KiB + 1 ==> total 682KiB + 5 ++ 512 * KiB, 1 * MiB, // will attempt 2MiB + 1 ==> total 2M + 682K + 6 ++ 2 * MiB, 4 * MiB, // will attempt 8MiB + 1 ==> total 10M + 682K + 7 ++ -(10 * MiB + 682 * KiB + 7)}, ++ // try to avoid every ceiling again, except on our last fill. ++ {2 * KiB + 1, // will attempt 2KiB + 1 ==> total 2KiB + 1 ++ 2 * KiB, 4 * KiB, // will attempt 8KiB + 1 ==> total 10KiB + 2 ++ 8 * KiB, 16 * KiB, // will attempt 32KiB + 1 ==> total 42KiB + 3 ++ 32 * KiB, 64 * KiB, // will attempt 128KiB + 1 ==> total 170KiB + 4 ++ 128 * KiB, 256 * KiB, // will attempt 512KiB + 1 ==> total 682KiB + 5 ++ 512 * KiB, 1 * MiB, // will attempt 2MiB + 1 ==> total 2M + 682K + 6 ++ 2 * MiB, 4 * MiB - 1, // will attempt 8MiB ==> total 10M + 682K + 6 ++ -(10 * MiB + 682 * KiB + 6)}, ++ ++ // try to hit ceilings on the way multiple times ++ {512 * KiB + 1, 256 * KiB, 128 * KiB, 128 * KiB - 1, // 1 MiB buffer ++ 512 * KiB + 1, 256 * KiB, 128 * KiB, 128 * KiB - 1, // 2 MiB buffer ++ 1 * MiB + 1, 512 * KiB, 256 * KiB, 256 * KiB - 1, // 4 MiB buffer ++ 2 * MiB + 1, 1 * MiB, 512 * KiB, // 8 MiB buffer ++ // we'll make a parse attempt at every parse call ++ -(45 * MiB + 12)}, ++ }; ++ const int testcount = sizeof(testcases) / sizeof(testcases[0]); ++ for (int test_i = 0; test_i < testcount; test_i++) { ++ const int *fillsize = testcases[test_i]; ++ XML_Parser parser = XML_ParserCreate(NULL); ++ assert_true(parser != NULL); ++ g_parseAttempts = 0; ++ ++ CharData storage; ++ CharData_Init(&storage); ++ XML_SetUserData(parser, &storage); ++ XML_SetStartElementHandler(parser, start_element_event_handler); ++ ++ int worstcase_bytes = 0; // sum of (buffered bytes at each XML_Parse call) ++ int scanned_bytes = 0; // sum of (buffered bytes at each actual parse) ++ int offset = 0; ++ while (*fillsize >= 0) { ++ assert_true(offset + *fillsize <= document_length); // or test is invalid ++ const unsigned attempts_before = g_parseAttempts; ++ const enum XML_Status status ++ = XML_Parse(parser, &document[offset], *fillsize, XML_FALSE); ++ if (status != XML_STATUS_OK) { ++ xml_failure(parser); ++ } ++ offset += *fillsize; ++ fillsize++; ++ assert_true(offset <= INT_MAX - worstcase_bytes); // avoid overflow ++ worstcase_bytes += offset; // we might've tried to parse all pending bytes ++ if (g_parseAttempts != attempts_before) { ++ assert_true(g_parseAttempts == attempts_before + 1); // max 1/XML_Parse ++ assert_true(offset <= INT_MAX - scanned_bytes); // avoid overflow ++ scanned_bytes += offset; // we *did* try to parse all pending bytes ++ } ++ } ++ assert_true(storage.count == 1); // the big token should've been parsed ++ assert_true(scanned_bytes > 0); // test-the-test: does our counter work? ++ if (g_reparseDeferralEnabledDefault) { ++ // heuristic is enabled; some XML_Parse calls may have deferred reparsing ++ const int max_bytes_scanned = -*fillsize; ++ if (scanned_bytes > max_bytes_scanned) { ++ fprintf(stderr, ++ "bytes scanned in parse attempts: actual=%d limit=%d \n", ++ scanned_bytes, max_bytes_scanned); ++ fail("too many bytes scanned in parse attempts"); ++ } ++ assert_true(scanned_bytes <= worstcase_bytes); ++ } else { ++ // heuristic is disabled; every XML_Parse() will have reparsed ++ assert_true(scanned_bytes == worstcase_bytes); ++ } ++ ++ XML_ParserFree(parser); ++ } ++ free(document); ++} ++END_TEST ++ ++ + /* + * Namespaces tests. + */ +@@ -6902,13 +7529,13 @@ START_TEST(test_return_ns_triplet) { + if (_XML_Parse_SINGLE_BYTES(g_parser, text, (int)strlen(text), XML_FALSE) + == XML_STATUS_ERROR) + xml_failure(g_parser); +- if (! triplet_start_flag) +- fail("triplet_start_checker not invoked"); + /* Check that unsetting "return triplets" fails while still parsing */ + XML_SetReturnNSTriplet(g_parser, XML_FALSE); + if (_XML_Parse_SINGLE_BYTES(g_parser, epilog, (int)strlen(epilog), XML_TRUE) + == XML_STATUS_ERROR) + xml_failure(g_parser); ++ if (! triplet_start_flag) ++ fail("triplet_start_checker not invoked"); + if (! triplet_end_flag) + fail("triplet_end_checker not invoked"); + if (dummy_handler_flags +@@ -12219,6 +12846,7 @@ make_suite(void) { + #if defined(XML_CONTEXT_BYTES) + tcase_add_test(tc_basic, test_get_buffer_3_overflow); + #endif ++ tcase_add_test(tc_basic, test_getbuffer_allocates_on_zero_len); + tcase_add_test(tc_basic, test_byte_info_at_end); + tcase_add_test(tc_basic, test_byte_info_at_error); + tcase_add_test(tc_basic, test_byte_info_at_cdata); +@@ -12337,7 +12965,14 @@ make_suite(void) { + tcase_add_test__ifdef_xml_dtd(tc_basic, + test_pool_integrity_with_unfinished_attr); + tcase_add_test(tc_basic, test_nested_entity_suspend); +- ++ tcase_add_test(tc_basic, test_big_tokens_take_linear_time); ++ tcase_add_test(tc_basic, test_set_reparse_deferral); ++ tcase_add_test(tc_basic, test_reparse_deferral_is_inherited); ++ tcase_add_test(tc_basic, test_set_reparse_deferral_on_null_parser); ++ tcase_add_test(tc_basic, test_set_reparse_deferral_on_the_fly); ++ tcase_add_test(tc_basic, test_set_bad_reparse_option); ++ tcase_add_test(tc_basic, test_bypass_heuristic_when_close_to_bufsize); ++ tcase_add_test(tc_basic, test_varying_buffer_fills); + suite_add_tcase(s, tc_namespace); + tcase_add_checked_fixture(tc_namespace, namespace_setup, namespace_teardown); + tcase_add_test(tc_namespace, test_return_ns_triplet); +diff --git a/xmlwf/xmlwf.c b/xmlwf/xmlwf.c +index 471f2a2..7c62919 100644 +--- a/xmlwf/xmlwf.c ++++ b/xmlwf/xmlwf.c +@@ -914,6 +914,9 @@ usage(const XML_Char *prog, int rc) { + T(" -a FACTOR set maximum tolerated [a]mplification factor (default: 100.0)\n") + T(" -b BYTES set number of output [b]ytes needed to activate (default: 8 MiB)\n") + T("\n") ++ T("reparse deferral:\n") ++ T(" -q disable reparse deferral, and allow [q]uadratic parse runtime with large tokens\n") ++ T("\n") + T("info arguments:\n") + T(" -h show this [h]elp message and exit\n") + T(" -v show program's [v]ersion number and exit\n") +@@ -967,6 +970,8 @@ tmain(int argc, XML_Char **argv) { + unsigned long long attackThresholdBytes; + XML_Bool attackThresholdGiven = XML_FALSE; + ++ XML_Bool disableDeferral = XML_FALSE; ++ + int exitCode = XMLWF_EXIT_SUCCESS; + enum XML_ParamEntityParsing paramEntityParsing + = XML_PARAM_ENTITY_PARSING_NEVER; +@@ -1089,6 +1094,11 @@ tmain(int argc, XML_Char **argv) { + #endif + break; + } ++ case T('q'): { ++ disableDeferral = XML_TRUE; ++ j++; ++ break; ++ } + case T('\0'): + if (j > 1) { + i++; +@@ -1134,6 +1144,16 @@ tmain(int argc, XML_Char **argv) { + #endif + } + ++ if (disableDeferral) { ++ const XML_Bool success = XML_SetReparseDeferralEnabled(parser, XML_FALSE); ++ if (! success) { ++ // This prevents tperror(..) from reporting misleading "[..]: Success" ++ errno = EINVAL; ++ tperror(T("Failed to disable reparse deferral")); ++ exit(XMLWF_EXIT_INTERNAL_ERROR); ++ } ++ } ++ + if (requireStandalone) + XML_SetNotStandaloneHandler(parser, notStandalone); + XML_SetParamEntityParsing(parser, paramEntityParsing); +diff --git a/xmlwf/xmlwf_helpgen.py b/xmlwf/xmlwf_helpgen.py +index c2a527f..1bd0a0a 100755 +--- a/xmlwf/xmlwf_helpgen.py ++++ b/xmlwf/xmlwf_helpgen.py +@@ -81,6 +81,10 @@ billion_laughs.add_argument('-a', metavar='FACTOR', + help='set maximum tolerated [a]mplification factor (default: 100.0)') + billion_laughs.add_argument('-b', metavar='BYTES', help='set number of output [b]ytes needed to activate (default: 8 MiB)') + ++reparse_deferral = parser.add_argument_group('reparse deferral') ++reparse_deferral.add_argument('-q', metavar='FACTOR', ++ help='disable reparse deferral, and allow [q]uadratic parse runtime with large tokens') ++ + parser.add_argument('files', metavar='FILE', nargs='*', help='file to process (default: STDIN)') + + info = parser.add_argument_group('info arguments') +diff --git a/testdata/largefiles/aaaaaa_attr.xml b/testdata/largefiles/aaaaaa_attr.xml +new file mode 100644 +index 0000000..66e3d25 +--- /dev/null ++++ b/testdata/largefiles/aaaaaa_attr.xml +@@ -0,0 +1 @@ ++ +\ No newline at end of file +diff --git a/testdata/largefiles/aaaaaa_cdata.xml b/testdata/largefiles/aaaaaa_cdata.xml +new file mode 100644 +index 0000000..66f64bd +--- /dev/null ++++ b/testdata/largefiles/aaaaaa_cdata.xml +@@ -0,0 +1 @@ ++ +\ No newline at end of file +diff --git a/testdata/largefiles/aaaaaa_comment.xml b/testdata/largefiles/aaaaaa_comment.xml +new file mode 100644 +index 0000000..bb9af13 +--- /dev/null ++++ b/testdata/largefiles/aaaaaa_comment.xml +@@ -0,0 +1 @@ ++ +\ No newline at end of file +diff --git a/testdata/largefiles/aaaaaa_tag.xml b/testdata/largefiles/aaaaaa_tag.xml +new file mode 100644 +index 0000000..946f701 +--- /dev/null ++++ b/testdata/largefiles/aaaaaa_tag.xml +@@ -0,0 +1 @@ ++ +\ No newline at end of file +diff --git a/testdata/largefiles/aaaaaa_text.xml b/testdata/largefiles/aaaaaa_text.xml +new file mode 100644 +index 0000000..e266acb +--- /dev/null ++++ b/testdata/largefiles/aaaaaa_text.xml +@@ -0,0 +1 @@ ++ACHARS +\ No newline at end of file diff --git a/pkgs/development/libraries/expat/CVE-2024-28757.patch b/pkgs/development/libraries/expat/CVE-2024-28757.patch new file mode 100644 index 000000000000..413b29a0cd50 --- /dev/null +++ b/pkgs/development/libraries/expat/CVE-2024-28757.patch @@ -0,0 +1,86 @@ +From c645f5996f3a8cff10606182a8031d3c3ade6ea3 Mon Sep 17 00:00:00 2001 +From: Sebastian Pipping +Date: Sun, 3 Mar 2024 02:19:58 +0100 +Subject: [PATCH 1/2] lib/xmlparse.c: Reject directly recursive parameter + entities + +(cherry picked from commit a4c86a395ee447c59175c762af3d17f7107b2261) +--- + lib/xmlparse.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/xmlparse.c b/lib/xmlparse.c +index b6c2eca9..2858946d 100644 +--- a/lib/xmlparse.c ++++ b/lib/xmlparse.c +@@ -6139,7 +6139,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, + dtd->keepProcessing = dtd->standalone; + goto endEntityValue; + } +- if (entity->open) { ++ if (entity->open || (entity == parser->m_declEntity)) { + if (enc == parser->m_encoding) + parser->m_eventPtr = entityTextPtr; + result = XML_ERROR_RECURSIVE_ENTITY_REF; +-- +2.44.0 + + +From b2170839eab38624df1d9842d738e4a2b6a08a2b Mon Sep 17 00:00:00 2001 +From: Sebastian Pipping +Date: Mon, 4 Mar 2024 23:49:06 +0100 +Subject: [PATCH 2/2] lib/xmlparse.c: Detect billion laughs attack with + isolated external parser + +When parsing DTD content with code like .. + + XML_Parser parser = XML_ParserCreate(NULL); + XML_Parser ext_parser = XML_ExternalEntityParserCreate(parser, NULL, NULL); + enum XML_Status status = XML_Parse(ext_parser, doc, (int)strlen(doc), XML_TRUE); + +.. there are 0 bytes accounted as direct input and all input from `doc` accounted +as indirect input. Now function accountingGetCurrentAmplification cannot calculate +the current amplification ratio as "(direct + indirect) / direct", and it did refuse +to divide by 0 as one would expect, but it returned 1.0 for this case to indicate +no amplification over direct input. As a result, billion laughs attacks from +DTD-only input were not detected with this isolated way of using an external parser. + +The new approach is to assume direct input of length not 0 but 22 -- derived from +ghost input "", the shortest possible way to include an external +DTD --, and do the usual "(direct + indirect) / direct" math with "direct := 22". + +GitHub issue #839 has more details on this issue and its origin in ClusterFuzz +finding 66812. + +(cherry picked from commit 1d50b80cf31de87750103656f6eb693746854aa8) +--- + lib/xmlparse.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/lib/xmlparse.c b/lib/xmlparse.c +index 2858946d..9458b092 100644 +--- a/lib/xmlparse.c ++++ b/lib/xmlparse.c +@@ -7655,6 +7655,8 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { + + static float + accountingGetCurrentAmplification(XML_Parser rootParser) { ++ // 1.........1.........12 => 22 ++ const size_t lenOfShortestInclude = sizeof("") - 1; + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; +@@ -7662,7 +7664,9 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { + = rootParser->m_accounting.countBytesDirect + ? (countBytesOutput + / (float)(rootParser->m_accounting.countBytesDirect)) +- : 1.0f; ++ : ((lenOfShortestInclude ++ + rootParser->m_accounting.countBytesIndirect) ++ / (float)lenOfShortestInclude); + assert(! rootParser->m_parentParser); + return amplificationFactor; + } +-- +2.44.0 + diff --git a/pkgs/development/libraries/expat/default.nix b/pkgs/development/libraries/expat/default.nix index ac6e9bfdc386..417127d3d2e3 100644 --- a/pkgs/development/libraries/expat/default.nix +++ b/pkgs/development/libraries/expat/default.nix @@ -23,6 +23,11 @@ stdenv.mkDerivation rec { sha256 = "1gnwihpfz4x18rwd6cbrdggmfqjzwsdfh1gpmc0ph21c4gq2097g"; }; + patches = [ + ./CVE-2023-52425.patch + ./CVE-2024-28757.patch + ]; + strictDeps = true; outputs = [ "out" "dev" ]; # TODO: fix referrers diff --git a/pkgs/development/libraries/giflib/4.1.nix b/pkgs/development/libraries/giflib/4.1.nix deleted file mode 100644 index 8f3ebcf7d3be..000000000000 --- a/pkgs/development/libraries/giflib/4.1.nix +++ /dev/null @@ -1,21 +0,0 @@ -{lib, stdenv, fetchurl}: - -stdenv.mkDerivation rec { - pname = "giflib"; - version = "4.1.6"; - - src = fetchurl { - url = "mirror://sourceforge/giflib/giflib-${version}.tar.bz2"; - sha256 = "1v9b7ywz7qg8hli0s9vv1b8q9xxb2xvqq2mg1zpr73xwqpcwxhg1"; - }; - - hardeningDisable = [ "format" ]; - - meta = with lib; { - description = "A library for reading and writing gif images"; - branch = "4.1"; - license = licenses.mit; - platforms = platforms.unix; - }; -} - diff --git a/pkgs/development/libraries/giflib/CVE-2021-40633.patch b/pkgs/development/libraries/giflib/CVE-2021-40633.patch new file mode 100644 index 000000000000..8a665bb1638b --- /dev/null +++ b/pkgs/development/libraries/giflib/CVE-2021-40633.patch @@ -0,0 +1,26 @@ +From ccbc956432650734c91acb3fc88837f7b81267ff Mon Sep 17 00:00:00 2001 +From: "Eric S. Raymond" +Date: Wed, 21 Feb 2024 18:55:00 -0500 +Subject: [PATCH] Clean up memory better at end of run (CVE-2021-40633) + +--- + gif2rgb.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/gif2rgb.c b/gif2rgb.c +index d51226d..fc2e683 100644 +--- a/gif2rgb.c ++++ b/gif2rgb.c +@@ -517,6 +517,9 @@ static void GIF2RGB(int NumFiles, char *FileName, bool OneFileFlag, + DumpScreen2RGB(OutFileName, OneFileFlag, ColorMap, ScreenBuffer, + GifFile->SWidth, GifFile->SHeight); + ++ for (i = 0; i < GifFile->SHeight; i++) { ++ (void)free(ScreenBuffer[i]); ++ } + (void)free(ScreenBuffer); + + { +-- +2.44.0 + diff --git a/pkgs/development/libraries/giflib/default.nix b/pkgs/development/libraries/giflib/default.nix index 8c8a587ed548..486aebf6703a 100644 --- a/pkgs/development/libraries/giflib/default.nix +++ b/pkgs/development/libraries/giflib/default.nix @@ -4,31 +4,20 @@ , fetchpatch , fixDarwinDylibNames , pkgsStatic +, imagemagick_light }: stdenv.mkDerivation rec { pname = "giflib"; - version = "5.2.1"; + version = "5.2.2"; src = fetchurl { url = "mirror://sourceforge/giflib/giflib-${version}.tar.gz"; - sha256 = "1gbrg03z1b6rlrvjyc6d41bc8j1bsr7rm8206gb1apscyii5bnii"; + hash = "sha256-vn/70FfK3r4qoURUL9kMaDjGoIO16KkEi47jtmsp1fs="; }; patches = [ - (fetchpatch { - name = "CVE-2022-28506.patch"; - url = "https://src.fedoraproject.org/rpms/giflib/raw/2e9917bf13df114354163f0c0211eccc00943596/f/CVE-2022-28506.patch"; - sha256 = "sha256-TBemEXkuox8FdS9RvjnWcTWPaHRo4crcwSR9czrUwBY="; - }) - ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # https://sourceforge.net/p/giflib/bugs/133/ - (fetchpatch { - name = "darwin-soname.patch"; - url = "https://sourceforge.net/p/giflib/bugs/_discuss/thread/4e811ad29b/c323/attachment/Makefile.patch"; - sha256 = "12afkqnlkl3n1hywwgx8sqnhp3bz0c5qrwcv8j9hifw1lmfhv67r"; - extraPrefix = "./"; - }) + ./CVE-2021-40633.patch ] ++ lib.optionals stdenv.hostPlatform.isMinGW [ # Build dll libraries. (fetchurl { @@ -40,7 +29,9 @@ stdenv.mkDerivation rec { ./mingw-install-exes.patch ]; - nativeBuildInputs = lib.optionals stdenv.isDarwin [ + nativeBuildInputs = [ + imagemagick_light + ] ++ lib.optionals stdenv.isDarwin [ fixDarwinDylibNames ]; @@ -50,10 +41,11 @@ stdenv.mkDerivation rec { postPatch = lib.optionalString stdenv.hostPlatform.isStatic '' # Upstream build system does not support NOT building shared libraries. - sed -i '/all:/ s/libgif.so//' Makefile - sed -i '/all:/ s/libutil.so//' Makefile - sed -i '/-m 755 libgif.so/ d' Makefile - sed -i '/ln -sf libgif.so/ d' Makefile + sed -i '/all:/ s/$(LIBGIFSO)//' Makefile + sed -i '/all:/ s/$(LIBUTILSO)//' Makefile + sed -i '/-m 755 $(LIBGIFSO)/ d' Makefile + sed -i '/ln -sf $(LIBGIFSOVER)/ d' Makefile + sed -i '/ln -sf $(LIBGIFSOMAJOR)/ d' Makefile ''; passthru.tests = { diff --git a/pkgs/development/libraries/gnutls/default.nix b/pkgs/development/libraries/gnutls/default.nix index b8c95653e366..39115dc6974a 100644 --- a/pkgs/development/libraries/gnutls/default.nix +++ b/pkgs/development/libraries/gnutls/default.nix @@ -1,4 +1,4 @@ -{ config, lib, stdenv, fetchurl, zlib, lzo, libtasn1, nettle, pkg-config, lzip +{ config, lib, stdenv, fetchurl, fetchpatch, zlib, lzo, libtasn1, nettle, pkg-config, lzip , perl, gmp, autoconf, automake, libidn2, libiconv , texinfo , unbound, dns-root-data, gettext, util-linux @@ -49,6 +49,16 @@ stdenv.mkDerivation rec { patches = [ ./nix-ssl-cert-file.patch + (fetchpatch { + name = "CVE-2024-28834.patch"; + url = "https://gitlab.com/gnutls/gnutls/-/commit/1c4701ffc342259fc5965d5a0de90d87f780e3e5.patch"; + hash = "sha256-QMqeEdpNy5MCuZwTPRVKnWMvGkZvIlumigvH3JgcjE4="; + }) + (fetchpatch { + name = "CVE-2024-28835.patch"; + url = "https://gitlab.com/gnutls/gnutls/-/commit/e369e67a62f44561d417cb233acc566cc696d82d.patch"; + hash = "sha256-K8wqypFKfoB7hiFWJPSVVEC7Aei7NGRijorK/tkou9o="; + }) ]; # Skip some tests: diff --git a/pkgs/development/libraries/libarchive/default.nix b/pkgs/development/libraries/libarchive/default.nix index d58ba0bc5c5c..e98cf3469221 100644 --- a/pkgs/development/libraries/libarchive/default.nix +++ b/pkgs/development/libraries/libarchive/default.nix @@ -44,6 +44,11 @@ stdenv.mkDerivation (finalAttrs: { url = "https://github.com/libarchive/libarchive/commit/3bd918d92f8c34ba12de9c6604d96f9e262a59fc.patch"; hash = "sha256-RM3xFM6S2DkM5DJ0kAba8eLzEXuY5/7AaU06maHJ6rM="; }) + (fetchpatch { + name = "fix-suspicious-commit-from-known-bad-actor.patch"; + url = "https://github.com/libarchive/libarchive/commit/6110e9c82d8ba830c3440f36b990483ceaaea52c.patch"; + hash = "sha256-/j6rJ0xWhtXU0YCu1LOokxxNppy5Of6Q0XyO4U6la7M="; + }) ]; outputs = [ "out" "lib" "dev" ]; diff --git a/pkgs/development/libraries/qpdf/11.6.1-CVE-2024-24246.patch b/pkgs/development/libraries/qpdf/11.6.1-CVE-2024-24246.patch new file mode 100644 index 000000000000..3a97ac1f629d --- /dev/null +++ b/pkgs/development/libraries/qpdf/11.6.1-CVE-2024-24246.patch @@ -0,0 +1,826 @@ +Based on upstream cb0f390cc1f98a8e82b27259f8f3cd5f162992eb, adjusted to +apply to 11.6.1 + +diff --git a/libqpdf/QPDF_json.cc b/libqpdf/QPDF_json.cc +index f8fd689a..7a712010 100644 +--- a/libqpdf/QPDF_json.cc ++++ b/libqpdf/QPDF_json.cc +@@ -14,7 +14,7 @@ + + // This chart shows an example of the state transitions that would occur in parsing a minimal file. + +-// | st_initial ++// | + // { | -> st_top + // "qpdf": [ | -> st_qpdf + // { | -> st_qpdf_meta +@@ -47,7 +47,7 @@ + // } | <- st_objects + // } | <- st_qpdf + // ] | <- st_top +-// } | <- st_initial ++// } | + + static char const* JSON_PDF = ( + // force line break +@@ -99,7 +99,7 @@ is_indirect_object(std::string const& v, int& obj, int& gen) + } + obj = QUtil::string_to_int(o_str.c_str()); + gen = QUtil::string_to_int(g_str.c_str()); +- return true; ++ return obj > 0; + } + + static bool +@@ -250,7 +250,6 @@ class QPDF::JSONReactor: public JSON::Reactor + + private: + enum state_e { +- st_initial, + st_top, + st_qpdf, + st_qpdf_meta, +@@ -262,28 +261,35 @@ class QPDF::JSONReactor: public JSON::Reactor + st_ignore, + }; + ++ struct StackFrame ++ { ++ StackFrame(state_e state) : ++ state(state){}; ++ StackFrame(state_e state, QPDFObjectHandle&& object) : ++ state(state), ++ object(object){}; ++ state_e state; ++ QPDFObjectHandle object; ++ }; ++ + void containerStart(); +- void nestedState(std::string const& key, JSON const& value, state_e); ++ bool setNextStateIfDictionary(std::string const& key, JSON const& value, state_e); + void setObjectDescription(QPDFObjectHandle& oh, JSON const& value); + QPDFObjectHandle makeObject(JSON const& value); + void error(qpdf_offset_t offset, std::string const& message); +- void +- replaceObject(QPDFObjectHandle to_replace, QPDFObjectHandle replacement, JSON const& value); ++ void replaceObject(QPDFObjectHandle&& replacement, JSON const& value); + + QPDF& pdf; + std::shared_ptr is; + bool must_be_complete{true}; + std::shared_ptr descr; + bool errors{false}; +- bool parse_error{false}; + bool saw_qpdf{false}; + bool saw_qpdf_meta{false}; + bool saw_objects{false}; + bool saw_json_version{false}; + bool saw_pdf_version{false}; + bool saw_trailer{false}; +- state_e state{st_initial}; +- state_e next_state{st_top}; + std::string cur_object; + bool saw_value{false}; + bool saw_stream{false}; +@@ -291,9 +297,10 @@ class QPDF::JSONReactor: public JSON::Reactor + bool saw_data{false}; + bool saw_datafile{false}; + bool this_stream_needs_data{false}; +- std::vector state_stack{st_initial}; +- std::vector object_stack; + std::set reserved; ++ std::vector stack; ++ QPDFObjectHandle next_obj; ++ state_e next_state{st_top}; + }; + + void +@@ -316,8 +323,12 @@ QPDF::JSONReactor::anyErrors() const + void + QPDF::JSONReactor::containerStart() + { +- state_stack.push_back(state); +- state = next_state; ++ if (next_obj.isInitialized()) { ++ stack.emplace_back(next_state, std::move(next_obj)); ++ next_obj = QPDFObjectHandle(); ++ } else { ++ stack.emplace_back(next_state); ++ } + } + + void +@@ -329,20 +340,19 @@ QPDF::JSONReactor::dictionaryStart() + void + QPDF::JSONReactor::arrayStart() + { +- containerStart(); +- if (state == st_top) { ++ if (stack.empty()) { + QTC::TC("qpdf", "QPDF_json top-level array"); + throw std::runtime_error("QPDF JSON must be a dictionary"); + } ++ containerStart(); + } + + void + QPDF::JSONReactor::containerEnd(JSON const& value) + { +- auto from_state = state; +- state = state_stack.back(); +- state_stack.pop_back(); +- if (state == st_initial) { ++ auto from_state = stack.back().state; ++ stack.pop_back(); ++ if (stack.empty()) { + if (!this->saw_qpdf) { + QTC::TC("qpdf", "QPDF_json missing qpdf"); + error(0, "\"qpdf\" object was not seen"); +@@ -365,26 +375,16 @@ QPDF::JSONReactor::containerEnd(JSON const& value) + } + } + } +- } else if (state == st_objects) { +- if (parse_error) { +- QTC::TC("qpdf", "QPDF_json don't check object after parse error"); +- } else if (cur_object == "trailer") { +- if (!saw_value) { +- QTC::TC("qpdf", "QPDF_json trailer no value"); +- error(value.getStart(), "\"trailer\" is missing \"value\""); +- } +- } else if (saw_value == saw_stream) { ++ } else if (from_state == st_trailer) { ++ if (!saw_value) { ++ QTC::TC("qpdf", "QPDF_json trailer no value"); ++ error(value.getStart(), "\"trailer\" is missing \"value\""); ++ } ++ } else if (from_state == st_object_top) { ++ if (saw_value == saw_stream) { + QTC::TC("qpdf", "QPDF_json value stream both or neither"); + error(value.getStart(), "object must have exactly one of \"value\" or \"stream\""); + } +- object_stack.clear(); +- this->cur_object = ""; +- this->saw_dict = false; +- this->saw_data = false; +- this->saw_datafile = false; +- this->saw_value = false; +- this->saw_stream = false; +- } else if (state == st_object_top) { + if (saw_stream) { + if (!saw_dict) { + QTC::TC("qpdf", "QPDF_json stream no dict"); +@@ -408,11 +408,7 @@ QPDF::JSONReactor::containerEnd(JSON const& value) + } + } + } +- } else if ((state == st_stream) || (state == st_object)) { +- if (!parse_error) { +- object_stack.pop_back(); +- } +- } else if ((state == st_top) && (from_state == st_qpdf)) { ++ } else if (from_state == st_qpdf) { + // Handle dangling indirect object references which the PDF spec says to treat as nulls. + // It's tempting to make this an error, but that would be wrong since valid input files may + // have these. +@@ -423,16 +419,27 @@ QPDF::JSONReactor::containerEnd(JSON const& value) + } + } + } ++ if (!stack.empty()) { ++ auto state = stack.back().state; ++ if (state == st_objects) { ++ this->cur_object = ""; ++ this->saw_dict = false; ++ this->saw_data = false; ++ this->saw_datafile = false; ++ this->saw_value = false; ++ this->saw_stream = false; ++ } ++ } + } + + void +-QPDF::JSONReactor::replaceObject( +- QPDFObjectHandle to_replace, QPDFObjectHandle replacement, JSON const& value) ++QPDF::JSONReactor::replaceObject(QPDFObjectHandle&& replacement, JSON const& value) + { +- auto og = to_replace.getObjGen(); ++ auto& tos = stack.back(); ++ auto og = tos.object.getObjGen(); + this->pdf.replaceObject(og, replacement); +- auto oh = pdf.getObject(og); +- setObjectDescription(oh, value); ++ next_obj = pdf.getObject(og); ++ setObjectDescription(tos.object, value); + } + + void +@@ -442,22 +449,26 @@ QPDF::JSONReactor::topLevelScalar() + throw std::runtime_error("QPDF JSON must be a dictionary"); + } + +-void +-QPDF::JSONReactor::nestedState(std::string const& key, JSON const& value, state_e next) ++bool ++QPDF::JSONReactor::setNextStateIfDictionary(std::string const& key, JSON const& value, state_e next) + { + // Use this method when the next state is for processing a nested dictionary. + if (value.isDictionary()) { + this->next_state = next; +- } else { +- error(value.getStart(), "\"" + key + "\" must be a dictionary"); +- this->next_state = st_ignore; +- this->parse_error = true; ++ return true; + } ++ error(value.getStart(), "\"" + key + "\" must be a dictionary"); ++ return false; + } + + bool + QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + { ++ if (stack.empty()) { ++ throw std::logic_error("stack is empty in dictionaryItem"); ++ } ++ next_state = st_ignore; ++ auto state = stack.back().state; + if (state == st_ignore) { + QTC::TC("qpdf", "QPDF_json ignoring in st_ignore"); + // ignore +@@ -467,51 +478,48 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + if (!value.isArray()) { + QTC::TC("qpdf", "QPDF_json qpdf not array"); + error(value.getStart(), "\"qpdf\" must be an array"); +- next_state = st_ignore; +- parse_error = true; + } else { + next_state = st_qpdf; + } + } else { + // Ignore all other fields. + QTC::TC("qpdf", "QPDF_json ignoring unknown top-level key"); +- next_state = st_ignore; + } + } else if (state == st_qpdf_meta) { + if (key == "pdfversion") { + this->saw_pdf_version = true; +- bool version_okay = false; + std::string v; ++ bool okay = false; + if (value.getString(v)) { + std::string version; + char const* p = v.c_str(); + if (QPDF::validatePDFVersion(p, version) && (*p == '\0')) { +- version_okay = true; + this->pdf.m->pdf_version = version; ++ okay = true; + } + } +- if (!version_okay) { ++ if (!okay) { + QTC::TC("qpdf", "QPDF_json bad pdf version"); +- error(value.getStart(), "invalid PDF version (must be x.y)"); ++ error(value.getStart(), "invalid PDF version (must be \"x.y\")"); + } + } else if (key == "jsonversion") { + this->saw_json_version = true; +- bool version_okay = false; + std::string v; ++ bool okay = false; + if (value.getNumber(v)) { + std::string version; + if (QUtil::string_to_int(v.c_str()) == 2) { +- version_okay = true; ++ okay = true; + } + } +- if (!version_okay) { ++ if (!okay) { + QTC::TC("qpdf", "QPDF_json bad json version"); +- error(value.getStart(), "invalid JSON version (must be 2)"); ++ error(value.getStart(), "invalid JSON version (must be numeric value 2)"); + } + } else if (key == "pushedinheritedpageresources") { + bool v; + if (value.getBool(v)) { +- if ((!this->must_be_complete) && v) { ++ if (!this->must_be_complete && v) { + this->pdf.pushInheritedAttributesToPage(); + } + } else { +@@ -521,7 +529,7 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + } else if (key == "calledgetallpages") { + bool v; + if (value.getBool(v)) { +- if ((!this->must_be_complete) && v) { ++ if (!this->must_be_complete && v) { + this->pdf.getAllPages(); + } + } else { +@@ -532,103 +540,95 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + // ignore unknown keys for forward compatibility and to skip keys we don't care about + // like "maxobjectid". + QTC::TC("qpdf", "QPDF_json ignore second-level key"); +- next_state = st_ignore; + } + } else if (state == st_objects) { + int obj = 0; + int gen = 0; + if (key == "trailer") { + this->saw_trailer = true; +- nestedState(key, value, st_trailer); + this->cur_object = "trailer"; ++ setNextStateIfDictionary(key, value, st_trailer); + } else if (is_obj_key(key, obj, gen)) { + this->cur_object = key; +- auto oh = pdf.reserveObjectIfNotExists(QPDFObjGen(obj, gen)); +- object_stack.push_back(oh); +- nestedState(key, value, st_object_top); ++ if (setNextStateIfDictionary(key, value, st_object_top)) { ++ next_obj = pdf.reserveObjectIfNotExists(QPDFObjGen(obj, gen)); ++ } + } else { + QTC::TC("qpdf", "QPDF_json bad object key"); + error(value.getStart(), "object key should be \"trailer\" or \"obj:n n R\""); +- next_state = st_ignore; +- parse_error = true; + } + } else if (state == st_object_top) { +- if (object_stack.size() == 0) { +- throw std::logic_error("no object on stack in st_object_top"); ++ if (stack.empty()) { ++ throw std::logic_error("stack empty in st_object_top"); ++ } ++ auto& tos = stack.back(); ++ if (!tos.object.isInitialized()) { ++ throw std::logic_error("current object uninitialized in st_object_top"); + } +- auto tos = object_stack.back(); +- QPDFObjectHandle replacement; + if (key == "value") { +- // Don't use nestedState since this can have any type. ++ // Don't use setNextStateIfDictionary since this can have any type. + this->saw_value = true; ++ replaceObject(makeObject(value), value); + next_state = st_object; +- replacement = makeObject(value); +- replaceObject(tos, replacement, value); + } else if (key == "stream") { + this->saw_stream = true; +- nestedState(key, value, st_stream); +- this->this_stream_needs_data = false; +- if (tos.isStream()) { +- QTC::TC("qpdf", "QPDF_json updating existing stream"); ++ if (setNextStateIfDictionary(key, value, st_stream)) { ++ this->this_stream_needs_data = false; ++ if (tos.object.isStream()) { ++ QTC::TC("qpdf", "QPDF_json updating existing stream"); ++ } else { ++ this->this_stream_needs_data = true; ++ replaceObject(pdf.reserveStream(tos.object.getObjGen()), value); ++ } ++ next_obj = tos.object; + } else { +- this->this_stream_needs_data = true; +- replacement = pdf.reserveStream(tos.getObjGen()); +- replaceObject(tos, replacement, value); ++ // Error message already given above ++ QTC::TC("qpdf", "QPDF_json stream not a dictionary"); + } + } else { + // Ignore unknown keys for forward compatibility + QTC::TC("qpdf", "QPDF_json ignore unknown key in object_top"); +- next_state = st_ignore; +- } +- if (replacement.isInitialized()) { +- object_stack.pop_back(); +- object_stack.push_back(replacement); + } + } else if (state == st_trailer) { + if (key == "value") { + this->saw_value = true; +- // The trailer must be a dictionary, so we can use nestedState. +- nestedState("trailer.value", value, st_object); +- this->pdf.m->trailer = makeObject(value); +- setObjectDescription(this->pdf.m->trailer, value); ++ // The trailer must be a dictionary, so we can use setNextStateIfDictionary. ++ if (setNextStateIfDictionary("trailer.value", value, st_object)) { ++ this->pdf.m->trailer = makeObject(value); ++ setObjectDescription(this->pdf.m->trailer, value); ++ } + } else if (key == "stream") { + // Don't need to set saw_stream here since there's already an error. + QTC::TC("qpdf", "QPDF_json trailer stream"); + error(value.getStart(), "the trailer may not be a stream"); +- next_state = st_ignore; +- parse_error = true; + } else { + // Ignore unknown keys for forward compatibility + QTC::TC("qpdf", "QPDF_json ignore unknown key in trailer"); +- next_state = st_ignore; + } + } else if (state == st_stream) { +- if (object_stack.size() == 0) { +- throw std::logic_error("no object on stack in st_stream"); ++ if (stack.empty()) { ++ throw std::logic_error("stack empty in st_stream"); + } +- auto tos = object_stack.back(); +- if (!tos.isStream()) { +- throw std::logic_error("top of stack is not stream in st_stream"); ++ auto& tos = stack.back(); ++ if (!tos.object.isStream()) { ++ throw std::logic_error("current object is not stream in st_stream"); + } + auto uninitialized = QPDFObjectHandle(); + if (key == "dict") { + this->saw_dict = true; +- // Since a stream dictionary must be a dictionary, we can use nestedState to transition +- // to st_value. +- nestedState("stream.dict", value, st_object); +- auto dict = makeObject(value); +- if (dict.isDictionary()) { +- tos.replaceDict(dict); ++ if (setNextStateIfDictionary("stream.dict", value, st_object)) { ++ tos.object.replaceDict(makeObject(value)); + } else { +- // An error had already been given by nestedState ++ // An error had already been given by setNextStateIfDictionary + QTC::TC("qpdf", "QPDF_json stream dict not dict"); +- parse_error = true; + } + } else if (key == "data") { + this->saw_data = true; + std::string v; + if (!value.getString(v)) { ++ QTC::TC("qpdf", "QPDF_json stream data not string"); + error(value.getStart(), "\"stream.data\" must be a string"); ++ tos.object.replaceStreamData("", uninitialized, uninitialized); + } else { + // The range includes the quotes. + auto start = value.getStart() + 1; +@@ -636,32 +636,40 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + if (end < start) { + throw std::logic_error("QPDF_json: JSON string length < 0"); + } +- tos.replaceStreamData(provide_data(is, start, end), uninitialized, uninitialized); ++ tos.object.replaceStreamData( ++ provide_data(is, start, end), uninitialized, uninitialized); + } + } else if (key == "datafile") { + this->saw_datafile = true; + std::string filename; +- if (value.getString(filename)) { +- tos.replaceStreamData(QUtil::file_provider(filename), uninitialized, uninitialized); +- } else { ++ if (!value.getString(filename)) { ++ QTC::TC("qpdf", "QPDF_json stream datafile not string"); + error( + value.getStart(), +- "\"stream.datafile\" must be a string containing a file " +- "name"); ++ "\"stream.datafile\" must be a string containing a file name"); ++ tos.object.replaceStreamData("", uninitialized, uninitialized); ++ } else { ++ tos.object.replaceStreamData( ++ QUtil::file_provider(filename), uninitialized, uninitialized); + } + } else { + // Ignore unknown keys for forward compatibility. + QTC::TC("qpdf", "QPDF_json ignore unknown key in stream"); +- next_state = st_ignore; + } + } else if (state == st_object) { +- if (!parse_error) { +- auto dict = object_stack.back(); +- if (dict.isStream()) { +- dict = dict.getDict(); +- } +- dict.replaceKey(key, makeObject(value)); ++ if (stack.empty()) { ++ throw std::logic_error("stack empty in st_object"); ++ } ++ auto& tos = stack.back(); ++ auto dict = tos.object; ++ if (dict.isStream()) { ++ dict = dict.getDict(); ++ } ++ if (!dict.isDictionary()) { ++ throw std::logic_error( ++ "current object is not stream or dictionary in st_object dictionary item"); + } ++ dict.replaceKey(key, makeObject(value)); + } else { + throw std::logic_error("QPDF_json: unknown state " + std::to_string(state)); + } +@@ -671,25 +679,24 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) + bool + QPDF::JSONReactor::arrayItem(JSON const& value) + { ++ if (stack.empty()) { ++ throw std::logic_error("stack is empty in arrayItem"); ++ } ++ next_state = st_ignore; ++ auto state = stack.back().state; + if (state == st_qpdf) { + if (!this->saw_qpdf_meta) { + this->saw_qpdf_meta = true; +- nestedState("qpdf[0]", value, st_qpdf_meta); ++ setNextStateIfDictionary("qpdf[0]", value, st_qpdf_meta); + } else if (!this->saw_objects) { + this->saw_objects = true; +- nestedState("qpdf[1]", value, st_objects); ++ setNextStateIfDictionary("qpdf[1]", value, st_objects); + } else { + QTC::TC("qpdf", "QPDF_json more than two qpdf elements"); + error(value.getStart(), "\"qpdf\" must have two elements"); +- next_state = st_ignore; +- parse_error = true; +- } +- } +- if (state == st_object) { +- if (!parse_error) { +- auto tos = object_stack.back(); +- tos.appendItem(makeObject(value)); + } ++ } else if (state == st_object) { ++ stack.back().object.appendItem(makeObject(value)); + } + return true; + } +@@ -714,10 +721,12 @@ QPDF::JSONReactor::makeObject(JSON const& value) + bool bool_v = false; + if (value.isDictionary()) { + result = QPDFObjectHandle::newDictionary(); +- object_stack.push_back(result); ++ next_obj = result; ++ next_state = st_object; + } else if (value.isArray()) { + result = QPDFObjectHandle::newArray(); +- object_stack.push_back(result); ++ next_obj = result; ++ next_state = st_object; + } else if (value.isNull()) { + result = QPDFObjectHandle::newNull(); + } else if (value.getBool(bool_v)) { +diff --git a/qpdf/qtest/qpdf-json.test b/qpdf/qtest/qpdf-json.test +index 2867f8a7..4982d440 100644 +--- a/qpdf/qtest/qpdf-json.test ++++ b/qpdf/qtest/qpdf-json.test +@@ -37,6 +37,8 @@ my @badfiles = ( + 'obj-key-errors', + 'bad-data', + 'bad-datafile', ++ 'bad-data2', ++ 'bad-datafile2', + ); + + $n_tests += scalar(@badfiles); +diff --git a/qpdf/qtest/qpdf/qjson-bad-data2.json b/qpdf/qtest/qpdf/qjson-bad-data2.json +new file mode 100644 +index 00000000..80206086 +--- /dev/null ++++ b/qpdf/qtest/qpdf/qjson-bad-data2.json +@@ -0,0 +1,71 @@ ++{ ++ "qpdf": [ ++ { ++ "jsonversion": 2, ++ "pdfversion": "1.3", ++ "maxobjectid": 6 ++ }, ++ { ++ "obj:1 0 R": { ++ "value": { ++ "/Pages": "2 0 R", ++ "/Type": "/Catalog" ++ } ++ }, ++ "obj:2 0 R": { ++ "value": { ++ "/Count": 1, ++ "/Kids": [ ++ "3 0 R" ++ ], ++ "/Type": "/Pages" ++ } ++ }, ++ "obj:3 0 R": { ++ "value": { ++ "/Contents": ["4 0 R", "7 0 R"], ++ "/MediaBox": [ ++ 0, ++ 0, ++ 612, ++ 792 ++ ], ++ "/Parent": "2 0 R", ++ "/Resources": { ++ "/Font": { ++ "/F1": "6 0 R" ++ }, ++ "/ProcSet": "5 0 R" ++ }, ++ "/Type": "/Page" ++ } ++ }, ++ "obj:4 0 R": { ++ "stream": { ++ "data": [[]], ++ "dict": {} ++ } ++ }, ++ "obj:5 0 R": { ++ "value": [ ++ "/PDF", ++ "/Text" ++ ] ++ }, ++ "obj:6 0 R": { ++ "value": { ++ "/BaseFont": "/Helvetica", ++ "/Encoding": "/WinAnsiEncoding", ++ "/Subtype": "/Type1", ++ "/Type": "/Font" ++ } ++ }, ++ "trailer": { ++ "value": { ++ "/Root": "1 0 R", ++ "/Size": 7 ++ } ++ } ++ } ++ ] ++} +diff --git a/qpdf/qtest/qpdf/qjson-bad-data2.out b/qpdf/qtest/qpdf/qjson-bad-data2.out +new file mode 100644 +index 00000000..47c83c8e +--- /dev/null ++++ b/qpdf/qtest/qpdf/qjson-bad-data2.out +@@ -0,0 +1,2 @@ ++WARNING: qjson-bad-data2.json (obj:4 0 R, offset 846): "stream.data" must be a string ++qpdf: qjson-bad-data2.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-bad-datafile2.json b/qpdf/qtest/qpdf/qjson-bad-datafile2.json +new file mode 100644 +index 00000000..b5c99820 +--- /dev/null ++++ b/qpdf/qtest/qpdf/qjson-bad-datafile2.json +@@ -0,0 +1,71 @@ ++{ ++ "qpdf": [ ++ { ++ "jsonversion": 2, ++ "pdfversion": "1.3", ++ "maxobjectid": 6 ++ }, ++ { ++ "obj:1 0 R": { ++ "value": { ++ "/Pages": "2 0 R", ++ "/Type": "/Catalog" ++ } ++ }, ++ "obj:2 0 R": { ++ "value": { ++ "/Count": 1, ++ "/Kids": [ ++ "3 0 R" ++ ], ++ "/Type": "/Pages" ++ } ++ }, ++ "obj:3 0 R": { ++ "value": { ++ "/Contents": ["4 0 R", "7 0 R"], ++ "/MediaBox": [ ++ 0, ++ 0, ++ 612, ++ 792 ++ ], ++ "/Parent": "2 0 R", ++ "/Resources": { ++ "/Font": { ++ "/F1": "6 0 R" ++ }, ++ "/ProcSet": "5 0 R" ++ }, ++ "/Type": "/Page" ++ } ++ }, ++ "obj:4 0 R": { ++ "stream": { ++ "datafile": [[]], ++ "dict": {} ++ } ++ }, ++ "obj:5 0 R": { ++ "value": [ ++ "/PDF", ++ "/Text" ++ ] ++ }, ++ "obj:6 0 R": { ++ "value": { ++ "/BaseFont": "/Helvetica", ++ "/Encoding": "/WinAnsiEncoding", ++ "/Subtype": "/Type1", ++ "/Type": "/Font" ++ } ++ }, ++ "trailer": { ++ "value": { ++ "/Root": "1 0 R", ++ "/Size": 7 ++ } ++ } ++ } ++ ] ++} +diff --git a/qpdf/qtest/qpdf/qjson-bad-datafile2.out b/qpdf/qtest/qpdf/qjson-bad-datafile2.out +new file mode 100644 +index 00000000..41949a21 +--- /dev/null ++++ b/qpdf/qtest/qpdf/qjson-bad-datafile2.out +@@ -0,0 +1,2 @@ ++WARNING: qjson-bad-datafile2.json (obj:4 0 R, offset 850): "stream.datafile" must be a string containing a file name ++qpdf: qjson-bad-datafile2.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-bad-pdf-version1.out b/qpdf/qtest/qpdf/qjson-bad-pdf-version1.out +index f364f1a6..476128e7 100644 +--- a/qpdf/qtest/qpdf/qjson-bad-pdf-version1.out ++++ b/qpdf/qtest/qpdf/qjson-bad-pdf-version1.out +@@ -1,3 +1,3 @@ +-WARNING: qjson-bad-pdf-version1.json (offset 41): invalid JSON version (must be 2) +-WARNING: qjson-bad-pdf-version1.json (offset 70): invalid PDF version (must be x.y) ++WARNING: qjson-bad-pdf-version1.json (offset 41): invalid JSON version (must be numeric value 2) ++WARNING: qjson-bad-pdf-version1.json (offset 70): invalid PDF version (must be "x.y") + qpdf: qjson-bad-pdf-version1.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-bad-pdf-version2.out b/qpdf/qtest/qpdf/qjson-bad-pdf-version2.out +index 9bc88ff4..cb914414 100644 +--- a/qpdf/qtest/qpdf/qjson-bad-pdf-version2.out ++++ b/qpdf/qtest/qpdf/qjson-bad-pdf-version2.out +@@ -1,5 +1,5 @@ +-WARNING: qjson-bad-pdf-version2.json (offset 41): invalid JSON version (must be 2) +-WARNING: qjson-bad-pdf-version2.json (offset 66): invalid PDF version (must be x.y) ++WARNING: qjson-bad-pdf-version2.json (offset 41): invalid JSON version (must be numeric value 2) ++WARNING: qjson-bad-pdf-version2.json (offset 66): invalid PDF version (must be "x.y") + WARNING: qjson-bad-pdf-version2.json (offset 97): calledgetallpages must be a boolean + WARNING: qjson-bad-pdf-version2.json (offset 138): pushedinheritedpageresources must be a boolean + qpdf: qjson-bad-pdf-version2.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-obj-key-errors.out b/qpdf/qtest/qpdf/qjson-obj-key-errors.out +index 0263f294..f1f0a369 100644 +--- a/qpdf/qtest/qpdf/qjson-obj-key-errors.out ++++ b/qpdf/qtest/qpdf/qjson-obj-key-errors.out +@@ -1,7 +1,7 @@ + WARNING: qjson-obj-key-errors.json (obj:2 0 R, offset 244): object must have exactly one of "value" or "stream" + WARNING: qjson-obj-key-errors.json (obj:3 0 R, offset 542): object must have exactly one of "value" or "stream" +-WARNING: qjson-obj-key-errors.json (obj:4 0 R, offset 710): "stream" is missing "dict" +-WARNING: qjson-obj-key-errors.json (obj:4 0 R, offset 710): new "stream" must have exactly one of "data" or "datafile" +-WARNING: qjson-obj-key-errors.json (obj:5 0 R, offset 800): new "stream" must have exactly one of "data" or "datafile" ++WARNING: qjson-obj-key-errors.json (obj:4 0 R, offset 690): "stream" is missing "dict" ++WARNING: qjson-obj-key-errors.json (obj:4 0 R, offset 690): new "stream" must have exactly one of "data" or "datafile" ++WARNING: qjson-obj-key-errors.json (obj:5 0 R, offset 780): new "stream" must have exactly one of "data" or "datafile" + WARNING: qjson-obj-key-errors.json (trailer, offset 1178): "trailer" is missing "value" + qpdf: qjson-obj-key-errors.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-stream-dict-not-dict.out b/qpdf/qtest/qpdf/qjson-stream-dict-not-dict.out +index a264839f..04df1518 100644 +--- a/qpdf/qtest/qpdf/qjson-stream-dict-not-dict.out ++++ b/qpdf/qtest/qpdf/qjson-stream-dict-not-dict.out +@@ -1,5 +1,4 @@ + WARNING: qjson-stream-dict-not-dict.json (obj:1 0 R, offset 142): "stream.dict" must be a dictionary +-WARNING: qjson-stream-dict-not-dict.json (obj:1 0 R, offset 142): unrecognized string value +-WARNING: qjson-stream-dict-not-dict.json (obj:1 0 R, offset 122): new "stream" must have exactly one of "data" or "datafile" ++WARNING: qjson-stream-dict-not-dict.json (obj:1 0 R, offset 102): new "stream" must have exactly one of "data" or "datafile" + WARNING: qjson-stream-dict-not-dict.json: "qpdf[1].trailer" was not seen + qpdf: qjson-stream-dict-not-dict.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-stream-not-dict.out b/qpdf/qtest/qpdf/qjson-stream-not-dict.out +index fbd953c6..db775b59 100644 +--- a/qpdf/qtest/qpdf/qjson-stream-not-dict.out ++++ b/qpdf/qtest/qpdf/qjson-stream-not-dict.out +@@ -1,3 +1,4 @@ + WARNING: qjson-stream-not-dict.json (obj:1 0 R, offset 122): "stream" must be a dictionary ++WARNING: qjson-stream-not-dict.json (obj:1 0 R, offset 102): "stream" is missing "dict" + WARNING: qjson-stream-not-dict.json: "qpdf[1].trailer" was not seen + qpdf: qjson-stream-not-dict.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/qjson-trailer-stream.out b/qpdf/qtest/qpdf/qjson-trailer-stream.out +index a625cd6d..fccb2a39 100644 +--- a/qpdf/qtest/qpdf/qjson-trailer-stream.out ++++ b/qpdf/qtest/qpdf/qjson-trailer-stream.out +@@ -1,2 +1,3 @@ + WARNING: qjson-trailer-stream.json (trailer, offset 1269): the trailer may not be a stream ++WARNING: qjson-trailer-stream.json (trailer, offset 1249): "trailer" is missing "value" + qpdf: qjson-trailer-stream.json: errors found in JSON +diff --git a/qpdf/qtest/qpdf/update-from-json-errors.out b/qpdf/qtest/qpdf/update-from-json-errors.out +index 530d707d..5e136c55 100644 +--- a/qpdf/qtest/qpdf/update-from-json-errors.out ++++ b/qpdf/qtest/qpdf/update-from-json-errors.out +@@ -1,4 +1,4 @@ +-WARNING: good13.pdf (obj:4 0 R from qpdf-json-update-errors.json, offset 95): existing "stream" may at most one of "data" or "datafile" ++WARNING: good13.pdf (obj:4 0 R from qpdf-json-update-errors.json, offset 75): existing "stream" may at most one of "data" or "datafile" + WARNING: good13.pdf (obj:20 0 R from qpdf-json-update-errors.json, offset 335): unrecognized string value +-WARNING: good13.pdf (obj:20 0 R from qpdf-json-update-errors.json, offset 293): new "stream" must have exactly one of "data" or "datafile" ++WARNING: good13.pdf (obj:20 0 R from qpdf-json-update-errors.json, offset 273): new "stream" must have exactly one of "data" or "datafile" + qpdf: qpdf-json-update-errors.json: errors found in JSON diff --git a/pkgs/development/libraries/qpdf/default.nix b/pkgs/development/libraries/qpdf/default.nix index d80309f2b16b..6d8ab1ac5132 100644 --- a/pkgs/development/libraries/qpdf/default.nix +++ b/pkgs/development/libraries/qpdf/default.nix @@ -11,6 +11,10 @@ stdenv.mkDerivation rec { hash = "sha256-QXRzvSMi6gKISJo44KIjTYENNqxh1yDhUUhEZa8uz6Q="; }; + patches = [ + ./11.6.1-CVE-2024-24246.patch + ]; + nativeBuildInputs = [ cmake perl ]; buildInputs = [ zlib libjpeg ]; diff --git a/pkgs/development/python-modules/aiohttp/3.8.6-CVE-2024-23334.patch b/pkgs/development/python-modules/aiohttp/3.8.6-CVE-2024-23334.patch new file mode 100644 index 000000000000..b76671ae6983 --- /dev/null +++ b/pkgs/development/python-modules/aiohttp/3.8.6-CVE-2024-23334.patch @@ -0,0 +1,203 @@ +Based on upstream 1c335944d6a8b1298baf179b7c0b3069f10c514b with a couple +of type hints removed from the added tests because they refer to type +aliases that don't exist in 3.8.6 + +diff --git a/CHANGES/8079.bugfix.rst b/CHANGES/8079.bugfix.rst +new file mode 100644 +index 00000000..57bc8bfe +--- /dev/null ++++ b/CHANGES/8079.bugfix.rst +@@ -0,0 +1 @@ ++Improved validation of paths for static resources -- by :user:`bdraco`. +diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py +index 5942e355..e8a8023e 100644 +--- a/aiohttp/web_urldispatcher.py ++++ b/aiohttp/web_urldispatcher.py +@@ -593,9 +593,14 @@ class StaticResource(PrefixResource): + url = url / filename + + if append_version: ++ unresolved_path = self._directory.joinpath(filename) + try: +- filepath = self._directory.joinpath(filename).resolve() +- if not self._follow_symlinks: ++ if self._follow_symlinks: ++ normalized_path = Path(os.path.normpath(unresolved_path)) ++ normalized_path.relative_to(self._directory) ++ filepath = normalized_path.resolve() ++ else: ++ filepath = unresolved_path.resolve() + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink +@@ -660,8 +665,13 @@ class StaticResource(PrefixResource): + # /static/\\machine_name\c$ or /static/D:\path + # where the static dir is totally different + raise HTTPForbidden() +- filepath = self._directory.joinpath(filename).resolve() +- if not self._follow_symlinks: ++ unresolved_path = self._directory.joinpath(filename) ++ if self._follow_symlinks: ++ normalized_path = Path(os.path.normpath(unresolved_path)) ++ normalized_path.relative_to(self._directory) ++ filepath = normalized_path.resolve() ++ else: ++ filepath = unresolved_path.resolve() + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError) as error: + # relatively safe +diff --git a/docs/web_advanced.rst b/docs/web_advanced.rst +index 3a98b78a..51293970 100644 +--- a/docs/web_advanced.rst ++++ b/docs/web_advanced.rst +@@ -136,12 +136,22 @@ instead could be enabled with ``show_index`` parameter set to ``True``:: + + web.static('/prefix', path_to_static_folder, show_index=True) + +-When a symlink from the static directory is accessed, the server responses to +-client with ``HTTP/404 Not Found`` by default. To allow the server to follow +-symlinks, parameter ``follow_symlinks`` should be set to ``True``:: ++When a symlink that leads outside the static directory is accessed, the server ++responds to the client with ``HTTP/404 Not Found`` by default. To allow the server to ++follow symlinks that lead outside the static root, the parameter ``follow_symlinks`` ++should be set to ``True``:: + + web.static('/prefix', path_to_static_folder, follow_symlinks=True) + ++.. caution:: ++ ++ Enabling ``follow_symlinks`` can be a security risk, and may lead to ++ a directory transversal attack. You do NOT need this option to follow symlinks ++ which point to somewhere else within the static directory, this option is only ++ used to break out of the security sandbox. Enabling this option is highly ++ discouraged, and only expected to be used for edge cases in a local ++ development setting where remote users do not have access to the server. ++ + When you want to enable cache busting, + parameter ``append_version`` can be set to ``True`` + +diff --git a/docs/web_reference.rst b/docs/web_reference.rst +index a156f47d..b1006764 100644 +--- a/docs/web_reference.rst ++++ b/docs/web_reference.rst +@@ -1836,9 +1836,15 @@ Router is any object that implements :class:`~aiohttp.abc.AbstractRouter` interf + by default it's not allowed and HTTP/403 will + be returned on directory access. + +- :param bool follow_symlinks: flag for allowing to follow symlinks from +- a directory, by default it's not allowed and +- HTTP/404 will be returned on access. ++ :param bool follow_symlinks: flag for allowing to follow symlinks that lead ++ outside the static root directory, by default it's not allowed and ++ HTTP/404 will be returned on access. Enabling ``follow_symlinks`` ++ can be a security risk, and may lead to a directory transversal attack. ++ You do NOT need this option to follow symlinks which point to somewhere ++ else within the static directory, this option is only used to break out ++ of the security sandbox. Enabling this option is highly discouraged, ++ and only expected to be used for edge cases in a local development ++ setting where remote users do not have access to the server. + + :param bool append_version: flag for adding file version (hash) + to the url query string, this value will +diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py +index f24f451e..6b8381c0 100644 +--- a/tests/test_web_urldispatcher.py ++++ b/tests/test_web_urldispatcher.py +@@ -123,6 +123,97 @@ async def test_follow_symlink(tmp_dir_path, aiohttp_client) -> None: + assert (await r.text()) == data + + ++async def test_follow_symlink_directory_traversal( ++ tmp_path: pathlib.Path, aiohttp_client ++) -> None: ++ # Tests that follow_symlinks does not allow directory transversal ++ data = "private" ++ ++ private_file = tmp_path / "private_file" ++ private_file.write_text(data) ++ ++ safe_path = tmp_path / "safe_dir" ++ safe_path.mkdir() ++ ++ app = web.Application() ++ ++ # Register global static route: ++ app.router.add_static("/", str(safe_path), follow_symlinks=True) ++ client = await aiohttp_client(app) ++ ++ await client.start_server() ++ # We need to use a raw socket to test this, as the client will normalize ++ # the path before sending it to the server. ++ reader, writer = await asyncio.open_connection(client.host, client.port) ++ writer.write(b"GET /../private_file HTTP/1.1\r\n\r\n") ++ response = await reader.readuntil(b"\r\n\r\n") ++ assert b"404 Not Found" in response ++ writer.close() ++ await writer.wait_closed() ++ await client.close() ++ ++ ++async def test_follow_symlink_directory_traversal_after_normalization( ++ tmp_path: pathlib.Path, aiohttp_client ++) -> None: ++ # Tests that follow_symlinks does not allow directory transversal ++ # after normalization ++ # ++ # Directory structure ++ # |-- secret_dir ++ # | |-- private_file (should never be accessible) ++ # | |-- symlink_target_dir ++ # | |-- symlink_target_file (should be accessible via the my_symlink symlink) ++ # | |-- sandbox_dir ++ # | |-- my_symlink -> symlink_target_dir ++ # ++ secret_path = tmp_path / "secret_dir" ++ secret_path.mkdir() ++ ++ # This file is below the symlink target and should not be reachable ++ private_file = secret_path / "private_file" ++ private_file.write_text("private") ++ ++ symlink_target_path = secret_path / "symlink_target_dir" ++ symlink_target_path.mkdir() ++ ++ sandbox_path = symlink_target_path / "sandbox_dir" ++ sandbox_path.mkdir() ++ ++ # This file should be reachable via the symlink ++ symlink_target_file = symlink_target_path / "symlink_target_file" ++ symlink_target_file.write_text("readable") ++ ++ my_symlink_path = sandbox_path / "my_symlink" ++ pathlib.Path(str(my_symlink_path)).symlink_to(str(symlink_target_path), True) ++ ++ app = web.Application() ++ ++ # Register global static route: ++ app.router.add_static("/", str(sandbox_path), follow_symlinks=True) ++ client = await aiohttp_client(app) ++ ++ await client.start_server() ++ # We need to use a raw socket to test this, as the client will normalize ++ # the path before sending it to the server. ++ reader, writer = await asyncio.open_connection(client.host, client.port) ++ writer.write(b"GET /my_symlink/../private_file HTTP/1.1\r\n\r\n") ++ response = await reader.readuntil(b"\r\n\r\n") ++ assert b"404 Not Found" in response ++ writer.close() ++ await writer.wait_closed() ++ ++ reader, writer = await asyncio.open_connection(client.host, client.port) ++ writer.write(b"GET /my_symlink/symlink_target_file HTTP/1.1\r\n\r\n") ++ response = await reader.readuntil(b"\r\n\r\n") ++ assert b"200 OK" in response ++ response = await reader.readuntil(b"readable") ++ assert response == b"readable" ++ writer.close() ++ await writer.wait_closed() ++ await client.close() ++ ++ + @pytest.mark.parametrize( + "dir_name,filename,data", + [ diff --git a/pkgs/development/python-modules/aiohttp/default.nix b/pkgs/development/python-modules/aiohttp/default.nix index ebbcf6ea82a4..75d2421c56b0 100644 --- a/pkgs/development/python-modules/aiohttp/default.nix +++ b/pkgs/development/python-modules/aiohttp/default.nix @@ -48,6 +48,7 @@ buildPythonPackage rec { url = "https://github.com/aio-libs/aiohttp/commit/7dcc235cafe0c4521bbbf92f76aecc82fee33e8b.patch"; hash = "sha256-ZzhlE50bmA+e2XX2RH1FuWQHZIAa6Dk/hZjxPoX5t4g="; }) + ./3.8.6-CVE-2024-23334.patch ]; postPatch = '' diff --git a/pkgs/os-specific/linux/apparmor/0001-aa-remove-unknown_empty-ruleset.patch b/pkgs/os-specific/linux/apparmor/0001-aa-remove-unknown_empty-ruleset.patch new file mode 100644 index 000000000000..ce2a0e7cc5b3 --- /dev/null +++ b/pkgs/os-specific/linux/apparmor/0001-aa-remove-unknown_empty-ruleset.patch @@ -0,0 +1,30 @@ +commit 166afaf144d6473464975438353257359dd51708 +Author: Andreas Wiese +Date: Thu Feb 1 11:35:02 2024 +0100 + + aa-remove-unknown: fix readability check + + This check is intended for ensuring that the profiles file can actually + be opened. The *actual* check is performed by the shell, not the read + utility, which won't even be executed if the input redirection (and + hence the test) fails. + + If the test succeeds, though, using `read` here might actually + jeopardize the test result if there are no profiles loaded and the file + is empty. + + This commit fixes that case by simply using `true` instead of `read`. + +diff --git a/utils/aa-remove-unknown b/utils/aa-remove-unknown +index 0e00d6a0..3351feef 100755 +--- a/utils/aa-remove-unknown ++++ b/utils/aa-remove-unknown +@@ -63,7 +63,7 @@ fi + # We have to do this check because error checking awk's getline() below is + # tricky and, as is, results in an infinite loop when apparmorfs returns an + # error from open(). +-if ! IFS= read -r _ < "$PROFILES" ; then ++if ! true < "$PROFILES" ; then + echo "ERROR: Unable to read apparmorfs profiles file" 1>&2 + exit 1 + elif [ ! -w "$REMOVE" ] ; then diff --git a/pkgs/os-specific/linux/apparmor/default.nix b/pkgs/os-specific/linux/apparmor/default.nix index ed1e31cc40eb..ba0bb4cd895f 100644 --- a/pkgs/os-specific/linux/apparmor/default.nix +++ b/pkgs/os-specific/linux/apparmor/default.nix @@ -56,7 +56,9 @@ let --replace "/usr/include/linux/capability.h" "${linuxHeaders}/include/linux/capability.h" ''; - patches = lib.optionals stdenv.hostPlatform.isMusl [ + patches = [ + ./0001-aa-remove-unknown_empty-ruleset.patch + ] ++ lib.optionals stdenv.hostPlatform.isMusl [ (fetchpatch { url = "https://git.alpinelinux.org/aports/plain/testing/apparmor/0003-Added-missing-typedef-definitions-on-parser.patch?id=74b8427cc21f04e32030d047ae92caa618105b53"; name = "0003-Added-missing-typedef-definitions-on-parser.patch"; diff --git a/pkgs/os-specific/linux/systemd/default.nix b/pkgs/os-specific/linux/systemd/default.nix index 0b5f57892592..9ef02fa397a5 100644 --- a/pkgs/os-specific/linux/systemd/default.nix +++ b/pkgs/os-specific/linux/systemd/default.nix @@ -159,7 +159,7 @@ assert !withPasswordQuality; let wantCurl = withRemote || withImportd; wantGcrypt = withResolved || withImportd; - version = "254.6"; + version = "254.10"; # Bump this variable on every (major) version change. See below (in the meson options list) for why. # command: @@ -176,7 +176,7 @@ stdenv.mkDerivation (finalAttrs: { owner = "systemd"; repo = "systemd-stable"; rev = "v${version}"; - hash = "sha256-Ku24ecDeQt0t7A8/adR3Jm47QZ19+wdMPyJRzCxU4uU="; + hash = "sha256-+8v1LLvXoK0tMOfB7m9+H+HIHpZuYrIyWwuz7rphbmA="; }; # On major changes, or when otherwise required, you *must* reformat the patches, diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh index b671d82b1597..87af0bfc93a3 100644 --- a/pkgs/stdenv/generic/setup.sh +++ b/pkgs/stdenv/generic/setup.sh @@ -815,15 +815,18 @@ fi ###################################################################### # Textual substitution functions. - substituteStream() { local var=$1 local description=$2 shift 2 while (( "$#" )); do + local replace_mode="$1" case "$1" in --replace) + replace_mode='--replace-warn' + ;& + --replace-quiet|--replace-warn|--replace-fail) pattern="$2" replacement="$3" shift 3 @@ -832,7 +835,12 @@ substituteStream() { eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}' if [ "$pattern" != "$replacement" ]; then if [ "${!var}" == "$savedvar" ]; then - echo "substituteStream(): WARNING: pattern '$pattern' doesn't match anything in $description" >&2 + if [ "$replace_mode" == --replace-warn ]; then + printf "substituteStream(): WARNING: pattern %q doesn't match anything in %s\n" "$pattern" "$description" >&2 + elif [ "$replace_mode" == --replace-fail ]; then + printf "substituteStream(): ERROR: pattern %q doesn't match anything in %s\n" "$pattern" "$description" >&2 + return 1 + fi fi fi ;; diff --git a/pkgs/tools/misc/coreutils/CVE-2024-0684.patch b/pkgs/tools/misc/coreutils/CVE-2024-0684.patch new file mode 100644 index 000000000000..64583afd3681 --- /dev/null +++ b/pkgs/tools/misc/coreutils/CVE-2024-0684.patch @@ -0,0 +1,31 @@ +From c4c5ed8f4e9cd55a12966d4f520e3a13101637d9 Mon Sep 17 00:00:00 2001 +From: Paul Eggert +Date: Tue, 16 Jan 2024 13:48:32 -0800 +Subject: [PATCH] split: do not shrink hold buffer +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +* src/split.c (line_bytes_split): Do not shrink hold buffer. +If it’s large for this batch it’s likely to be large for the next +batch, and for ‘split’ it’s not worth the complexity/CPU hassle to +shrink it. Do not assume hold_size can be bufsize. +--- + src/split.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/src/split.c b/src/split.c +index 64020c859..037960a59 100644 +--- a/src/split.c ++++ b/src/split.c +@@ -809,10 +809,7 @@ line_bytes_split (intmax_t n_bytes, char *buf, idx_t bufsize) + { + cwrite (n_out == 0, hold, n_hold); + n_out += n_hold; +- if (n_hold > bufsize) +- hold = xirealloc (hold, bufsize); + n_hold = 0; +- hold_size = bufsize; + } + + /* Output to eol if present. */ diff --git a/pkgs/tools/misc/coreutils/default.nix b/pkgs/tools/misc/coreutils/default.nix index beee4241c541..cc2a5f18ecb1 100644 --- a/pkgs/tools/misc/coreutils/default.nix +++ b/pkgs/tools/misc/coreutils/default.nix @@ -39,6 +39,11 @@ stdenv.mkDerivation rec { hash = "sha256-rbz8/omSNbceh2jc8HzVMlILf1T5qAZIQ/jRmakEu6o="; }; + patches = [ + # https://www.openwall.com/lists/oss-security/2024/01/18/2 + ./CVE-2024-0684.patch + ]; + postPatch = '' # The test tends to fail on btrfs, f2fs and maybe other unusual filesystems. sed '2i echo Skipping dd sparse test && exit 77' -i ./tests/dd/sparse.sh diff --git a/pkgs/tools/networking/curl/0003-CVE-2024-2398.patch b/pkgs/tools/networking/curl/0003-CVE-2024-2398.patch new file mode 100644 index 000000000000..c456691974f5 --- /dev/null +++ b/pkgs/tools/networking/curl/0003-CVE-2024-2398.patch @@ -0,0 +1,94 @@ +From b1b7ab02e055249c02eac8c2a80295701f2530a2 Mon Sep 17 00:00:00 2001 +From: Stefan Eissing +Date: Wed, 6 Mar 2024 09:36:08 +0100 +Subject: [PATCH 3/4] http2: push headers better cleanup + +- provide common cleanup method for push headers + +Closes #13054 + +(cherry picked from commit deca8039991886a559b67bcd6701db800a5cf764) +--- + lib/http2.c | 34 +++++++++++++++------------------- + 1 file changed, 15 insertions(+), 19 deletions(-) + +diff --git a/lib/http2.c b/lib/http2.c +index c8b059498..8c422d156 100644 +--- a/lib/http2.c ++++ b/lib/http2.c +@@ -271,6 +271,15 @@ static CURLcode http2_data_setup(struct Curl_cfilter *cf, + return CURLE_OK; + } + ++static void free_push_headers(struct stream_ctx *stream) ++{ ++ size_t i; ++ for(i = 0; ipush_headers_used; i++) ++ free(stream->push_headers[i]); ++ Curl_safefree(stream->push_headers); ++ stream->push_headers_used = 0; ++} ++ + static void http2_data_done(struct Curl_cfilter *cf, + struct Curl_easy *data, bool premature) + { +@@ -318,15 +327,7 @@ static void http2_data_done(struct Curl_cfilter *cf, + Curl_bufq_free(&stream->recvbuf); + Curl_h1_req_parse_free(&stream->h1); + Curl_dynhds_free(&stream->resp_trailers); +- if(stream->push_headers) { +- /* if they weren't used and then freed before */ +- for(; stream->push_headers_used > 0; --stream->push_headers_used) { +- free(stream->push_headers[stream->push_headers_used - 1]); +- } +- free(stream->push_headers); +- stream->push_headers = NULL; +- } +- ++ free_push_headers(stream); + free(stream); + H2_STREAM_LCTX(data) = NULL; + } +@@ -865,7 +866,6 @@ static int push_promise(struct Curl_cfilter *cf, + struct curl_pushheaders heads; + CURLMcode rc; + CURLcode result; +- size_t i; + /* clone the parent */ + struct Curl_easy *newhandle = h2_duphandle(cf, data); + if(!newhandle) { +@@ -910,11 +910,7 @@ static int push_promise(struct Curl_cfilter *cf, + Curl_set_in_callback(data, false); + + /* free the headers again */ +- for(i = 0; ipush_headers_used; i++) +- free(stream->push_headers[i]); +- free(stream->push_headers); +- stream->push_headers = NULL; +- stream->push_headers_used = 0; ++ free_push_headers(stream); + + if(rv) { + DEBUGASSERT((rv > CURL_PUSH_OK) && (rv <= CURL_PUSH_ERROROUT)); +@@ -1455,14 +1451,14 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, + if(stream->push_headers_alloc > 1000) { + /* this is beyond crazy many headers, bail out */ + failf(data_s, "Too many PUSH_PROMISE headers"); +- Curl_safefree(stream->push_headers); ++ free_push_headers(stream); + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + stream->push_headers_alloc *= 2; +- headp = Curl_saferealloc(stream->push_headers, +- stream->push_headers_alloc * sizeof(char *)); ++ headp = realloc(stream->push_headers, ++ stream->push_headers_alloc * sizeof(char *)); + if(!headp) { +- stream->push_headers = NULL; ++ free_push_headers(stream); + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + stream->push_headers = headp; +-- +2.44.0 + diff --git a/pkgs/tools/networking/curl/0004-CVE-2024-2004.patch b/pkgs/tools/networking/curl/0004-CVE-2024-2004.patch new file mode 100644 index 000000000000..e1acb7594bd8 --- /dev/null +++ b/pkgs/tools/networking/curl/0004-CVE-2024-2004.patch @@ -0,0 +1,137 @@ +From b091564d9fd187c6dc6410e3badf4621df8f4e0a Mon Sep 17 00:00:00 2001 +From: Daniel Gustafsson +Date: Tue, 27 Feb 2024 15:43:56 +0100 +Subject: [PATCH] setopt: Fix disabling all protocols + +When disabling all protocols without enabling any, the resulting +set of allowed protocols remained the default set. Clearing the +allowed set before inspecting the passed value from --proto make +the set empty even in the errorpath of no protocols enabled. + +Co-authored-by: Dan Fandrich +Reported-by: Dan Fandrich +Reviewed-by: Daniel Stenberg +Closes: #13004 +(cherry picked from commit 17d302e56221f5040092db77d4f85086e8a20e0e) +--- + lib/setopt.c | 16 ++++++++-------- + tests/data/Makefile.inc | 2 +- + tests/data/test1475 | 42 +++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 51 insertions(+), 9 deletions(-) + create mode 100644 tests/data/test1475 + +diff --git a/lib/setopt.c b/lib/setopt.c +index 0d399adfe..e022096f6 100644 +--- a/lib/setopt.c ++++ b/lib/setopt.c +@@ -154,6 +154,12 @@ static CURLcode setstropt_userpwd(char *option, char **userp, char **passwdp) + + static CURLcode protocol2num(const char *str, curl_prot_t *val) + { ++ /* ++ * We are asked to cherry-pick protocols, so play it safe and disallow all ++ * protocols to start with, and re-add the wanted ones back in. ++ */ ++ *val = 0; ++ + if(!str) + return CURLE_BAD_FUNCTION_ARGUMENT; + +@@ -162,8 +168,6 @@ static CURLcode protocol2num(const char *str, curl_prot_t *val) + return CURLE_OK; + } + +- *val = 0; +- + do { + const char *token = str; + size_t tlen; +@@ -2690,22 +2694,18 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) + break; + + case CURLOPT_PROTOCOLS_STR: { +- curl_prot_t prot; + argptr = va_arg(param, char *); +- result = protocol2num(argptr, &prot); ++ result = protocol2num(argptr, &data->set.allowed_protocols); + if(result) + return result; +- data->set.allowed_protocols = prot; + break; + } + + case CURLOPT_REDIR_PROTOCOLS_STR: { +- curl_prot_t prot; + argptr = va_arg(param, char *); +- result = protocol2num(argptr, &prot); ++ result = protocol2num(argptr, &data->set.redir_protocols); + if(result) + return result; +- data->set.redir_protocols = prot; + break; + } + +diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc +index 1472b1954..bdb2ef742 100644 +--- a/tests/data/Makefile.inc ++++ b/tests/data/Makefile.inc +@@ -185,7 +185,7 @@ test1439 test1440 test1441 test1442 test1443 test1444 test1445 test1446 \ + test1447 test1448 test1449 test1450 test1451 test1452 test1453 test1454 \ + test1455 test1456 test1457 test1458 test1459 test1460 test1461 test1462 \ + test1463 test1464 test1465 test1466 test1467 test1468 test1469 test1470 \ +-test1471 test1472 test1473 test1474 \ ++test1471 test1472 test1473 test1474 test1475 \ + \ + test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \ + test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \ +diff --git a/tests/data/test1475 b/tests/data/test1475 +new file mode 100644 +index 000000000..c66fa2810 +--- /dev/null ++++ b/tests/data/test1475 +@@ -0,0 +1,42 @@ ++ ++ ++ ++HTTP ++HTTP GET ++--proto ++ ++ ++ ++# ++# Server-side ++ ++ ++ ++ ++ ++# ++# Client-side ++ ++ ++none ++ ++ ++http ++ ++ ++--proto -all disables all protocols ++ ++ ++--proto -all http://%HOSTIP:%NOLISTENPORT/%TESTNUMBER ++ ++ ++ ++# ++# Verify data after the test has been "shot" ++ ++# 1 - Protocol "http" disabled ++ ++1 ++ ++ ++ +-- +2.44.0 + diff --git a/pkgs/tools/networking/curl/default.nix b/pkgs/tools/networking/curl/default.nix index c1d806c89b3f..330f86ae810e 100644 --- a/pkgs/tools/networking/curl/default.nix +++ b/pkgs/tools/networking/curl/default.nix @@ -65,6 +65,10 @@ stdenv.mkDerivation (finalAttrs: { ./0001-CVE-2023-42619.patch # https://curl.se/docs/CVE-2023-46218.html ./0002-CVE-2023-42618.patch + # https://curl.se/docs/CVE-2024-2398.html + ./0003-CVE-2024-2398.patch + # https://curl.se/docs/CVE-2024-2004.html + ./0004-CVE-2024-2004.patch ]; outputs = [ "bin" "dev" "out" "man" "devdoc" ]; diff --git a/pkgs/tools/networking/networkmanager/default.nix b/pkgs/tools/networking/networkmanager/default.nix index 5490977df12d..7b6ebb56f723 100644 --- a/pkgs/tools/networking/networkmanager/default.nix +++ b/pkgs/tools/networking/networkmanager/default.nix @@ -57,11 +57,11 @@ let in stdenv.mkDerivation rec { pname = "networkmanager"; - version = "1.44.2"; + version = "1.44.4"; src = fetchurl { url = "mirror://gnome/sources/NetworkManager/${lib.versions.majorMinor version}/NetworkManager-${version}.tar.xz"; - sha256 = "sha256-S1i/OsV+LO+1ZS79CUXrC0vDamPZKmGrRx2LssmkIOE="; + hash = "sha256-Aq+4X1qKPT1w0LahLIttst4uSWt3evW1cYLmC4CgNIk="; }; outputs = [ "out" "dev" "devdoc" "man" "doc" ]; diff --git a/pkgs/tools/networking/unbound/default.nix b/pkgs/tools/networking/unbound/default.nix index 2298ea0d7003..a921f5be66a8 100644 --- a/pkgs/tools/networking/unbound/default.nix +++ b/pkgs/tools/networking/unbound/default.nix @@ -49,11 +49,11 @@ stdenv.mkDerivation (finalAttrs: { pname = "unbound"; - version = "1.19.1"; + version = "1.19.2"; src = fetchurl { url = "https://nlnetlabs.nl/downloads/unbound/unbound-${finalAttrs.version}.tar.gz"; - hash = "sha256-vB1Xbz3YRqBzmtxB/6pwJATGdn0rYILeufL5fLsko6k="; + hash = "sha256-zFYNNFc0ImwbOecadpeX5/3eImXLt3685UJwS7pInlU="; }; outputs = [ "out" "lib" "man" ]; # "dev" would only split ~20 kB