Adding one more data correctness patch

Author: stamparm

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.252"
+VERSION = "1.10.7.253"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/inject.py

@@ -594,6 +594,34 @@ def _(item):
     applyFunctionRecursively(value, _)
     return retVal[0]
 
+def _looksLikeMisdecodedUtf8(value):
+    """
+    True if a retrieved value looks like UTF-8 data shown through a single-byte (ISO-8859-1) charset -
+    the '<page charset too WIDE>' mismatch that leaves NO undecodable bytes (latin-1 accepts every byte),
+    so _pageCharsetCorrupted() cannot see it, yet the data is silently mojibake (a UTF-8 accent shows as
+    two spurious latin-1 chars).
+
+    Precise by construction: real mojibake re-encodes to latin-1 and decodes back to a DIFFERENT valid
+    UTF-8 string, whereas correctly-decoded UTF-8 ('cafe' with U+00E9) and genuine latin-1 text ('resume')
+    do not (a lone accent is a UTF-8 lead byte with no continuation -> decode fails). So it fires only on
+    the corruption, not on clean data. Complements _pageCharsetCorrupted() (the too-NARROW direction).
+    """
+
+    retVal = [False]
+
+    def _(item):
+        if not retVal[0] and isinstance(item, six.string_types) and any(0x80 <= ord(_) <= 0xFF for _ in item):
+            try:
+                decoded = item.encode("latin-1").decode("utf-8")
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                return item
+            if decoded != item and any(ord(_) > 0x7F for _ in decoded):
+                retVal[0] = True
+        return item
+
+    applyFunctionRecursively(value, _)
+    return retVal[0]
+
 @lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
@@ -699,7 +727,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                 if (found and not conf.hexConvert and not conf.binaryFields and expected not in (EXPECTED.BOOL, EXPECTED.INT)
                         and getTechnique() in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)
                         and Backend.getIdentifiedDbms() and hasattr(queries[Backend.getIdentifiedDbms()], "hex")
-                        and _pageCharsetCorrupted(value)):
+                        and (_pageCharsetCorrupted(value) or _looksLikeMisdecodedUtf8(value))):
                     warnMsg = "retrieved data appears corrupted because of a charset mismatch between the "
                     warnMsg += "DBMS and the web page. Re-fetching using hexadecimal encoding"
                     singleTimeWarnMessage(warnMsg)
@@ -710,7 +738,10 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                     finally:
                         conf.hexConvert = False
 
-                    if _value is not None:
+                    # only adopt the hex re-fetch if it is actually cleaner: a rare mis-trigger (or a
+                    # column whose real charset the hex decode still cannot resolve) must never replace
+                    # the original with equal-or-worse data
+                    if _value is not None and not _pageCharsetCorrupted(_value) and not _looksLikeMisdecodedUtf8(_value):
                         value = _value
 
                 if found and conf.dnsDomain:

tests/test_charset.py

@@ -22,6 +22,7 @@
 bootstrap()
 
 from lib.request.basic import checkCharEncoding
+from lib.request.inject import _pageCharsetCorrupted, _looksLikeMisdecodedUtf8
 from lib.core.common import extractRegexResult, paramToDict
 from lib.core.enums import PLACE
 from lib.core.settings import META_CHARSET_REGEX, HTML_TITLE_REGEX, META_REFRESH_REGEX
@@ -58,6 +59,38 @@ def test_no_match_is_none(self):
         self.assertIsNone(extractRegexResult(HTML_TITLE_REGEX, "<body>no title here</body>"))
 
 
+class TestCharsetMismatchDetection(unittest.TestCase):
+    """The auto-hex recovery triggers on two complementary mismatch directions:
+    _pageCharsetCorrupted (page charset too NARROW -> undecodable bytes) and
+    _looksLikeMisdecodedUtf8 (page charset too WIDE -> UTF-8 shown as latin-1, no
+    undecodable byte). Both must fire on the corruption and NOT on clean data."""
+
+    # correctly-decoded UTF-8 and genuine latin-1 text: neither detector may fire (no wasted re-fetch).
+    # \u escapes keep this source pure-ASCII (py2): cafe / naive / Zurich / CJK / Cyrillic / resume / garcon
+    CLEAN = [u"admin", u"caf\u00e9", u"na\u00efve", u"Z\u00fcrich", u"\u65e5\u672c\u8a9e",
+             u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"r\u00e9sum\u00e9", u"gar\u00e7on"]
+
+    def test_clean_values_trigger_neither(self):
+        for v in self.CLEAN:
+            self.assertFalse(_pageCharsetCorrupted(v), msg="narrow FP: %r" % v)
+            self.assertFalse(_looksLikeMisdecodedUtf8(v), msg="wide FP: %r" % v)
+
+    def test_utf8_shown_as_latin1_is_caught(self):
+        # gap #1: UTF-8 column bytes decoded as latin-1 -> valid mojibake, no undecodable byte
+        for word in (u"caf\u00e9", u"\u65e5\u672c\u8a9e", u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"\u20ac"):
+            mojibake = word.encode("utf-8").decode("latin-1")
+            self.assertFalse(_pageCharsetCorrupted(mojibake), msg="narrow should miss: %r" % mojibake)
+            self.assertTrue(_looksLikeMisdecodedUtf8(mojibake), msg="wide should catch: %r" % mojibake)
+
+    def test_undecodable_bytes_still_caught_by_narrow(self):
+        # the other direction (page charset too narrow) leaves reversible \xNN escapes
+        self.assertTrue(_pageCharsetCorrupted(u"foo\\xe9bar"))
+
+    def test_list_and_nonstring_inputs(self):
+        self.assertTrue(_looksLikeMisdecodedUtf8([u"ok", u"caf\u00c3\u00a9"]))   # 'cafe' mojibake
+        self.assertFalse(_looksLikeMisdecodedUtf8([u"ok", 123, None]))
+
+
 class TestParamToDict(unittest.TestCase):
     # NOTE: GET parsing is covered in test_urls.py; here we only cover the COOKIE place,
     # which uses a different (semicolon) delimiter and is a distinct code path.