diff --git a/data/xml/queries.xml b/data/xml/queries.xml
index f53cf0e1086..dfaabd38c81 100644
--- a/data/xml/queries.xml
+++ b/data/xml/queries.xml
@@ -83,7 +83,7 @@
-
+
@@ -119,8 +119,8 @@
-
-
+
+
@@ -163,7 +163,7 @@
-
+
@@ -222,7 +222,7 @@
-
+
@@ -384,7 +384,7 @@
-
+
@@ -959,8 +959,8 @@
-
-
+
+
@@ -1158,7 +1158,7 @@
-
+
/>
diff --git a/lib/core/agent.py b/lib/core/agent.py
index 5db8f450a66..520fc99146d 100644
--- a/lib/core/agent.py
+++ b/lib/core/agent.py
@@ -508,10 +508,26 @@ def nullAndCastField(self, field):
if field and Backend.getIdentifiedDbms():
rootQuery = queries[Backend.getIdentifiedDbms()]
+ kb.binaryField = conf.binaryFields and field in conf.binaryFields
+ hexConvert = conf.hexConvert or kb.binaryField
+
+ # For BINARY fields, wrap the RAW column with the hex function rather than the text-casted one:
+ # text-casting binary first (e.g. CAST( AS NCHAR) on MySQL) NULLs non-text bytes, so HEX()
+ # would encode the NULL placeholder and silently lose the value (e.g. binary-stored password
+ # hashes). This is limited to binary fields: the blanket '--hex' keeps the cast-first path because
+ # raw-hexing a numeric/temporal column uses the DBMS's numeric HEX semantics (MySQL HEX(255)='FF'),
+ # which is NOT the hex of its string representation. Every DBMS hex function takes binary directly
+ # EXCEPT PostgreSQL's CONVERT_TO() (needs text), so PostgreSQL stays on the cast-first path too.
+ hexRaw = kb.binaryField and not Backend.isDbms(DBMS.PGSQL)
+
if field.startswith("(CASE") or field.startswith("(IIF") or conf.noCast and not (field.startswith("COUNT(") and Backend.getIdentifiedDbms() == DBMS.MSSQL):
nulledCastedField = field
+ if hexConvert:
+ nulledCastedField = self.hexConvertField(nulledCastedField)
else:
- if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
+ if hexRaw:
+ nulledCastedField = self.hexConvertField(field)
+ elif not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
nulledCastedField = rootQuery.cast.query % field
if re.search(r"COUNT\(", field) and Backend.getIdentifiedDbms() in (DBMS.RAIMA,):
@@ -521,9 +537,8 @@ def nullAndCastField(self, field):
else:
nulledCastedField = rootQuery.isnull.query % nulledCastedField
- kb.binaryField = conf.binaryFields and field in conf.binaryFields
- if conf.hexConvert or kb.binaryField:
- nulledCastedField = self.hexConvertField(nulledCastedField)
+ if hexConvert and not hexRaw:
+ nulledCastedField = self.hexConvertField(nulledCastedField)
if suffix:
nulledCastedField += suffix
diff --git a/lib/core/settings.py b/lib/core/settings.py
index c0bb332ee58..e396b3782f3 100644
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -20,7 +20,7 @@
from thirdparty import six
# sqlmap version (...)
-VERSION = "1.10.7.73"
+VERSION = "1.10.7.77"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -879,15 +879,17 @@
# Warn user of possible delay due to large page dump in full UNION query injections
LARGE_OUTPUT_THRESHOLD = 1024 ** 2
-# On huge tables there is a considerable slowdown if every row retrieval requires ORDER BY (most noticable in table dumping using ERROR injections)
-SLOW_ORDER_COUNT_THRESHOLD = 10000
-
# Give up on hash recognition if nothing was found in first given number of rows
HASH_RECOGNITION_QUIT_THRESHOLD = 1000
# Regular expression used for automatic hex conversion and hash cracking of (RAW) binary column values
HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash"
+# Regular expression matching (declared) binary column types, used to auto-hex their values during dumping
+# so raw bytes (e.g. password hashes stored in binary form) are not silently truncated at NUL / mangled by
+# the text extraction channel (mirrors a manual '--binary-fields', using the already-fetched column type)
+BINARY_FIELDS_TYPE_REGEX = r"(?i)binary|blob|bytea|image|\braw\b"
+
# Maximum number of redirections to any single URL - this is needed because of the state that cookies introduce
MAX_SINGLE_URL_REDIRECTIONS = 4
diff --git a/lib/core/target.py b/lib/core/target.py
index 0aff961612c..dd1fd34ae35 100644
--- a/lib/core/target.py
+++ b/lib/core/target.py
@@ -214,6 +214,22 @@ def process(match, repl):
conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER)
conf.data = re.sub(r"(<(?P[^>]+)( [^<]*)?>)([^<]+)(\g<4>%s\g<5>" % kb.customInjectionMark), conf.data)
+ # Also expose XML attribute values (e.g. - ) as injection points, not just
+ # element text (Reference: https://github.com/sqlmapproject/sqlmap/issues/5993). Done per
+ # opening tag (skipping declarations, and closing tags) so every attribute
+ # is covered while the XML declaration/namespaces are left intact.
+ def processAttributes(match):
+ def _attr(m):
+ name = m.group("name")
+ if name == "xmlns" or name.startswith("xmlns:"): # namespace declarations are not injection points
+ return m.group(0)
+ if conf.testParameter and name not in (removePostHintPrefix(_) for _ in conf.testParameter):
+ return m.group(0)
+ hintNames.append(("%s%s%s" % (m.group(1), m.group(3), m.group(4)), name))
+ return "%s%s%s%s%s" % (m.group(1), m.group(3), m.group(4), kb.customInjectionMark, m.group(5))
+ return re.sub(r'(\s(?P[\w:.-]+)\s*=\s*)(["\'])([^"\'<>]*)(["\'])', _attr, match.group(0))
+ conf.data = re.sub(r"(?s)<[^>?!/][^>]*>", processAttributes, conf.data)
+
elif re.search(MULTIPART_RECOGNITION_REGEX, conf.data):
message = "Multipart-like data found in %s body. " % conf.method
message += "Do you want to process it? [Y/n/q] "
diff --git a/lib/techniques/error/use.py b/lib/techniques/error/use.py
index 4b5a645c51e..17511a0f4e7 100644
--- a/lib/techniques/error/use.py
+++ b/lib/techniques/error/use.py
@@ -29,7 +29,6 @@
from lib.core.common import isListLike
from lib.core.common import isNumPosStrValue
from lib.core.common import listToStrValue
-from lib.core.common import readInput
from lib.core.common import unArrayizeValue
from lib.core.common import wasLastResponseHTTPError
from lib.core.compat import xrange
@@ -51,7 +50,6 @@
from lib.core.settings import NULL
from lib.core.settings import PARTIAL_VALUE_MARKER
from lib.core.settings import ROTATING_CHARS
-from lib.core.settings import SLOW_ORDER_COUNT_THRESHOLD
from lib.core.settings import SQL_SCALAR_REGEX
from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
from lib.core.threads import getCurrentThreadData
@@ -322,7 +320,7 @@ def errorUse(expression, dump=False):
# entry at a time
# NOTE: we assume that only queries that get data from a table can
# return multiple entries
- if (dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+ if not re.search(SQL_SCALAR_REGEX, expression, re.I) and ((dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression)))):
expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
if limitCond:
@@ -367,13 +365,10 @@ def errorUse(expression, dump=False):
return value
if isNumPosStrValue(count) and int(count) > 1:
- if " ORDER BY " in expression and (stopLimit - startLimit) > SLOW_ORDER_COUNT_THRESHOLD:
- message = "due to huge table size do you want to remove "
- message += "ORDER BY clause gaining speed over consistency? [y/N] "
-
- if readInput(message, default='N', boolean=True):
- expression = expression[:expression.index(" ORDER BY ")]
-
+ # NOTE: the ORDER BY clause is deliberately NOT stripped here. This path fetches each column
+ # of a row in a separate LIMIT/OFFSET query, so dropping ORDER BY would let the per-column
+ # offsets resolve to different physical rows and silently misalign cells across the row. Huge
+ # tables are handled cheaply and safely by keyset (seek) pagination (see plugins/generic/entries.py).
numThreads = min(conf.threads, (stopLimit - startLimit))
threadData = getCurrentThreadData()
diff --git a/lib/techniques/union/test.py b/lib/techniques/union/test.py
index 5c2022c3a7f..fd0b6bd8376 100644
--- a/lib/techniques/union/test.py
+++ b/lib/techniques/union/test.py
@@ -61,14 +61,37 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
@stackedmethod
def _orderByTechnique(lowerCount=None, upperCount=None):
- def _orderByTest(cols):
+ def _orderByProbe(cols):
query = agent.prefixQuery("ORDER BY %d" % cols, prefix=prefix)
query = agent.suffixQuery(query, suffix=suffix, comment=comment)
payload = agent.payload(newValue=query, place=place, parameter=parameter, where=where)
page, headers, code = Request.queryPage(payload, place=place, content=True, raise404=False)
- return not any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order (by|clause)", "unknown column", "failed")) and not kb.heavilyDynamic and comparison(page, headers, code) or re.search(r"data types cannot be compared or sorted", page or "", re.I) is not None
+ errored = any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order (by|clause)", "unknown column", "failed"))
+ return page, headers, code, errored
+
+ lowCount = 1 if lowerCount is None else lowerCount
+ highCount = randomInt() if upperCount is None else upperCount + 1
+
+ lowPage, lowHeaders, lowCode, lowErrored = _orderByProbe(lowCount)
+ highPage, highHeaders, highCode, highErrored = _orderByProbe(highCount)
+
+ # When an out-of-range ORDER BY position visibly errors on this target, the column count can be
+ # bisected on error presence alone - immune to result-set shifts caused by the injection context
+ # (e.g. a LIKE '%...%' search whose matches change once the string literal is closed), which would
+ # otherwise make the response-similarity oracle reject a still-valid position.
+ errorOracle = highErrored and not lowErrored
+
+ def _orderByValid(page, headers, code, errored):
+ if re.search(r"data types cannot be compared or sorted", page or "", re.I):
+ return True
+ if errored:
+ return False
+ return errorOracle or (not kb.heavilyDynamic and comparison(page, headers, code))
+
+ def _orderByTest(cols):
+ return _orderByValid(*_orderByProbe(cols))
- if _orderByTest(1 if lowerCount is None else lowerCount) and not _orderByTest(randomInt() if upperCount is None else upperCount + 1):
+ if _orderByValid(lowPage, lowHeaders, lowCode, lowErrored) and not _orderByValid(highPage, highHeaders, highCode, highErrored):
infoMsg = "'ORDER BY' technique appears to be usable. "
infoMsg += "This should reduce the time needed "
infoMsg += "to find the right number "
diff --git a/lib/utils/keysetdump.py b/lib/utils/keysetdump.py
index 2b38f2c5719..eaed7b2f9b0 100644
--- a/lib/utils/keysetdump.py
+++ b/lib/utils/keysetdump.py
@@ -189,12 +189,12 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
produced = 0
while produced < target:
- if pivotValue is None:
- query = blind.keyset_first % (field, tableRef)
- else:
- query = _embed(blind.keyset_next, pivotValue, field, tableRef, field)
-
- query = agent.whereQuery(query)
+ # Advance with ORDER BY ... LIMIT 1 (like the composite path), NOT MIN(): the value-extraction
+ # casts the aggregated column to VARCHAR *inside* MIN(), yielding a LEXICAL minimum ('10' after
+ # '1') that disagrees with the numeric '>' comparison and silently skips rows (2..9, 11..). The
+ # ORDER BY is on the raw (numeric) column, so the next cursor value is the true successor.
+ condition = "1=1" if pivotValue is None else "%s>%s" % (field, _lit(pivotValue))
+ query = agent.whereQuery(blind.keyset_ordered % (field, tableRef, condition, field))
value = unArrayizeValue(inject.getValue(query))
if isNoneValue(value) or value == NULL:
diff --git a/plugins/generic/entries.py b/plugins/generic/entries.py
index 4e8ef5a7d96..ca4a3d54a19 100644
--- a/plugins/generic/entries.py
+++ b/plugins/generic/entries.py
@@ -40,6 +40,7 @@
from lib.core.exception import SqlmapMissingMandatoryOptionException
from lib.core.exception import SqlmapNoneDataException
from lib.core.exception import SqlmapUnsupportedFeatureException
+from lib.core.settings import BINARY_FIELDS_TYPE_REGEX
from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD
from lib.core.settings import CURRENT_DB
from lib.core.settings import KEYSET_MIN_ROWS
@@ -114,6 +115,8 @@ def dumpTable(self, foundData=None):
for tbl in tblList:
tblList[tblList.index(tbl)] = safeSQLIdentificatorNaming(tbl, True)
+ binaryFields = conf.binaryFields # user-provided '--binary-fields' (auto-detected ones are added per-table)
+
for tbl in tblList:
if kb.dumpKeyboardInterrupt:
break
@@ -169,6 +172,18 @@ def dumpTable(self, foundData=None):
colNames = colString = ','.join(column for column in colList)
rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
+ # Auto-treat binary-typed columns (blob/varbinary/bytea/image/raw) as '--binary-fields', so their
+ # raw bytes (e.g. password hashes stored in binary form) are hex-extracted instead of being
+ # silently truncated at a NUL / mangled by the text channel (issues #8, #582, #2827). The column
+ # type is already known from the enumeration above, so this costs no extra request.
+ # (PostgreSQL excluded: its bytea already renders as readable '\xHEX' through the default text
+ # cast, and its hex path needs text input, so auto-hexing would double-encode.)
+ autoBinary = [] if Backend.isDbms(DBMS.PGSQL) else [column for column in colList if columns.get(column) and re.search(BINARY_FIELDS_TYPE_REGEX, getUnicode(columns[column]))]
+ conf.binaryFields = (list(binaryFields) if binaryFields else []) + [_ for _ in autoBinary if not (binaryFields and _ in binaryFields)]
+ if autoBinary:
+ debugMsg = "auto-treating binary column(s) '%s' as binary fields" % ', '.join(unsafeSQLIdentificatorNaming(_) for _ in autoBinary)
+ logger.debug(debugMsg)
+
infoMsg = "fetching entries"
if conf.col:
infoMsg += " of column(s) '%s'" % colNames
@@ -184,7 +199,49 @@ def dumpTable(self, foundData=None):
entriesCount = 0
- if any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
+ def _dumpCountQuery():
+ if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE, DBMS.MIMERSQL, DBMS.SNOWFLAKE):
+ _ = rootQuery.blind.count % (tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())))
+ elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.MAXDB, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA):
+ _ = rootQuery.blind.count % tbl
+ elif Backend.getIdentifiedDbms() in (DBMS.SYBASE, DBMS.MSSQL):
+ _ = rootQuery.blind.count % ("%s.%s" % (conf.db, tbl)) if conf.db else tbl
+ elif Backend.isDbms(DBMS.INFORMIX):
+ _ = rootQuery.blind.count % (conf.db, tbl)
+ else:
+ _ = rootQuery.blind.count % (conf.db, tbl)
+ return agent.whereQuery(_)
+
+ # Keyset (seek) pagination for the error/query (inband) path too, not just the blind path below.
+ # It fetches every cell value-anchored (MAX(col) WHERE key=K_r on a unique integer key), so the
+ # per-cell retrievals of one row are pinned to the SAME physical row regardless of scan order,
+ # threads or MVCC - unlike ORDER BY ... LIMIT/OFFSET, which silently misaligns cells once a
+ # stable order is unavailable. UNION is intrinsically aligned (whole-row), so it is never preempted.
+ keysetDone = False
+ if not conf.direct and not conf.noKeyset and not isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION) \
+ and any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)):
+ count = inject.getValue(_dumpCountQuery(), expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS)
+ if isNumPosStrValue(count) and (conf.keyset or int(count) >= KEYSET_MIN_ROWS):
+ keysetCursor = resolveKeysetCursor(tbl, colList)
+ if keysetCursor:
+ infoMsg = "using keyset (seek) pagination on column(s) '%s' " % ', '.join(keysetCursor)
+ infoMsg += "for table '%s'" % unsafeSQLIdentificatorNaming(tbl)
+ logger.info(infoMsg)
+
+ try:
+ entries, lengths = keysetDumpTable(tbl, colList, int(count), keysetCursor)
+ for column, columnEntries in entries.items():
+ length = max(lengths[column], getConsoleLength(column))
+ kb.data.dumpedTable[column] = {"length": length, "values": columnEntries}
+ entriesCount = len(columnEntries)
+ keysetDone = bool(kb.data.dumpedTable)
+ except KeyboardInterrupt:
+ kb.dumpKeyboardInterrupt = True
+ clearConsoleLine()
+ warnMsg = "Ctrl+C detected in dumping phase"
+ logger.warning(warnMsg)
+
+ if not keysetDone and (any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct):
entries = []
query = None
@@ -213,7 +270,7 @@ def dumpTable(self, foundData=None):
for index in indexRange:
row = []
for column in colList:
- query = rootQuery.blind.query3 % (column, column, table, index)
+ query = rootQuery.blind.query3 % (column, column, prioritySortColumns(colList)[0], table, index)
query = agent.whereQuery(query)
value = inject.getValue(query, blind=False, time=False, dump=True) or ""
row.append(value)
@@ -369,7 +426,7 @@ def dumpTable(self, foundData=None):
for index in indexRange:
for column in colList:
- query = rootQuery.blind.query3 % (column, column, table, index)
+ query = rootQuery.blind.query3 % (column, column, prioritySortColumns(colList)[0], table, index)
query = agent.whereQuery(query)
value = inject.getValue(query, union=False, error=False, dump=True) or ""
@@ -530,6 +587,7 @@ def cellQuery(column, index):
finally:
kb.dumpColumns = None
kb.dumpTable = None
+ conf.binaryFields = binaryFields # restore user-provided value (drop this table's auto-detected ones)
def dumpAll(self):
if conf.db is not None and conf.tbl is None:
diff --git a/tests/test_dialect.py b/tests/test_dialect.py
index 5e82127633c..871f1622996 100644
--- a/tests/test_dialect.py
+++ b/tests/test_dialect.py
@@ -68,9 +68,10 @@ class TestNullAndCastField(unittest.TestCase):
CASES = {
DBMS.MYSQL: "IFNULL(CAST(col AS NCHAR),' ')",
- DBMS.MSSQL: "ISNULL(CAST(col AS NVARCHAR(4000)),' ')",
+ # MSSQL/PGSQL casts are unbounded (NVARCHAR(MAX)/TEXT) so long values are not silently truncated
+ DBMS.MSSQL: "ISNULL(CAST(col AS NVARCHAR(MAX)),' ')",
DBMS.SYBASE: "ISNULL(CONVERT(VARCHAR(4000),col),' ')",
- DBMS.PGSQL: "COALESCE(CAST(col AS VARCHAR(10000))::text,' ')",
+ DBMS.PGSQL: "COALESCE(CAST(col AS TEXT)::text,' ')",
DBMS.ORACLE: "NVL(CAST(col AS VARCHAR(4000)),' ')",
}
diff --git a/tests/test_payload_marking.py b/tests/test_payload_marking.py
index f0271bf9c53..5b014e82999 100644
--- a/tests/test_payload_marking.py
+++ b/tests/test_payload_marking.py
@@ -175,10 +175,17 @@ def mark(self, data):
CASES = [
("x", "x*"),
- ('x', 'x*'),
+ # attribute values are now marked too, not just element text (issue #5993)
+ ('x', 'x*'),
("bob5", "bob*5*"),
("v", "v*"),
("
1
", "1*
"),
+ # multiple attributes and single-quoted attribute values
+ ('y', 'y*'),
+ ("y", "y*"),
+ # XML declaration and namespace declarations are left intact (not injection points)
+ ('y', 'y*'),
+ ('y', 'y*'),
]
def test_cases(self):