[TupleVariation] Fix 32bit reading / writing

This commit is contained in:
Behdad Esfahbod 2023-12-15 21:46:39 -07:00
parent 8cce745d90
commit 68277fc0b5

View File

@ -447,6 +447,10 @@ class TupleVariation(object):
and (-128 <= deltas[pos + 1] <= 127) and (-128 <= deltas[pos + 1] <= 127)
): ):
break break
if not (-32768 <= value <= 32767):
break
pos += 1 pos += 1
runLength = pos - offset runLength = pos - offset
while runLength >= 64: while runLength >= 64:
@ -471,11 +475,13 @@ class TupleVariation(object):
numDeltas = len(deltas) numDeltas = len(deltas)
while pos < numDeltas: while pos < numDeltas:
value = deltas[pos] value = deltas[pos]
if -32768 <= value <= 32767:
break
pos += 1 pos += 1
runLength = pos - offset runLength = pos - offset
while runLength >= 64: while runLength >= 64:
bytearr.append(DELTAS_ARE_LONGS | 63) bytearr.append(DELTAS_ARE_LONGS | 63)
a = array.array("l", deltas[offset : offset + 64]) a = array.array("i", deltas[offset : offset + 64])
if sys.byteorder != "big": if sys.byteorder != "big":
a.byteswap() a.byteswap()
bytearr.extend(a) bytearr.extend(a)
@ -483,7 +489,7 @@ class TupleVariation(object):
runLength -= 64 runLength -= 64
if runLength: if runLength:
bytearr.append(DELTAS_ARE_LONGS | (runLength - 1)) bytearr.append(DELTAS_ARE_LONGS | (runLength - 1))
a = array.array("l", deltas[offset:pos]) a = array.array("i", deltas[offset:pos])
if sys.byteorder != "big": if sys.byteorder != "big":
a.byteswap() a.byteswap()
bytearr.extend(a) bytearr.extend(a)
@ -502,9 +508,9 @@ class TupleVariation(object):
result.extend([0] * numDeltasInRun) result.extend([0] * numDeltasInRun)
else: else:
if (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_LONGS: if (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_LONGS:
deltas = array.array("l") deltas = array.array("i")
deltasSize = numDeltasInRun * 4 deltasSize = numDeltasInRun * 4
if (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_WORDS: elif (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_WORDS:
deltas = array.array("h") deltas = array.array("h")
deltasSize = numDeltasInRun * 2 deltasSize = numDeltasInRun * 2
else: else: