Exact equivalent of `b'...'.decode("utf-8", "backslashreplace")` in Python 2
I attempted a more complete backport of the cpython implementation
This handles both UnicodeDecodeError
(from .decode()
) as well as UnicodeEncodeError
from .encode()
and UnicodeTranslateError
from .translate()
:
from __future__ import unicode_literals
import codecs
def _bytes_repr(c):
"""py2: bytes, py3: int"""
if not isinstance(c, int):
c = ord(c)
return '\\x{:x}'.format(c)
def _text_repr(c):
d = ord(c)
if d >= 0x10000:
return '\\U{:08x}'.format(d)
else:
return '\\u{:04x}'.format(d)
def backslashescape_backport(ex):
s, start, end = ex.object, ex.start, ex.end
c_repr = _bytes_repr if isinstance(ex, UnicodeDecodeError) else _text_repr
return ''.join(c_repr(c) for c in s[start:end]), end
codecs.register_error('backslashescape_backport', backslashescape_backport)
print(b'\xc2\xa1\xa1after'.decode('utf-8', 'backslashescape_backport'))
print(u'\u2603'.encode('latin1', 'backslashescape_backport'))
You can write your own error handler. Here's a solution that I tested on Python 2.7, 3.3 and 3.6:
from __future__ import print_function
import codecs
import sys
print(sys.version)
def myreplace(ex):
# The error handler receives the UnicodeDecodeError, which contains arguments of the
# string and start/end indexes of the bad portion.
bstr,start,end = ex.object,ex.start,ex.end
# The return value is a tuple of Unicode string and the index to continue conversion.
# Note: iterating byte strings returns int on 3.x but str on 2.x
return u''.join('\\x{:02x}'.format(c if isinstance(c,int) else ord(c))
for c in bstr[start:end]),end
codecs.register_error('myreplace',myreplace)
print(b'\xc2\xa1\xa1ABC'.decode("utf-8", "myreplace"))
Output:
C:\>py -2.7 test.py 2.7.13 (v2.7.13:a06454b1afa1, Dec 17 2016, 20:42:59) [MSC v.1500 32 bit (Intel)] ¡\xa1ABC C:\>py -3.3 test.py 3.3.5 (v3.3.5:62cf4e77f785, Mar 9 2014, 10:35:05) [MSC v.1600 64 bit (AMD64)] ¡\xa1ABC C:\>py -3.6 test.py 3.6.1 (v3.6.1:69c0db5, Mar 21 2017, 18:41:36) [MSC v.1900 64 bit (AMD64)] ¡\xa1ABC