gh-66788: Add the imap4-utf-7 codec

Implement the modified UTF-7 encoding used for international IMAP4
mailbox names (RFC 3501, section 5.1.3).  It differs from UTF-7:
"&" is the shift character ("&-" encodes a literal "&"), "," replaces
"/" in the Base64 alphabet, and all non-printable-ASCII characters are
Base64-encoded.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/codecs.rst

@@ -1384,6 +1384,14 @@ encodings.
 |                    |         | Only ``errors='strict'``  |
 |                    |         | is supported.             |
 +--------------------+---------+---------------------------+
+| imap4-utf-7        |         | Modified UTF-7 encoding   |
+|                    |         | of :rfc:`3501` for IMAP4  |
+|                    |         | mailbox names.  Only      |
+|                    |         | ``errors='strict'`` is    |
+|                    |         | supported.                |
+|                    |         |                           |
+|                    |         | .. versionadded:: next    |
++--------------------+---------+---------------------------+
 | mbcs               | ansi,   | Windows only: Encode the  |
 |                    | dbcs    | operand according to the  |
 |                    |         | ANSI codepage (CP_ACP).   |

Doc/whatsnew/3.16.rst

@@ -214,6 +214,14 @@ curses
   wide-character support.
   (Contributed by Serhiy Storchaka in :gh:`133031`.)
 
+encodings
+---------
+
+* Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding
+  used for international IMAP4 mailbox names (:rfc:`3501`).
+  (Contributed by Serhiy Storchaka in :gh:`66788`.)
+
+
 gzip
 ----
 

Lib/encodings/imap4_utf_7.py

@@ -0,0 +1,126 @@
+""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names,
+as specified in RFC 3501, section 5.1.3.
+
+It differs from UTF-7 (RFC 2152) as follows:
+
+* "&" (not "+") is the shift character introducing a Base64 sequence,
+  and "&-" encodes a literal "&";
+* "," is used instead of "/" in the modified Base64 alphabet;
+* only printable US-ASCII characters (except "&") may be represented
+  directly, so all other characters, including other controls, are
+  Base64-encoded.
+"""
+
+import binascii
+import codecs
+from base64 import b64encode, b64decode
+
+### Codec APIs
+
+def imap4_utf_7_encode(input, errors='strict'):
+    if errors != 'strict':
+        raise UnicodeError(f"Unsupported error handling: {errors}")
+    res = bytearray()
+    start = 0                   # start of the current run
+    b64run = False              # is input[start:i] a Base64 run?
+    def flush(end):
+        if start < end:
+            if b64run:
+                b64 = b64encode(input[start:end].encode('utf-16-be'),
+                                altchars=b'+,', padded=False)
+                res.extend(b'&' + b64 + b'-')
+            else:
+                res.extend(input[start:end].encode('ascii'))
+    for i, ch in enumerate(input):
+        if ch == '&':
+            flush(i)
+            res.extend(b'&-')
+            start = i + 1
+            b64run = False
+        elif ' ' <= ch <= '~':      # printable ASCII, represented directly
+            if b64run:
+                flush(i)
+                start = i
+                b64run = False
+        else:                        # everything else is Base64-encoded
+            if not b64run:
+                flush(i)
+                start = i
+                b64run = True
+    flush(len(input))
+    return res.take_bytes(), len(input)
+
+def imap4_utf_7_decode(input, errors='strict'):
+    if errors != 'strict':
+        raise UnicodeError(f"Unsupported error handling: {errors}")
+    input = bytes(input)
+    res = []
+    start = 0                   # start of the current direct ASCII run
+    i = 0
+    n = len(input)
+    def flush(end):
+        if start < end:
+            res.append(input[start:end].decode('ascii'))
+    while i < n:
+        c = input[i]
+        if c == b'&'[0]:
+            flush(i)
+            j = input.find(b'-', i + 1)
+            if j < 0:
+                raise UnicodeDecodeError('imap4-utf-7', input, i, n,
+                                         'unterminated shift sequence')
+            if j == i + 1:          # '&-'
+                res.append('&')
+            else:
+                b64 = input[i + 1:j]
+                try:
+                    data = b64decode(b64, altchars=b'+,',
+                                     validate=True, padded=False)
+                except binascii.Error:
+                    data = b''
+                if not data or len(data) % 2:
+                    raise UnicodeDecodeError('imap4-utf-7', input, i, j + 1,
+                                             'invalid shift sequence')
+                res.append(data.decode('utf-16-be'))
+            i = j + 1
+            start = i
+        elif b' '[0] <= c <= b'~'[0]:
+            i += 1
+        else:
+            raise UnicodeDecodeError('imap4-utf-7', input, i, i + 1,
+                                     'unexpected byte')
+    flush(n)
+    return ''.join(res), len(input)
+
+class Codec(codecs.Codec):
+    def encode(self, input, errors='strict'):
+        return imap4_utf_7_encode(input, errors)
+    def decode(self, input, errors='strict'):
+        return imap4_utf_7_decode(input, errors)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return imap4_utf_7_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    def decode(self, input, final=False):
+        return imap4_utf_7_decode(input, self.errors)[0]
+
+class StreamWriter(Codec, codecs.StreamWriter):
+    pass
+
+class StreamReader(Codec, codecs.StreamReader):
+    pass
+
+### encodings module API
+
+def getregentry():
+    return codecs.CodecInfo(
+        name='imap4-utf-7',
+        encode=Codec().encode,
+        decode=Codec().decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamwriter=StreamWriter,
+        streamreader=StreamReader,
+    )

Lib/test/test_codecs.py

@@ -1401,6 +1401,73 @@ def test_decode_invalid(self):
                     self.assertEqual(puny.decode("punycode", errors), expected)
 
 
+imap4_utf_7_testcases = [
+    # (unicode, modified UTF-7)
+    ('', b''),
+    ('INBOX', b'INBOX'),
+    # Printable US-ASCII represents itself.
+    ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'),
+    # "&" is escaped as "&-".
+    ('&', b'&-'),
+    ('&&', b'&-&-'),
+    ('A&B', b'A&-B'),
+    # RFC 3501 section 5.1.3 example.
+    ('~peter/mail/台北/日本語',
+     b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'),
+    # Non-printable ASCII (including TAB) is Base64-encoded, not direct.
+    ('a\tb', b'a&AAk-b'),
+    ('\x00', b'&AAA-'),
+    ('Entw\xfcrfe', b'Entw&APw-rfe'),
+    ('☃', b'&JgM-'),                     # snowman
+    ('\U0001f600', b'&2D3eAA-'),              # non-BMP (surrogate pair)
+    ('Sent &\N{DELETE}', b'Sent &-&AH8-'),
+]
+
+class Imap4Utf7Test(unittest.TestCase):
+    def test_encode(self):
+        for uni, encoded in imap4_utf_7_testcases:
+            with self.subTest(uni=uni):
+                self.assertEqual(uni.encode('imap4-utf-7'), encoded)
+
+    def test_decode(self):
+        for uni, encoded in imap4_utf_7_testcases:
+            with self.subTest(encoded=encoded):
+                self.assertEqual(encoded.decode('imap4-utf-7'), uni)
+
+    def test_decode_invalid(self):
+        # position of the first offending byte in each case
+        testcases = [
+            (b'&AAk', 0),           # unterminated shift sequence
+            (b'&Jgg', 0),           # unterminated shift sequence
+            (b'&AB-', 0),           # Base64 length not a multiple of a code unit
+            (b'&@@@-', 0),          # invalid Base64
+            (b'&====-', 0),         # invalid Base64
+            (b'a\x80b', 1),         # 8-bit byte outside a shift sequence
+            (b'a\x1fb', 1),         # control byte outside a shift sequence
+        ]
+        for encoded, start in testcases:
+            with self.subTest(encoded=encoded):
+                with self.assertRaises(UnicodeDecodeError) as cm:
+                    encoded.decode('imap4-utf-7')
+                self.assertEqual(cm.exception.encoding, 'imap4-utf-7')
+                self.assertEqual(cm.exception.start, start)
+
+    def test_encode_lone_surrogate(self):
+        with self.assertRaises(UnicodeEncodeError):
+            '\ud800'.encode('imap4-utf-7')
+
+    def test_only_strict_errors(self):
+        with self.assertRaises(UnicodeError):
+            'x'.encode('imap4-utf-7', 'replace')
+        with self.assertRaises(UnicodeError):
+            b'x'.decode('imap4-utf-7', 'ignore')
+
+    def test_stateless(self):
+        # The codec is registered and exposes the standard interface.
+        info = codecs.lookup('imap4-utf-7')
+        self.assertEqual(info.name, 'imap4-utf-7')
+
+
 # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
 nameprep_tests = [
     # 3.1 Map to nothing.

Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst

@@ -0,0 +1,2 @@
+Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding
+used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3).