Back to index

python3.2  3.2.2
Public Member Functions | Static Public Attributes
test.test_codecs.UTF32BETest Class Reference
Inheritance diagram for test.test_codecs.UTF32BETest:
Inheritance graph
[legend]
Collaboration diagram for test.test_codecs.UTF32BETest:
Collaboration graph
[legend]

List of all members.

Public Member Functions

def test_partial
def test_simple
def test_errors
def test_issue8941
def check_partial
def test_readline
def test_bug1175396
def test_readlinequeue
def test_bug1098990_a
def test_bug1098990_b
def check_state_handling_decode
def check_state_handling_encode

Static Public Attributes

string encoding = "utf-32-be"

Detailed Description

Definition at line 407 of file test_codecs.py.


Member Function Documentation

def test.test_codecs.ReadTest.check_partial (   self,
  input,
  partialresults 
) [inherited]

Definition at line 61 of file test_codecs.py.

00061 
00062     def check_partial(self, input, partialresults):
00063         # get a StreamReader for the encoding and feed the bytestring version
00064         # of input to the reader byte by byte. Read everything available from
00065         # the StreamReader and check that the results equal the appropriate
00066         # entries from partialresults.
00067         q = Queue(b"")
00068         r = codecs.getreader(self.encoding)(q)
00069         result = ""
00070         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00071             q.write(bytes([c]))
00072             result += r.read()
00073             self.assertEqual(result, partialresult)
00074         # check that there's nothing left in the buffers
00075         self.assertEqual(r.read(), "")
00076         self.assertEqual(r.bytebuffer, b"")
00077 
00078         # do the check again, this time using a incremental decoder
00079         d = codecs.getincrementaldecoder(self.encoding)()
00080         result = ""
00081         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00082             result += d.decode(bytes([c]))
00083             self.assertEqual(result, partialresult)
00084         # check that there's nothing left in the buffers
00085         self.assertEqual(d.decode(b"", True), "")
00086         self.assertEqual(d.buffer, b"")
00087 
00088         # Check whether the reset method works properly
00089         d.reset()
00090         result = ""
00091         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00092             result += d.decode(bytes([c]))
00093             self.assertEqual(result, partialresult)
00094         # check that there's nothing left in the buffers
00095         self.assertEqual(d.decode(b"", True), "")
00096         self.assertEqual(d.buffer, b"")
00097 
00098         # check iterdecode()
00099         encoded = input.encode(self.encoding)
00100         self.assertEqual(
00101             input,
00102             "".join(codecs.iterdecode([bytes([c]) for c in encoded], self.encoding))
00103         )

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.MixInCheckStateHandling.check_state_handling_decode (   self,
  encoding,
  u,
  s 
) [inherited]

Definition at line 28 of file test_codecs.py.

00028 
00029     def check_state_handling_decode(self, encoding, u, s):
00030         for i in range(len(s)+1):
00031             d = codecs.getincrementaldecoder(encoding)()
00032             part1 = d.decode(s[:i])
00033             state = d.getstate()
00034             self.assertIsInstance(state[1], int)
00035             # Check that the condition stated in the documentation for
00036             # IncrementalDecoder.getstate() holds
00037             if not state[1]:
00038                 # reset decoder to the default state without anything buffered
00039                 d.setstate((state[0][:0], 0))
00040                 # Feeding the previous input may not produce any output
00041                 self.assertTrue(not d.decode(state[0]))
00042                 # The decoder must return to the same state
00043                 self.assertEqual(state, d.getstate())
00044             # Create a new decoder and set it to the state
00045             # we extracted from the old one
00046             d = codecs.getincrementaldecoder(encoding)()
00047             d.setstate(state)
00048             part2 = d.decode(s[i:], True)
00049             self.assertEqual(u, part1+part2)

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.MixInCheckStateHandling.check_state_handling_encode (   self,
  encoding,
  u,
  s 
) [inherited]

Definition at line 50 of file test_codecs.py.

00050 
00051     def check_state_handling_encode(self, encoding, u, s):
00052         for i in range(len(u)+1):
00053             d = codecs.getincrementalencoder(encoding)()
00054             part1 = d.encode(u[:i])
00055             state = d.getstate()
00056             d = codecs.getincrementalencoder(encoding)()
00057             d.setstate(state)
00058             part2 = d.encode(u[i:], True)
00059             self.assertEqual(s, part1+part2)

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.ReadTest.test_bug1098990_a (   self) [inherited]

Definition at line 250 of file test_codecs.py.

00250 
00251     def test_bug1098990_a(self):
00252         s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
00253         s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
00254         s3 = "next line.\r\n"
00255 
00256         s = (s1+s2+s3).encode(self.encoding)
00257         stream = io.BytesIO(s)
00258         reader = codecs.getreader(self.encoding)(stream)
00259         self.assertEqual(reader.readline(), s1)
00260         self.assertEqual(reader.readline(), s2)
00261         self.assertEqual(reader.readline(), s3)
00262         self.assertEqual(reader.readline(), "")

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_bug1098990_b (   self) [inherited]

Definition at line 263 of file test_codecs.py.

00263 
00264     def test_bug1098990_b(self):
00265         s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
00266         s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
00267         s3 = "stillokay:bbbbxx\r\n"
00268         s4 = "broken!!!!badbad\r\n"
00269         s5 = "againokay.\r\n"
00270 
00271         s = (s1+s2+s3+s4+s5).encode(self.encoding)
00272         stream = io.BytesIO(s)
00273         reader = codecs.getreader(self.encoding)(stream)
00274         self.assertEqual(reader.readline(), s1)
00275         self.assertEqual(reader.readline(), s2)
00276         self.assertEqual(reader.readline(), s3)
00277         self.assertEqual(reader.readline(), s4)
00278         self.assertEqual(reader.readline(), s5)
00279         self.assertEqual(reader.readline(), "")

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_bug1175396 (   self) [inherited]

Definition at line 154 of file test_codecs.py.

00154 
00155     def test_bug1175396(self):
00156         s = [
00157             '<%!--===================================================\r\n',
00158             '    BLOG index page: show recent articles,\r\n',
00159             '    today\'s articles, or articles of a specific date.\r\n',
00160             '========================================================--%>\r\n',
00161             '<%@inputencoding="ISO-8859-1"%>\r\n',
00162             '<%@pagetemplate=TEMPLATE.y%>\r\n',
00163             '<%@import=import frog.util, frog%>\r\n',
00164             '<%@import=import frog.objects%>\r\n',
00165             '<%@import=from frog.storageerrors import StorageError%>\r\n',
00166             '<%\r\n',
00167             '\r\n',
00168             'import logging\r\n',
00169             'log=logging.getLogger("Snakelets.logger")\r\n',
00170             '\r\n',
00171             '\r\n',
00172             'user=self.SessionCtx.user\r\n',
00173             'storageEngine=self.SessionCtx.storageEngine\r\n',
00174             '\r\n',
00175             '\r\n',
00176             'def readArticlesFromDate(date, count=None):\r\n',
00177             '    entryids=storageEngine.listBlogEntries(date)\r\n',
00178             '    entryids.reverse() # descending\r\n',
00179             '    if count:\r\n',
00180             '        entryids=entryids[:count]\r\n',
00181             '    try:\r\n',
00182             '        return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
00183             '    except StorageError,x:\r\n',
00184             '        log.error("Error loading articles: "+str(x))\r\n',
00185             '        self.abort("cannot load articles")\r\n',
00186             '\r\n',
00187             'showdate=None\r\n',
00188             '\r\n',
00189             'arg=self.Request.getArg()\r\n',
00190             'if arg=="today":\r\n',
00191             '    #-------------------- TODAY\'S ARTICLES\r\n',
00192             '    self.write("<h2>Today\'s articles</h2>")\r\n',
00193             '    showdate = frog.util.isodatestr() \r\n',
00194             '    entries = readArticlesFromDate(showdate)\r\n',
00195             'elif arg=="active":\r\n',
00196             '    #-------------------- ACTIVE ARTICLES redirect\r\n',
00197             '    self.Yredirect("active.y")\r\n',
00198             'elif arg=="login":\r\n',
00199             '    #-------------------- LOGIN PAGE redirect\r\n',
00200             '    self.Yredirect("login.y")\r\n',
00201             'elif arg=="date":\r\n',
00202             '    #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
00203             '    showdate = self.Request.getParameter("date")\r\n',
00204             '    self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n',
00205             '    entries = readArticlesFromDate(showdate)\r\n',
00206             'else:\r\n',
00207             '    #-------------------- RECENT ARTICLES\r\n',
00208             '    self.write("<h2>Recent articles</h2>")\r\n',
00209             '    dates=storageEngine.listBlogEntryDates()\r\n',
00210             '    if dates:\r\n',
00211             '        entries=[]\r\n',
00212             '        SHOWAMOUNT=10\r\n',
00213             '        for showdate in dates:\r\n',
00214             '            entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
00215             '            if len(entries)>=SHOWAMOUNT:\r\n',
00216             '                break\r\n',
00217             '                \r\n',
00218         ]
00219         stream = io.BytesIO("".join(s).encode(self.encoding))
00220         reader = codecs.getreader(self.encoding)(stream)
00221         for (i, line) in enumerate(reader):
00222             self.assertEqual(line, s[i])

Here is the call graph for this function:

Definition at line 436 of file test_codecs.py.

00436 
00437     def test_errors(self):
00438         self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
00439                           b"\xff", "strict", True)

Here is the call graph for this function:

Definition at line 440 of file test_codecs.py.

00440 
00441     def test_issue8941(self):
00442         # Issue #8941: insufficient result allocation when decoding into
00443         # surrogate pairs on UCS-2 builds.
00444         encoded = b'\x00\x01\x00\x00' * 1024
00445         self.assertEqual('\U00010000' * 1024,
00446                          codecs.utf_32_be_decode(encoded)[0])
00447 

Here is the call graph for this function:

Definition at line 410 of file test_codecs.py.

00410 
00411     def test_partial(self):
00412         self.check_partial(
00413             "\x00\xff\u0100\uffff",
00414             [
00415                 "",
00416                 "",
00417                 "",
00418                 "\x00",
00419                 "\x00",
00420                 "\x00",
00421                 "\x00",
00422                 "\x00\xff",
00423                 "\x00\xff",
00424                 "\x00\xff",
00425                 "\x00\xff",
00426                 "\x00\xff\u0100",
00427                 "\x00\xff\u0100",
00428                 "\x00\xff\u0100",
00429                 "\x00\xff\u0100",
00430                 "\x00\xff\u0100\uffff",
00431             ]
00432         )

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_readline (   self) [inherited]

Definition at line 104 of file test_codecs.py.

00104 
00105     def test_readline(self):
00106         def getreader(input):
00107             stream = io.BytesIO(input.encode(self.encoding))
00108             return codecs.getreader(self.encoding)(stream)
00109 
00110         def readalllines(input, keepends=True, size=None):
00111             reader = getreader(input)
00112             lines = []
00113             while True:
00114                 line = reader.readline(size=size, keepends=keepends)
00115                 if not line:
00116                     break
00117                 lines.append(line)
00118             return "|".join(lines)
00119 
00120         s = "foo\nbar\r\nbaz\rspam\u2028eggs"
00121         sexpected = "foo\n|bar\r\n|baz\r|spam\u2028|eggs"
00122         sexpectednoends = "foo|bar|baz|spam|eggs"
00123         self.assertEqual(readalllines(s, True), sexpected)
00124         self.assertEqual(readalllines(s, False), sexpectednoends)
00125         self.assertEqual(readalllines(s, True, 10), sexpected)
00126         self.assertEqual(readalllines(s, False, 10), sexpectednoends)
00127 
00128         # Test long lines (multiple calls to read() in readline())
00129         vw = []
00130         vwo = []
00131         for (i, lineend) in enumerate("\n \r\n \r \u2028".split()):
00132             vw.append((i*200)*"\3042" + lineend)
00133             vwo.append((i*200)*"\3042")
00134         self.assertEqual(readalllines("".join(vw), True), "".join(vw))
00135         self.assertEqual(readalllines("".join(vw), False),"".join(vwo))
00136 
00137         # Test lines where the first read might end with \r, so the
00138         # reader has to look ahead whether this is a lone \r or a \r\n
00139         for size in range(80):
00140             for lineend in "\n \r\n \r \u2028".split():
00141                 s = 10*(size*"a" + lineend + "xxx\n")
00142                 reader = getreader(s)
00143                 for i in range(10):
00144                     self.assertEqual(
00145                         reader.readline(keepends=True),
00146                         size*"a" + lineend,
00147                     )
00148                 reader = getreader(s)
00149                 for i in range(10):
00150                     self.assertEqual(
00151                         reader.readline(keepends=False),
00152                         size*"a",
00153                     )

Here is the call graph for this function:

Definition at line 223 of file test_codecs.py.

00223 
00224     def test_readlinequeue(self):
00225         q = Queue(b"")
00226         writer = codecs.getwriter(self.encoding)(q)
00227         reader = codecs.getreader(self.encoding)(q)
00228 
00229         # No lineends
00230         writer.write("foo\r")
00231         self.assertEqual(reader.readline(keepends=False), "foo")
00232         writer.write("\nbar\r")
00233         self.assertEqual(reader.readline(keepends=False), "")
00234         self.assertEqual(reader.readline(keepends=False), "bar")
00235         writer.write("baz")
00236         self.assertEqual(reader.readline(keepends=False), "baz")
00237         self.assertEqual(reader.readline(keepends=False), "")
00238 
00239         # Lineends
00240         writer.write("foo\r")
00241         self.assertEqual(reader.readline(keepends=True), "foo\r")
00242         writer.write("\nbar\r")
00243         self.assertEqual(reader.readline(keepends=True), "\n")
00244         self.assertEqual(reader.readline(keepends=True), "bar\r")
00245         writer.write("baz")
00246         self.assertEqual(reader.readline(keepends=True), "baz")
00247         self.assertEqual(reader.readline(keepends=True), "")
00248         writer.write("foo\r\n")
00249         self.assertEqual(reader.readline(keepends=True), "foo\r\n")

Here is the call graph for this function:

Definition at line 433 of file test_codecs.py.

00433 
00434     def test_simple(self):
00435         self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03")

Here is the call graph for this function:


Member Data Documentation

Definition at line 408 of file test_codecs.py.


The documentation for this class was generated from the following file: