Back to index

python3.2  3.2.2
Public Member Functions | Static Public Attributes
test.test_codecs.UTF16BETest Class Reference
Inheritance diagram for test.test_codecs.UTF16BETest:
Inheritance graph
[legend]
Collaboration diagram for test.test_codecs.UTF16BETest:
Collaboration graph
[legend]

List of all members.

Public Member Functions

def test_partial
def test_errors
def test_nonbmp
def check_partial
def test_readline
def test_bug1175396
def test_readlinequeue
def test_bug1098990_a
def test_bug1098990_b
def check_state_handling_decode
def check_state_handling_encode

Static Public Attributes

string encoding = "utf-16-be"

Detailed Description

Definition at line 552 of file test_codecs.py.


Member Function Documentation

def test.test_codecs.ReadTest.check_partial (   self,
  input,
  partialresults 
) [inherited]

Definition at line 61 of file test_codecs.py.

00061 
00062     def check_partial(self, input, partialresults):
00063         # get a StreamReader for the encoding and feed the bytestring version
00064         # of input to the reader byte by byte. Read everything available from
00065         # the StreamReader and check that the results equal the appropriate
00066         # entries from partialresults.
00067         q = Queue(b"")
00068         r = codecs.getreader(self.encoding)(q)
00069         result = ""
00070         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00071             q.write(bytes([c]))
00072             result += r.read()
00073             self.assertEqual(result, partialresult)
00074         # check that there's nothing left in the buffers
00075         self.assertEqual(r.read(), "")
00076         self.assertEqual(r.bytebuffer, b"")
00077 
00078         # do the check again, this time using a incremental decoder
00079         d = codecs.getincrementaldecoder(self.encoding)()
00080         result = ""
00081         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00082             result += d.decode(bytes([c]))
00083             self.assertEqual(result, partialresult)
00084         # check that there's nothing left in the buffers
00085         self.assertEqual(d.decode(b"", True), "")
00086         self.assertEqual(d.buffer, b"")
00087 
00088         # Check whether the reset method works properly
00089         d.reset()
00090         result = ""
00091         for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
00092             result += d.decode(bytes([c]))
00093             self.assertEqual(result, partialresult)
00094         # check that there's nothing left in the buffers
00095         self.assertEqual(d.decode(b"", True), "")
00096         self.assertEqual(d.buffer, b"")
00097 
00098         # check iterdecode()
00099         encoded = input.encode(self.encoding)
00100         self.assertEqual(
00101             input,
00102             "".join(codecs.iterdecode([bytes([c]) for c in encoded], self.encoding))
00103         )

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.MixInCheckStateHandling.check_state_handling_decode (   self,
  encoding,
  u,
  s 
) [inherited]

Definition at line 28 of file test_codecs.py.

00028 
00029     def check_state_handling_decode(self, encoding, u, s):
00030         for i in range(len(s)+1):
00031             d = codecs.getincrementaldecoder(encoding)()
00032             part1 = d.decode(s[:i])
00033             state = d.getstate()
00034             self.assertIsInstance(state[1], int)
00035             # Check that the condition stated in the documentation for
00036             # IncrementalDecoder.getstate() holds
00037             if not state[1]:
00038                 # reset decoder to the default state without anything buffered
00039                 d.setstate((state[0][:0], 0))
00040                 # Feeding the previous input may not produce any output
00041                 self.assertTrue(not d.decode(state[0]))
00042                 # The decoder must return to the same state
00043                 self.assertEqual(state, d.getstate())
00044             # Create a new decoder and set it to the state
00045             # we extracted from the old one
00046             d = codecs.getincrementaldecoder(encoding)()
00047             d.setstate(state)
00048             part2 = d.decode(s[i:], True)
00049             self.assertEqual(u, part1+part2)

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.MixInCheckStateHandling.check_state_handling_encode (   self,
  encoding,
  u,
  s 
) [inherited]

Definition at line 50 of file test_codecs.py.

00050 
00051     def check_state_handling_encode(self, encoding, u, s):
00052         for i in range(len(u)+1):
00053             d = codecs.getincrementalencoder(encoding)()
00054             part1 = d.encode(u[:i])
00055             state = d.getstate()
00056             d = codecs.getincrementalencoder(encoding)()
00057             d.setstate(state)
00058             part2 = d.encode(u[i:], True)
00059             self.assertEqual(s, part1+part2)

Here is the call graph for this function:

Here is the caller graph for this function:

def test.test_codecs.ReadTest.test_bug1098990_a (   self) [inherited]

Definition at line 250 of file test_codecs.py.

00250 
00251     def test_bug1098990_a(self):
00252         s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
00253         s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
00254         s3 = "next line.\r\n"
00255 
00256         s = (s1+s2+s3).encode(self.encoding)
00257         stream = io.BytesIO(s)
00258         reader = codecs.getreader(self.encoding)(stream)
00259         self.assertEqual(reader.readline(), s1)
00260         self.assertEqual(reader.readline(), s2)
00261         self.assertEqual(reader.readline(), s3)
00262         self.assertEqual(reader.readline(), "")

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_bug1098990_b (   self) [inherited]

Definition at line 263 of file test_codecs.py.

00263 
00264     def test_bug1098990_b(self):
00265         s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
00266         s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
00267         s3 = "stillokay:bbbbxx\r\n"
00268         s4 = "broken!!!!badbad\r\n"
00269         s5 = "againokay.\r\n"
00270 
00271         s = (s1+s2+s3+s4+s5).encode(self.encoding)
00272         stream = io.BytesIO(s)
00273         reader = codecs.getreader(self.encoding)(stream)
00274         self.assertEqual(reader.readline(), s1)
00275         self.assertEqual(reader.readline(), s2)
00276         self.assertEqual(reader.readline(), s3)
00277         self.assertEqual(reader.readline(), s4)
00278         self.assertEqual(reader.readline(), s5)
00279         self.assertEqual(reader.readline(), "")

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_bug1175396 (   self) [inherited]

Definition at line 154 of file test_codecs.py.

00154 
00155     def test_bug1175396(self):
00156         s = [
00157             '<%!--===================================================\r\n',
00158             '    BLOG index page: show recent articles,\r\n',
00159             '    today\'s articles, or articles of a specific date.\r\n',
00160             '========================================================--%>\r\n',
00161             '<%@inputencoding="ISO-8859-1"%>\r\n',
00162             '<%@pagetemplate=TEMPLATE.y%>\r\n',
00163             '<%@import=import frog.util, frog%>\r\n',
00164             '<%@import=import frog.objects%>\r\n',
00165             '<%@import=from frog.storageerrors import StorageError%>\r\n',
00166             '<%\r\n',
00167             '\r\n',
00168             'import logging\r\n',
00169             'log=logging.getLogger("Snakelets.logger")\r\n',
00170             '\r\n',
00171             '\r\n',
00172             'user=self.SessionCtx.user\r\n',
00173             'storageEngine=self.SessionCtx.storageEngine\r\n',
00174             '\r\n',
00175             '\r\n',
00176             'def readArticlesFromDate(date, count=None):\r\n',
00177             '    entryids=storageEngine.listBlogEntries(date)\r\n',
00178             '    entryids.reverse() # descending\r\n',
00179             '    if count:\r\n',
00180             '        entryids=entryids[:count]\r\n',
00181             '    try:\r\n',
00182             '        return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
00183             '    except StorageError,x:\r\n',
00184             '        log.error("Error loading articles: "+str(x))\r\n',
00185             '        self.abort("cannot load articles")\r\n',
00186             '\r\n',
00187             'showdate=None\r\n',
00188             '\r\n',
00189             'arg=self.Request.getArg()\r\n',
00190             'if arg=="today":\r\n',
00191             '    #-------------------- TODAY\'S ARTICLES\r\n',
00192             '    self.write("<h2>Today\'s articles</h2>")\r\n',
00193             '    showdate = frog.util.isodatestr() \r\n',
00194             '    entries = readArticlesFromDate(showdate)\r\n',
00195             'elif arg=="active":\r\n',
00196             '    #-------------------- ACTIVE ARTICLES redirect\r\n',
00197             '    self.Yredirect("active.y")\r\n',
00198             'elif arg=="login":\r\n',
00199             '    #-------------------- LOGIN PAGE redirect\r\n',
00200             '    self.Yredirect("login.y")\r\n',
00201             'elif arg=="date":\r\n',
00202             '    #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
00203             '    showdate = self.Request.getParameter("date")\r\n',
00204             '    self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n',
00205             '    entries = readArticlesFromDate(showdate)\r\n',
00206             'else:\r\n',
00207             '    #-------------------- RECENT ARTICLES\r\n',
00208             '    self.write("<h2>Recent articles</h2>")\r\n',
00209             '    dates=storageEngine.listBlogEntryDates()\r\n',
00210             '    if dates:\r\n',
00211             '        entries=[]\r\n',
00212             '        SHOWAMOUNT=10\r\n',
00213             '        for showdate in dates:\r\n',
00214             '            entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
00215             '            if len(entries)>=SHOWAMOUNT:\r\n',
00216             '                break\r\n',
00217             '                \r\n',
00218         ]
00219         stream = io.BytesIO("".join(s).encode(self.encoding))
00220         reader = codecs.getreader(self.encoding)(stream)
00221         for (i, line) in enumerate(reader):
00222             self.assertEqual(line, s[i])

Here is the call graph for this function:

Definition at line 570 of file test_codecs.py.

00570 
00571     def test_errors(self):
00572         self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
00573                           b"\xff", "strict", True)

Here is the call graph for this function:

Definition at line 574 of file test_codecs.py.

00574 
00575     def test_nonbmp(self):
00576         self.assertEqual("\U00010203".encode(self.encoding),
00577                          b'\xd8\x00\xde\x03')
00578         self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
00579                          "\U00010203")

Here is the call graph for this function:

Definition at line 555 of file test_codecs.py.

00555 
00556     def test_partial(self):
00557         self.check_partial(
00558             "\x00\xff\u0100\uffff",
00559             [
00560                 "",
00561                 "\x00",
00562                 "\x00",
00563                 "\x00\xff",
00564                 "\x00\xff",
00565                 "\x00\xff\u0100",
00566                 "\x00\xff\u0100",
00567                 "\x00\xff\u0100\uffff",
00568             ]
00569         )

Here is the call graph for this function:

def test.test_codecs.ReadTest.test_readline (   self) [inherited]

Definition at line 104 of file test_codecs.py.

00104 
00105     def test_readline(self):
00106         def getreader(input):
00107             stream = io.BytesIO(input.encode(self.encoding))
00108             return codecs.getreader(self.encoding)(stream)
00109 
00110         def readalllines(input, keepends=True, size=None):
00111             reader = getreader(input)
00112             lines = []
00113             while True:
00114                 line = reader.readline(size=size, keepends=keepends)
00115                 if not line:
00116                     break
00117                 lines.append(line)
00118             return "|".join(lines)
00119 
00120         s = "foo\nbar\r\nbaz\rspam\u2028eggs"
00121         sexpected = "foo\n|bar\r\n|baz\r|spam\u2028|eggs"
00122         sexpectednoends = "foo|bar|baz|spam|eggs"
00123         self.assertEqual(readalllines(s, True), sexpected)
00124         self.assertEqual(readalllines(s, False), sexpectednoends)
00125         self.assertEqual(readalllines(s, True, 10), sexpected)
00126         self.assertEqual(readalllines(s, False, 10), sexpectednoends)
00127 
00128         # Test long lines (multiple calls to read() in readline())
00129         vw = []
00130         vwo = []
00131         for (i, lineend) in enumerate("\n \r\n \r \u2028".split()):
00132             vw.append((i*200)*"\3042" + lineend)
00133             vwo.append((i*200)*"\3042")
00134         self.assertEqual(readalllines("".join(vw), True), "".join(vw))
00135         self.assertEqual(readalllines("".join(vw), False),"".join(vwo))
00136 
00137         # Test lines where the first read might end with \r, so the
00138         # reader has to look ahead whether this is a lone \r or a \r\n
00139         for size in range(80):
00140             for lineend in "\n \r\n \r \u2028".split():
00141                 s = 10*(size*"a" + lineend + "xxx\n")
00142                 reader = getreader(s)
00143                 for i in range(10):
00144                     self.assertEqual(
00145                         reader.readline(keepends=True),
00146                         size*"a" + lineend,
00147                     )
00148                 reader = getreader(s)
00149                 for i in range(10):
00150                     self.assertEqual(
00151                         reader.readline(keepends=False),
00152                         size*"a",
00153                     )

Here is the call graph for this function:

Definition at line 223 of file test_codecs.py.

00223 
00224     def test_readlinequeue(self):
00225         q = Queue(b"")
00226         writer = codecs.getwriter(self.encoding)(q)
00227         reader = codecs.getreader(self.encoding)(q)
00228 
00229         # No lineends
00230         writer.write("foo\r")
00231         self.assertEqual(reader.readline(keepends=False), "foo")
00232         writer.write("\nbar\r")
00233         self.assertEqual(reader.readline(keepends=False), "")
00234         self.assertEqual(reader.readline(keepends=False), "bar")
00235         writer.write("baz")
00236         self.assertEqual(reader.readline(keepends=False), "baz")
00237         self.assertEqual(reader.readline(keepends=False), "")
00238 
00239         # Lineends
00240         writer.write("foo\r")
00241         self.assertEqual(reader.readline(keepends=True), "foo\r")
00242         writer.write("\nbar\r")
00243         self.assertEqual(reader.readline(keepends=True), "\n")
00244         self.assertEqual(reader.readline(keepends=True), "bar\r")
00245         writer.write("baz")
00246         self.assertEqual(reader.readline(keepends=True), "baz")
00247         self.assertEqual(reader.readline(keepends=True), "")
00248         writer.write("foo\r\n")
00249         self.assertEqual(reader.readline(keepends=True), "foo\r\n")

Here is the call graph for this function:


Member Data Documentation

Definition at line 553 of file test_codecs.py.


The documentation for this class was generated from the following file: