Back to index

supertuxkart  0.5+dfsg1
Public Types | Public Member Functions | Private Types | Private Member Functions | Private Attributes
lisp::Lexer Class Reference

#include <lexer.hpp>

Collaboration diagram for lisp::Lexer:
Collaboration graph
[legend]

List of all members.

Public Types

enum  TokenType {
  TOKEN_EOF, TOKEN_OPEN_PAREN, TOKEN_CLOSE_PAREN, TOKEN_TRANSLATION,
  TOKEN_SYMBOL, TOKEN_STRING, TOKEN_INTEGER, TOKEN_REAL,
  TOKEN_TRUE, TOKEN_FALSE
}

Public Member Functions

 Lexer (std::istream &stream)
 ~Lexer ()
TokenType getNextToken ()
const char * getString () const
int getLineNumber () const

Private Types

enum  { MAX_TOKEN_LENGTH = 4096, LEXER_BUFFER_SIZE = 1024 }

Private Member Functions

void nextChar ()

Private Attributes

std::istream & m_stream
bool m_is_eof
int m_line_number
char m_buffer [LEXER_BUFFER_SIZE+1]
char * m_buffer_end
char * m_c
char m_token_string [MAX_TOKEN_LENGTH+1]
int m_token_length

Detailed Description

Definition at line 26 of file lexer.hpp.


Member Enumeration Documentation

anonymous enum [private]
Enumerator:
MAX_TOKEN_LENGTH 
LEXER_BUFFER_SIZE 

Definition at line 55 of file lexer.hpp.

        {
            MAX_TOKEN_LENGTH = 4096,
            LEXER_BUFFER_SIZE = 1024
        };
Enumerator:
TOKEN_EOF 
TOKEN_OPEN_PAREN 
TOKEN_CLOSE_PAREN 
TOKEN_TRANSLATION 
TOKEN_SYMBOL 
TOKEN_STRING 
TOKEN_INTEGER 
TOKEN_REAL 
TOKEN_TRUE 
TOKEN_FALSE 

Definition at line 29 of file lexer.hpp.


Constructor & Destructor Documentation

lisp::Lexer::Lexer ( std::istream &  stream)

Definition at line 36 of file lexer.cpp.

            : m_stream(newstream), m_is_eof(false)
    {
        try
        {
            // trigger a refill of the m_buffer
            m_c = 0;
            m_buffer_end = m_c + 1;
            nextChar();
        }
        catch(EOFException& e)
        {(void)e;  // avoid 'unreferenced local variable' warning
        }
    }

Here is the call graph for this function:

Definition at line 53 of file lexer.cpp.

    {}

Member Function Documentation

int lisp::Lexer::getLineNumber ( ) const [inline]

Definition at line 51 of file lexer.hpp.

            { return m_line_number; }

Here is the caller graph for this function:

Definition at line 87 of file lexer.cpp.

    {
        static const char* delims = "\"();";

        try
        {
            while(isspace(*m_c))
            {
                nextChar();
                if(*m_c == '\n')
                    ++m_line_number;
            };

            m_token_length = 0;

            switch(*m_c)
            {
            case ';': // comment
                while(true)
                {
                    nextChar();
                    if(*m_c == '\n')
                    {
                        ++m_line_number;
                        break;
                    }
                }
                return getNextToken(); // and again
            case '(':
                nextChar();
                return TOKEN_OPEN_PAREN;
            case ')':
                nextChar();
                return TOKEN_CLOSE_PAREN;
            case '"': // string
                {
                    const int STARTLINE = m_line_number;
                    try
                    {
                        while(1)
                        {
                            nextChar();
                            if(*m_c == '"')
                                break;

                            if(*m_c == '\\')
                            {
                                nextChar();
                                switch(*m_c)
                                {
                                case 'n':
                                    *m_c = '\n';
                                    break;
                                case 't':
                                    *m_c = '\t';
                                    break;
                                }
                            }
                            if(m_token_length < MAX_TOKEN_LENGTH)
                                m_token_string[m_token_length++] = *m_c;
                        }
                        m_token_string[m_token_length] = 0;
                    }
                    catch(EOFException& )
                    {
                        char msg[MAX_ERROR_MESSAGE_LENGTH];
                        snprintf(msg, sizeof(msg),
                                 "Parse error in line %d: EOF while parsing string.",
                                 STARTLINE);
                        throw std::runtime_error(msg);
                    }
                    nextChar();
                    return TOKEN_STRING;
                }
            case '#': // constant
                try
                {
                    nextChar();

                    while(isalnum(*m_c) || *m_c == '_')
                    {
                        if(m_token_length < MAX_TOKEN_LENGTH)
                            m_token_string[m_token_length++] = *m_c;
                        nextChar();
                    }
                    m_token_string[m_token_length] = 0;
                }
                catch(EOFException& )
                {
                    char msg[MAX_ERROR_MESSAGE_LENGTH];
                    snprintf(msg, sizeof(msg), 
                             "Parse Error in line %d: EOF while parsing constant.",
                             m_line_number);
                    throw std::runtime_error(msg);
                }

                if(strcmp(m_token_string, "t") == 0)
                    return TOKEN_TRUE;
                if(strcmp(m_token_string, "f") == 0)
                    return TOKEN_FALSE;

                // this would be the place to add more sophisticated handling of
                // constants

                {
                    char msg[MAX_ERROR_MESSAGE_LENGTH];
                    snprintf(msg, sizeof(msg), 
                             "Parse Error in line %d: Unknown constant '%s'.",
                             m_line_number, m_token_string);
                    throw std::runtime_error(msg);
                }

            case '_': // can be begin translation
                try
              {
                  nextChar();
                  if(*m_c == '(')
                  {
                    nextChar();
                    return TOKEN_TRANSLATION;
                  }
                  m_token_string[m_token_length++] = '_';
                  // Fall through to symbol handling
              }  
                catch(EOFException& )
                {
              }
            default:
                if(isdigit(*m_c) || *m_c == '-')
                {
                    bool have_nondigits = false;
                    bool have_digits = false;
                    int have_floating_point = 0;

                    do
                    {
                        if(isdigit(*m_c))
                            have_digits = true;
                        else if(*m_c == '.')
                            ++have_floating_point;
                        else if(isalnum(*m_c) || *m_c == '_')
                            have_nondigits = true;

                        if(m_token_length < MAX_TOKEN_LENGTH)
                            m_token_string[m_token_length++] = *m_c;

                        nextChar();
                    }
                    while(!isspace(*m_c) && !strchr(delims, *m_c));

                    m_token_string[m_token_length] = 0;

                    // no nextChar

                    if(have_nondigits || !have_digits || have_floating_point > 1)
                        return TOKEN_SYMBOL;
                    else if(have_floating_point == 1)
                        return TOKEN_REAL;
                    else
                        return TOKEN_INTEGER;
                }
                else
                {
                    do
                    {
                        if(m_token_length < MAX_TOKEN_LENGTH)
                            m_token_string[m_token_length++] = *m_c;
                        nextChar();
                    }
                    while(!isspace(*m_c) && !strchr(delims, *m_c));
                    m_token_string[m_token_length] = 0;

                    // no nextChar

                    return TOKEN_SYMBOL;
                }
            }
        }
        catch(EOFException& )
        {
            return TOKEN_EOF;
        }
    }

Here is the call graph for this function:

Here is the caller graph for this function:

const char* lisp::Lexer::getString ( ) const [inline]

Definition at line 48 of file lexer.hpp.

            { return m_token_string; }

Here is the caller graph for this function:

void lisp::Lexer::nextChar ( ) [inline, private]

Definition at line 59 of file lexer.cpp.

    {
        ++m_c;
        if(m_c >= m_buffer_end)
        {
            if(m_is_eof)
                throw EOFException();
            m_stream.read(m_buffer, LEXER_BUFFER_SIZE);
            std::streamsize n = m_stream.gcount();

            m_c = m_buffer;
            m_buffer_end = m_buffer + n;

            // the following is a hack that appends an additional ' ' at the end of
            // the file to avoid problems when parsing symbols/elements and a sudden
            // EOF. This is faster than relying on unget and IMO also nicer.
            if(n < LEXER_BUFFER_SIZE || n == 0)
            {
                *m_buffer_end = ' ';
                ++m_buffer_end;
                m_is_eof = true;
            }
        }
    }

Here is the caller graph for this function:


Member Data Documentation

Definition at line 66 of file lexer.hpp.

char* lisp::Lexer::m_buffer_end [private]

Definition at line 67 of file lexer.hpp.

char* lisp::Lexer::m_c [private]

Definition at line 68 of file lexer.hpp.

bool lisp::Lexer::m_is_eof [private]

Definition at line 64 of file lexer.hpp.

Definition at line 65 of file lexer.hpp.

std::istream& lisp::Lexer::m_stream [private]

Definition at line 63 of file lexer.hpp.

Definition at line 70 of file lexer.hpp.

Definition at line 69 of file lexer.hpp.


The documentation for this class was generated from the following files: