Back to index

python3.2  3.2.2
Classes | Defines | Enumerations | Functions
tokenizer.h File Reference
#include "object.h"
#include "token.h"
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  tok_state

Defines

#define MAXINDENT   100 /* Max indentation level */

Enumerations

enum  decoding_state { STATE_INIT, STATE_RAW, STATE_NORMAL }

Functions

struct tok_statePyTokenizer_FromString (const char *, int)
struct tok_statePyTokenizer_FromUTF8 (const char *, int)
struct tok_statePyTokenizer_FromFile (FILE *, char *, char *, char *)
void PyTokenizer_Free (struct tok_state *)
int PyTokenizer_Get (struct tok_state *, char **, char **)
char * PyTokenizer_RestoreEncoding (struct tok_state *tok, int len, int *offset)
char * PyTokenizer_FindEncoding (int)

Class Documentation

struct tok_state

Definition at line 22 of file tokenizer.h.

Class Members
int alterror
int altindstack
int alttabsize
int altwarning
int atbol
char * buf
char * cur
int done
char * end
const char * filename
FILE * fp
int indent
int indstack
char * inp
int level
int lineno
char * nextprompt
int pendin
char * prompt
char * start
int tabsize

Define Documentation

#define MAXINDENT   100 /* Max indentation level */

Definition at line 13 of file tokenizer.h.


Enumeration Type Documentation

Enumerator:
STATE_INIT 
STATE_RAW 
STATE_NORMAL 

Definition at line 15 of file tokenizer.h.

                    {
    STATE_INIT,
    STATE_RAW,
    STATE_NORMAL        /* have a codec associated with input */
};

Function Documentation

Definition at line 1705 of file tokenizer.c.

{
    struct tok_state *tok;
    FILE *fp;
    char *p_start =NULL , *p_end =NULL , *encoding = NULL;

    fd = dup(fd);
    if (fd < 0) {
        return NULL;
    }
    fp = fdopen(fd, "r");
    if (fp == NULL) {
        return NULL;
    }
    tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
    if (tok == NULL) {
        fclose(fp);
        return NULL;
    }
    while (tok->lineno < 2 && tok->done == E_OK) {
        PyTokenizer_Get(tok, &p_start, &p_end);
    }
    fclose(fp);
    if (tok->encoding) {
        encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
        if (encoding)
        strcpy(encoding, tok->encoding);
    }
    PyTokenizer_Free(tok);
    return encoding;
}

Definition at line 852 of file tokenizer.c.

{
    if (tok->encoding != NULL)
        PyMem_FREE(tok->encoding);
#ifndef PGEN
    Py_XDECREF(tok->decoding_readline);
    Py_XDECREF(tok->decoding_buffer);
#endif
    if (tok->fp != NULL && tok->buf != NULL)
        PyMem_FREE(tok->buf);
    if (tok->input)
        PyMem_FREE((char *)tok->input);
    PyMem_FREE(tok);
}

Here is the caller graph for this function:

struct tok_state* PyTokenizer_FromFile ( FILE *  ,
char *  ,
char *  ,
char *   
) [read]

Definition at line 820 of file tokenizer.c.

{
    struct tok_state *tok = tok_new();
    if (tok == NULL)
        return NULL;
    if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
        PyTokenizer_Free(tok);
        return NULL;
    }
    tok->cur = tok->inp = tok->buf;
    tok->end = tok->buf + BUFSIZ;
    tok->fp = fp;
    tok->prompt = ps1;
    tok->nextprompt = ps2;
    if (enc != NULL) {
        /* Must copy encoding declaration since it
           gets copied into the parse tree. */
        tok->encoding = PyMem_MALLOC(strlen(enc)+1);
        if (!tok->encoding) {
            PyTokenizer_Free(tok);
            return NULL;
        }
        strcpy(tok->encoding, enc);
        tok->decoding_state = STATE_NORMAL;
    }
    return tok;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct tok_state* PyTokenizer_FromString ( const char *  ,
int   
) [read]

Definition at line 772 of file tokenizer.c.

{
    struct tok_state *tok = tok_new();
    if (tok == NULL)
        return NULL;
    str = (char *)decode_str(str, exec_input, tok);
    if (str == NULL) {
        PyTokenizer_Free(tok);
        return NULL;
    }

    /* XXX: constify members. */
    tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
    return tok;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct tok_state* PyTokenizer_FromUTF8 ( const char *  ,
int   
) [read]

Definition at line 789 of file tokenizer.c.

{
    struct tok_state *tok = tok_new();
    if (tok == NULL)
        return NULL;
#ifndef PGEN
    tok->input = str = translate_newlines(str, exec_input, tok);
#endif
    if (str == NULL) {
        PyTokenizer_Free(tok);
        return NULL;
    }
    tok->decoding_state = STATE_RAW;
    tok->read_coding_spec = 1;
    tok->enc = NULL;
    tok->str = str;
    tok->encoding = (char *)PyMem_MALLOC(6);
    if (!tok->encoding) {
        PyTokenizer_Free(tok);
        return NULL;
    }
    strcpy(tok->encoding, "utf-8");

    /* XXX: constify members. */
    tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
    return tok;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int PyTokenizer_Get ( struct tok_state ,
char **  ,
char **   
)

Definition at line 1685 of file tokenizer.c.

{
    int result = tok_get(tok, p_start, p_end);
    if (tok->decoding_erred) {
        result = ERRORTOKEN;
        tok->done = E_DECODE;
    }
    return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* PyTokenizer_RestoreEncoding ( struct tok_state tok,
int  len,
int offset 
)