Simple Lexical Analyser in C

-
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>

#define MAX_TOKEN_LENGTH 100

// Token types
typedef enum {
    TOKEN_KEYWORD,
    TOKEN_IDENTIFIER,
    TOKEN_NUMBER,
    TOKEN_STRING_LITERAL,
    TOKEN_CHAR_LITERAL,
    TOKEN_OPERATOR,
    TOKEN_PUNCTUATION,
    TOKEN_UNKNOWN,
    TOKEN_END
} TokenType;

// Token structure
typedef struct {
    TokenType type;
    char value[MAX_TOKEN_LENGTH];
} Token;

// Keywords in C
const char *keywords[] = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "int", "long", "register", "return", "short", "signed",
    "sizeof", "static", "struct", "switch", "typedef", "union",
    "unsigned", "void", "volatile", "while", "include", "define"
};

// Function prototypes
Token getNextToken(const char **input);
void printToken(Token token);
int isKeyword(const char *str);
int isOperatorChar(char c);

int main() {
    const char *input = "int main() { int x = 42; char y = 'a'; if (x > 0) { printf(\"Hello, World!\"); } }";
    const char *currentPosition = input;
    Token token;

    printf("Input: %s\n", input);
    printf("Tokens:\n");

    while ((token = getNextToken(&currentPosition)).type != TOKEN_END) {
        printToken(token);
    }

    return 0;
}

// Function to get the next token from the input
Token getNextToken(const char **input) {
    Token token;
    token.type = TOKEN_UNKNOWN;
    token.value[0] = '\0';

    // Skip whitespace
    while (isspace(**input)) {
        (*input)++;
    }

    // End of input
    if (**input == '\0') {
        token.type = TOKEN_END;
        return token;
    }

    // String literal
    if (**input == '"') {
        int length = 0;
        (*input)++;
        while (**input != '"' && **input != '\0' && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        if (**input == '"') {
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_STRING_LITERAL;
        return token;
    }

    // Character literal
    if (**input == '\'') {
        int length = 0;
        (*input)++;
        while (**input != '\'' && **input != '\0' && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        if (**input == '\'') {
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_CHAR_LITERAL;
        return token;
    }

    // Identifier or keyword
    if (isalpha(**input) || **input == '_') {
        int length = 0;
        while ((isalnum(**input) || **input == '_') && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = isKeyword(token.value) ? TOKEN_KEYWORD : TOKEN_IDENTIFIER;
        return token;
    }

    // Number
    if (isdigit(**input)) {
        int length = 0;
        while (isdigit(**input) && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_NUMBER;
        return token;
    }

    // Operators and punctuations
    if (isOperatorChar(**input) || ispunct(**input)) {
        token.value[0] = **input;
        token.value[1] = '\0';
        token.type = isOperatorChar(**input) ? TOKEN_OPERATOR : TOKEN_PUNCTUATION;
        (*input)++;
        return token;
    }

    // Unknown character
    token.value[0] = **input;
    token.value[1] = '\0';
    token.type = TOKEN_UNKNOWN;
    (*input)++;

    return token;
}

// Function to print the token
void printToken(Token token) {
    const char *typeStr;
    switch (token.type) {
        case TOKEN_KEYWORD:
            typeStr = "Keyword";
            break;
        case TOKEN_IDENTIFIER:
            typeStr = "Identifier";
            break;
        case TOKEN_NUMBER:
            typeStr = "Number";
            break;
        case TOKEN_STRING_LITERAL:
            typeStr = "String Literal";
            break;
        case TOKEN_CHAR_LITERAL:
            typeStr = "Char Literal";
            break;
        case TOKEN_OPERATOR:
            typeStr = "Operator";
            break;
        case TOKEN_PUNCTUATION:
            typeStr = "Punctuation";
            break;
        case TOKEN_UNKNOWN:
            typeStr = "Unknown";
            break;
        case TOKEN_END:
            typeStr = "End";
            break;
        default:
            typeStr = "Invalid";
            break;
    }
    printf("Token: %-15s Value: %s\n", typeStr, token.value);
}

// Function to check if a string is a keyword
int isKeyword(const char *str) {
    for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
        if (strcmp(str, keywords[i]) == 0) {
            return 1;
        }
    }
    return 0;
}

// Function to check if a character is an operator
int isOperatorChar(char c) {
    return strchr("+-*/=<>!&|%^", c) != NULL;
}

Comments

Post a Comment

Popular posts from this blog

KTU Compiler Lab CSL411 - Dr Binu V P

-
Preamble:  This course aims to offer students hands-on experience on compiler design concepts.Students will be able to familiarize with tools such as LEX and YACC and automate different phases of a compiler. This course helps the learners to enhance the capability to design and implement a compiler. Prerequisite: A sound knowledge in C programming, Data Structures, Formal languages and Automata Theory and Compiler design. About Me Syllabus and practise questions Lex yacc tutorial by Tom Neimann How to Install  flex and Bison Introduction to lex/flex Format of lex input files Lex Regular Expressions Input Matching in Lex Actions in Lex File Start Conditions in Lex Values available to the user in action section Reading and writing files Lex/Flex  Programs     Simple lexical analyser in C     count number of lines ,words and characters     check for valid identifiers     count the identifiers     converting the substring "abc" to "ABC"          count vowels and consona
Read more

lexical analyzer for a c program

-
 This program identifies tokens in a C program ignoring white spaces and comments.The input C program is given as argument ******************************* %{ #include <stdio.h> %} %option noyywrap /* Define regular expressions for different token types */ DIGIT      [0-9] LETTER     [a-zA-Z] IDENTIFIER {LETTER}({LETTER}|{DIGIT})* NUMBER     {DIGIT}+ WHITESPACE [ \t\n]+ COMMENT1    "//".* COMMENT2  "/*"([^*]|\*+[^/*])*\*+"/" %% {WHITESPACE}        { /* Ignore whitespace */ } {COMMENT1}           { /* Ignore comments */ } {COMMENT2}           { /* Ignore comments */ } "int"               { printf("TOKEN_INT\n"); } "return"            { printf("TOKEN_RETURN\n"); } "if"                { printf("TOKEN_IF\n"); } "else"              { printf("TOKEN_ELSE\n"); } "while"             { printf("TOKEN_WHILE\n"); } "for"               { printf("TOKEN_
Read more

13.Precedence and conflict resolution in yacc

-
We had seen that , when you feed yacc a grammar that is not LALR(1), it shows a warning and resolve the conflicts automatically. For a shift/reduce conflict, yacc will choose the shift. In a reduce/reduce conflict, it will reduce using the rule declared first in the file. We can control what happens by explicitly declaring precedence and associativity for operators. Rather than re-writing the grammar to implicitly control the precedence and associativity with a bunch of intermediate non-terminals, we can directly indicate the precedence so that yacc will know how to break ties. In the definitions section, we can add any number of precedence levels, one per line, from lowest to highest, and indicate the associativity (either left, right, or nonassoc). Several terminals can be on the same line to assign them equal precedence. Example: %token T_Int %left '+' %left '*' %right '^' %% E : E '+' E |E '*'
Read more