Simple Lexical Analyser in C

-
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>

#define MAX_TOKEN_LENGTH 100

// Token types
typedef enum {
    TOKEN_KEYWORD,
    TOKEN_IDENTIFIER,
    TOKEN_NUMBER,
    TOKEN_STRING_LITERAL,
    TOKEN_CHAR_LITERAL,
    TOKEN_OPERATOR,
    TOKEN_PUNCTUATION,
    TOKEN_UNKNOWN,
    TOKEN_END
} TokenType;

// Token structure
typedef struct {
    TokenType type;
    char value[MAX_TOKEN_LENGTH];
} Token;

// Keywords in C
const char *keywords[] = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "int", "long", "register", "return", "short", "signed",
    "sizeof", "static", "struct", "switch", "typedef", "union",
    "unsigned", "void", "volatile", "while", "include", "define"
};

// Function prototypes
Token getNextToken(const char **input);
void printToken(Token token);
int isKeyword(const char *str);
int isOperatorChar(char c);

int main() {
    const char *input = "int main() { int x = 42; char y = 'a'; if (x > 0) { printf(\"Hello, World!\"); } }";
    const char *currentPosition = input;
    Token token;

    printf("Input: %s\n", input);
    printf("Tokens:\n");

    while ((token = getNextToken(&currentPosition)).type != TOKEN_END) {
        printToken(token);
    }

    return 0;
}

// Function to get the next token from the input
Token getNextToken(const char **input) {
    Token token;
    token.type = TOKEN_UNKNOWN;
    token.value[0] = '\0';

    // Skip whitespace
    while (isspace(**input)) {
        (*input)++;
    }

    // End of input
    if (**input == '\0') {
        token.type = TOKEN_END;
        return token;
    }

    // String literal
    if (**input == '"') {
        int length = 0;
        (*input)++;
        while (**input != '"' && **input != '\0' && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        if (**input == '"') {
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_STRING_LITERAL;
        return token;
    }

    // Character literal
    if (**input == '\'') {
        int length = 0;
        (*input)++;
        while (**input != '\'' && **input != '\0' && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        if (**input == '\'') {
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_CHAR_LITERAL;
        return token;
    }

    // Identifier or keyword
    if (isalpha(**input) || **input == '_') {
        int length = 0;
        while ((isalnum(**input) || **input == '_') && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = isKeyword(token.value) ? TOKEN_KEYWORD : TOKEN_IDENTIFIER;
        return token;
    }

    // Number
    if (isdigit(**input)) {
        int length = 0;
        while (isdigit(**input) && length < MAX_TOKEN_LENGTH - 1) {
            token.value[length++] = **input;
            (*input)++;
        }
        token.value[length] = '\0';
        token.type = TOKEN_NUMBER;
        return token;
    }

    // Operators and punctuations
    if (isOperatorChar(**input) || ispunct(**input)) {
        token.value[0] = **input;
        token.value[1] = '\0';
        token.type = isOperatorChar(**input) ? TOKEN_OPERATOR : TOKEN_PUNCTUATION;
        (*input)++;
        return token;
    }

    // Unknown character
    token.value[0] = **input;
    token.value[1] = '\0';
    token.type = TOKEN_UNKNOWN;
    (*input)++;

    return token;
}

// Function to print the token
void printToken(Token token) {
    const char *typeStr;
    switch (token.type) {
        case TOKEN_KEYWORD:
            typeStr = "Keyword";
            break;
        case TOKEN_IDENTIFIER:
            typeStr = "Identifier";
            break;
        case TOKEN_NUMBER:
            typeStr = "Number";
            break;
        case TOKEN_STRING_LITERAL:
            typeStr = "String Literal";
            break;
        case TOKEN_CHAR_LITERAL:
            typeStr = "Char Literal";
            break;
        case TOKEN_OPERATOR:
            typeStr = "Operator";
            break;
        case TOKEN_PUNCTUATION:
            typeStr = "Punctuation";
            break;
        case TOKEN_UNKNOWN:
            typeStr = "Unknown";
            break;
        case TOKEN_END:
            typeStr = "End";
            break;
        default:
            typeStr = "Invalid";
            break;
    }
    printf("Token: %-15s Value: %s\n", typeStr, token.value);
}

// Function to check if a string is a keyword
int isKeyword(const char *str) {
    for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
        if (strcmp(str, keywords[i]) == 0) {
            return 1;
        }
    }
    return 0;
}

// Function to check if a character is an operator
int isOperatorChar(char c) {
    return strchr("+-*/=<>!&|%^", c) != NULL;
}

Comments

Post a Comment

Popular posts from this blog

KTU Compiler Lab CSL411 - Dr Binu V P

-
Preamble:  This course aims to offer students hands-on experience on compiler design concepts.Students will be able to familiarize with tools such as LEX and YACC and automate different phases of a compiler. This course helps the learners to enhance the capability to design and implement a compiler. Prerequisite: A sound knowledge in C programming, Data Structures, Formal languages and Automata Theory and Compiler design. About Me Syllabus and practise questions Lex yacc tutorial by Tom Neimann How to Install  flex and Bison Introduction to lex/flex Format of lex input files Lex Regular Expressions Input Matching in Lex Actions in Lex File Start Conditions in Lex Values available to the user in action section Reading and writing files Lex/Flex  Programs     Simple lexical analyser in C     count number of lines ,words and characters     check for valid identifiers     count the identifiers     converting the substring "abc...
Read more

lexical analyzer for a c program

-
 This program identifies tokens in a C program ignoring white spaces and comments.The input C program is given as argument ******************************* %{ #include <stdio.h> %} %option noyywrap /* Define regular expressions for different token types */ DIGIT      [0-9] LETTER     [a-zA-Z] IDENTIFIER {LETTER}({LETTER}|{DIGIT})* NUMBER     {DIGIT}+ WHITESPACE [ \t\n]+ COMMENT1    "//".* COMMENT2  "/*"([^*]|\*+[^/*])*\*+"/" %% {WHITESPACE}        { /* Ignore whitespace */ } {COMMENT1}           { /* Ignore comments */ } {COMMENT2}           { /* Ignore comments */ } "int"               { printf("TOKEN_INT\n"); } "return"            { printf("TOKEN_RETURN\n"); } "if"                { printf("TOKEN_IF\n"); } "else"    ...
Read more

count frequency of occurrence of a word - lex program

-
 LEX code to count the frequency of the given word(sachin)  in an input.txt file *************************************************** %{ #include<stdio.h> #include<string.h> char word [] = "sachin"; int count = 0; %} %option noyywrap /* Rule Section */ %% [a-zA-Z]+ { if(strcmp(yytext, word)==0)                                 count++; } .      ; \n      ; %% /* code section */ int main() {                 extern FILE *yyin, *yyout;                 /* open the input file                 in read mode */                 yyin=fopen("input.txt", "r");                 yylex();                 printf("No of occurance of the word ...
Read more