Class DObjcTokenizer

Inherits from:
Object
Declared in:
DObjcTokenizer.h

Class Hierarchy

    Object
      |
      +---DObjcTokenizer

Class Description

The class DObjcTokenizer implements an Objective-C lexical scanner. The tokenizer scans a text stream for whitespace, strings, keywords, comments, operators, numbers, predefined types, names, character constants, etc. There are two methods: nextToken and checkToken. The first method checks for a token and if there is a match, the scanner location is moved to the next location in the source. The second method only returns the result of the match, the client must call the 'next' method to move the scanner location.

Example:
#include <stdio.h>
#include "ofc/DObjcTokenizer.h"
#include "ofc/DFile.h"

int main(int argc, char *argv[])
{
  DObjcTokenizer *src  = [DObjcTokenizer new];
  DFile          *file = [DFile          new];

  if ([file open :"DObjcTokenizer_exp.m" :"r"])
  {
    if ([src source :file :"DObjcTokenizer_exp.m"])
    {
      [src skipWhiteSpace :YES];       // Set: skip whitespace

      while (![src isEof])
      {
        int token = [src nextToken];   // Scan all tokens in the file

        if ([DObjcTokenizer isLogicalOperator :token])
        {
          printf("Logical operator: %s\n", [src text]);
        }
        else if ([DObjcTokenizer isComparisonOperator :token])
        {
          printf("Comparison operator: %s\n", [src text]);
        }
        else if ([DObjcTokenizer isBitOperator :token])
        {
          printf("Bit operator:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isAssignmentOperator :token])
        {
          printf("Assignment operator:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isArithmeticOperator :token])
        {
          printf("Arithmetic operator:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isOperator :token])
        {
          printf("Operator:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isPredefinedTypeKeyword :token])
        {
          printf("Predefined type:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isStorageKeyword :token])
        {
          printf("Storage:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isTypeDefinitionKeyword :token])
        {
          printf("Typedefinition:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isFlowKeyword :token])
        {
          printf("Flow keyword:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isObjcKeyword :token])
        {
          printf("Objc-keyword:%s\n", [src text]);
        }
        else if ([DObjcTokenizer isDirective :token])
        {
          printf("Directive:%s\n", [src text]);
        }
        else
          printf("Token:%d Text:%s\n", token, [src text]);
      }
    }
    else
      printf("File \"DObjcTokenizer_exp.m\" could not be used for the objc-tokenizer.\n");

    [file close];
  }
  else
    printf("File \"DObjcTokenizer_exp.m\" could not be opened:%d.\n", [file error]);

  [file free];                         // Cleanup
  [src  free];

  return 0;
}
Last modified:
25-Aug-2008 (DObjcTokenizer.h)

Instance Variables

private DList *_sources
the stack with the sources
private DSource *_source
the source for the tokenizer
private DText *_text
the last scanned, processed text
private DText *_raw
the last scanned, unprocessed text
private BOOL _eoff
is end of file reached ?
private BOOL _eofl
is end of line reached ?
private BOOL _sofl
is start of line active ?
private BOOL _comment
is (block) comment active ?
private unsigned _scanned
the length of the scanned text
private BOOL _skipWhiteSpace
should whitespace be skipped ? (def. YES)
private const char *_scanLine
the current line for the scanner
private int _currIndex
the current index in the scanner line
private int _nextIndex
the next index in the scanner line
private int _thrdIndex
the third index in the scanner line
private char _currChar
the current character from the scanner line
private char _nextChar
the next character from the scanner line
private char _thrdChar
the third character from the scanner line
private BOOL _currEsc
is the current character escaped ?
private BOOL _nextEsc
is the next character escaped ?
private BOOL _thrdEsc
is the third character escaped ?

Method Index


generated 06-Sep-2008 by ObjcDoc 3.0.0