class Shell::Tokenizer

Shell::Tokenizer.new(String input) -> Tokenizer

instance.create_token(Symbol type, Integer start_pos, Integer length) -> token_hash

instance.eof?() -> bool

instance.next_token() -> token_hash?

instance.skip_whitespace() -> void

instance.tokenize_quoted_string(String quote_char, Integer start_pos) -> token_hash

instance.tokenize_whitespace(Integer start_pos) -> token_hash

instance.tokenize_word(Integer start_pos) -> token_hash