.. index:: pair: class; EE::UI::Doc::SyntaxTokenizer
.. _doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer:

class EE::UI::Doc::SyntaxTokenizer
==================================

.. toctree::
	:hidden:




.. ref-code-block:: cpp
	:class: doxyrest-overview-code-block

	#include <syntaxtokenizer.hpp>
	
	class SyntaxTokenizer {
	public:
		// methods
	
		static std::pair<std::vector<:ref:`SyntaxToken<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_token>`>, :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`> :target:`tokenize<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer_1a99b07713794d7e7583def44df294c709>`(const :ref:`SyntaxDefinition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_definition>`& syntax, const std::string& text, const :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false);
		static std::pair<std::vector<:ref:`SyntaxTokenPosition<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_token_position>`>, :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`> :target:`tokenizePosition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer_1a96fc0b6debcd3bb87aba0443ee236482>`(const :ref:`SyntaxDefinition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_definition>`& syntax, const std::string& text, const :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false);
		static std::pair<std::vector<:ref:`SyntaxTokenComplete<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_token_complete>`>, :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`> :target:`tokenizeComplete<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer_1ab91cab95f7921041ca346f3146efe4de>`(const :ref:`SyntaxDefinition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_definition>`& syntax, const std::string& text, const :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false);
	
		static :ref:`Text<doxid-class_e_e_1_1_graphics_1_1_text>`& :target:`tokenizeText<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer_1a1a9c847b2ef68b58cecb30c03fa39a77>`(
			const :ref:`SyntaxDefinition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_definition>`& syntax,
			const :ref:`SyntaxColorScheme<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_color_scheme>`& colorScheme,
			:ref:`Text<doxid-class_e_e_1_1_graphics_1_1_text>`& text,
			const size_t& startIndex = 0,
			const size_t& endIndex = 0xFFFFFFFF,
			bool skipSubSyntaxSeparator = false,
			const std::string& trimChars = ""
		);
	
		static :ref:`SyntaxStateRestored<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state_restored>` :target:`retrieveSyntaxState<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_tokenizer_1a08b9b2720e3e2b895242e77dd19cd1d1>`(const :ref:`SyntaxDefinition<doxid-class_e_e_1_1_u_i_1_1_doc_1_1_syntax_definition>`& syntax, const :ref:`SyntaxState<doxid-struct_e_e_1_1_u_i_1_1_doc_1_1_syntax_state>`& state);
	};