Refactored some c++ tokeniser classes.

extras/Introjucer/Builds/MacOSX/The Introjucer.xcodeproj/project.pbxproj

@@ -228,6 +228,7 @@
 		3F1501FE4867042830B25DD5 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_CustomTypeface.cpp"; path = "../../../../modules/juce_graphics/fonts/juce_CustomTypeface.cpp"; sourceTree = "SOURCE_ROOT"; };
 		3F222D26A164E7AD78A33B3D = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ToggleButton.h"; path = "../../../../modules/juce_gui_basics/buttons/juce_ToggleButton.h"; sourceTree = "SOURCE_ROOT"; };
 		3FAFD16944A6674164D23914 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CharPointer_ASCII.h"; path = "../../../../modules/juce_core/text/juce_CharPointer_ASCII.h"; sourceTree = "SOURCE_ROOT"; };
+		4076F14FE0016A4DA9087771 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		40A83A20ED109AAAE6D63AF4 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_DrawableRectangle.h"; path = "../../../../modules/juce_gui_basics/drawables/juce_DrawableRectangle.h"; sourceTree = "SOURCE_ROOT"; };
 		41105E536155E394E54BDD35 = { isa = PBXFileReference; lastKnownFileType = file.xml; name = "colourscheme_dark.xml"; path = "../../Source/BinaryData/colourscheme_dark.xml"; sourceTree = "SOURCE_ROOT"; };
 		412B48D0F05D78A73064B36B = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_TopLevelWindow.cpp"; path = "../../../../modules/juce_gui_basics/windows/juce_TopLevelWindow.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -1535,7 +1536,8 @@
 				CC3F5263B16932FF4E74B9D8,
 				318F4AB64A5D4068CC1C4F29,
 				09349C0C8300CA414AAE93A0,
-				6F8A53EDF09094C83DE4D111 ); name = "code_editor"; sourceTree = "<group>"; };
+				6F8A53EDF09094C83DE4D111,
+				4076F14FE0016A4DA9087771 ); name = "code_editor"; sourceTree = "<group>"; };
 		1D09D8A2B803860D8927C366 = { isa = PBXGroup; children = (
 				3419904525A3F1FF0CC50687,
 				BA95268EEE80F9509EC1F76F ); name = documents; sourceTree = "<group>"; };

extras/Introjucer/Builds/VisualStudio2005/The Introjucer.vcproj

@@ -3339,6 +3339,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/Introjucer/Builds/VisualStudio2008/The Introjucer.vcproj

@@ -3339,6 +3339,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/Introjucer/Builds/VisualStudio2010/The Introjucer.vcxproj

@@ -1319,6 +1319,7 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeEditorComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeTokeniser.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_ActiveXControlComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_NSViewComponent.h"/>

extras/Introjucer/Builds/VisualStudio2010/The Introjucer.vcxproj.filters

@@ -2244,6 +2244,9 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h">
       <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h">
+      <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h">
       <Filter>Juce Modules\juce_gui_extra\documents</Filter>
     </ClInclude>

extras/JuceDemo/Builds/MacOSX/Juce Demo.xcodeproj/project.pbxproj

@@ -634,6 +634,7 @@
 		A58203002289D424A2CDF5DD = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_Application.cpp"; path = "../../../../modules/juce_gui_basics/application/juce_Application.cpp"; sourceTree = "SOURCE_ROOT"; };
 		A588C458AC395D8AC09A433D = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_MarkerList.cpp"; path = "../../../../modules/juce_gui_basics/positioning/juce_MarkerList.cpp"; sourceTree = "SOURCE_ROOT"; };
 		A5A6819C57A0399C81E03301 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_android_Network.cpp"; path = "../../../../modules/juce_core/native/juce_android_Network.cpp"; sourceTree = "SOURCE_ROOT"; };
+		A5EB11F01D92F20648E2CB81 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		A6073115FE5E1E635C78DDFE = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_SpinLock.h"; path = "../../../../modules/juce_core/threads/juce_SpinLock.h"; sourceTree = "SOURCE_ROOT"; };
 		A618A1B49D562ECB86DC46EF = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_OpenGL_android.h"; path = "../../../../modules/juce_opengl/native/juce_OpenGL_android.h"; sourceTree = "SOURCE_ROOT"; };
 		A6541F1A868B78F7E4A53C77 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_OpenGL_osx.h"; path = "../../../../modules/juce_opengl/native/juce_OpenGL_osx.h"; sourceTree = "SOURCE_ROOT"; };
@@ -1856,7 +1857,8 @@
 				441F40BFFD11D433F2FED531,
 				282C1DA3A161FEF2AD6F94A6,
 				F84DBFF75F5334EA86AB2E37,
-				618F07B344089C7B13EED411 ); name = "code_editor"; sourceTree = "<group>"; };
+				618F07B344089C7B13EED411,
+				A5EB11F01D92F20648E2CB81 ); name = "code_editor"; sourceTree = "<group>"; };
 		640482925394FB311FDAB984 = { isa = PBXGroup; children = (
 				D6C9614D1C6493DBEE370646,
 				82AD87FA68D9DDD78F9A4674 ); name = documents; sourceTree = "<group>"; };

extras/JuceDemo/Builds/VisualStudio2005/Juce Demo.vcproj

@@ -4092,6 +4092,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/JuceDemo/Builds/VisualStudio2008/Juce Demo.vcproj

@@ -4092,6 +4092,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/JuceDemo/Builds/VisualStudio2010/Juce Demo.vcxproj

@@ -1593,6 +1593,7 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeEditorComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeTokeniser.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_ActiveXControlComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_NSViewComponent.h"/>

extras/JuceDemo/Builds/VisualStudio2010/Juce Demo.vcxproj.filters

@@ -2700,6 +2700,9 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h">
       <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h">
+      <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h">
       <Filter>Juce Modules\juce_gui_extra\documents</Filter>
     </ClInclude>

extras/JuceDemo/Builds/iOS/Juce Demo.xcodeproj/project.pbxproj

@@ -627,6 +627,7 @@
 		A58203002289D424A2CDF5DD = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_Application.cpp"; path = "../../../../modules/juce_gui_basics/application/juce_Application.cpp"; sourceTree = "SOURCE_ROOT"; };
 		A588C458AC395D8AC09A433D = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_MarkerList.cpp"; path = "../../../../modules/juce_gui_basics/positioning/juce_MarkerList.cpp"; sourceTree = "SOURCE_ROOT"; };
 		A5A6819C57A0399C81E03301 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_android_Network.cpp"; path = "../../../../modules/juce_core/native/juce_android_Network.cpp"; sourceTree = "SOURCE_ROOT"; };
+		A5EB11F01D92F20648E2CB81 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		A6073115FE5E1E635C78DDFE = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_SpinLock.h"; path = "../../../../modules/juce_core/threads/juce_SpinLock.h"; sourceTree = "SOURCE_ROOT"; };
 		A618A1B49D562ECB86DC46EF = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_OpenGL_android.h"; path = "../../../../modules/juce_opengl/native/juce_OpenGL_android.h"; sourceTree = "SOURCE_ROOT"; };
 		A6541F1A868B78F7E4A53C77 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_OpenGL_osx.h"; path = "../../../../modules/juce_opengl/native/juce_OpenGL_osx.h"; sourceTree = "SOURCE_ROOT"; };
@@ -1848,7 +1849,8 @@
 				441F40BFFD11D433F2FED531,
 				282C1DA3A161FEF2AD6F94A6,
 				F84DBFF75F5334EA86AB2E37,
-				618F07B344089C7B13EED411 ); name = "code_editor"; sourceTree = "<group>"; };
+				618F07B344089C7B13EED411,
+				A5EB11F01D92F20648E2CB81 ); name = "code_editor"; sourceTree = "<group>"; };
 		640482925394FB311FDAB984 = { isa = PBXGroup; children = (
 				D6C9614D1C6493DBEE370646,
 				82AD87FA68D9DDD78F9A4674 ); name = documents; sourceTree = "<group>"; };

extras/audio plugin demo/Builds/MacOSX/JuceDemoPlugin.xcodeproj/project.pbxproj

@@ -117,6 +117,7 @@
 		0B53F71A8493EA1807E70057 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ImageComponent.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_ImageComponent.h"; sourceTree = "SOURCE_ROOT"; };
 		0B8D1CD99B98B08B83C16CC6 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ComboBox.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_ComboBox.h"; sourceTree = "SOURCE_ROOT"; };
 		0BEC77B5C0B5DCE51EE24DA7 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_win32_DirectSound.cpp"; path = "../../../../modules/juce_audio_devices/native/juce_win32_DirectSound.cpp"; sourceTree = "SOURCE_ROOT"; };
+		0C4BA1D16AD18E7FD8D242EE = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		0C68BAE8F95098A8AD14F2A0 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_android_OpenSL.cpp"; path = "../../../../modules/juce_audio_devices/native/juce_android_OpenSL.cpp"; sourceTree = "SOURCE_ROOT"; };
 		0D06541766ED7D5D99E403C8 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_DrawableButton.h"; path = "../../../../modules/juce_gui_basics/buttons/juce_DrawableButton.h"; sourceTree = "SOURCE_ROOT"; };
 		0D7BAB3BA700850A464A9F83 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_mac_CoreMidi.cpp"; path = "../../../../modules/juce_audio_devices/native/juce_mac_CoreMidi.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -1817,7 +1818,8 @@
 				5FF425ECF12EC3A8D942CDCB,
 				92EC4C0A72F7312F1FE85B2D,
 				EA339FA929EF8AE42A8CE5BF,
-				39AD51EB5E59758E6FC4F313 ); name = "code_editor"; sourceTree = "<group>"; };
+				39AD51EB5E59758E6FC4F313,
+				0C4BA1D16AD18E7FD8D242EE ); name = "code_editor"; sourceTree = "<group>"; };
 		F430D956469E7E4528571CD8 = { isa = PBXGroup; children = (
 				D6EB2F9DDC938830D6CA84CB,
 				2168ED9FD91B4475612D7785 ); name = documents; sourceTree = "<group>"; };

extras/audio plugin demo/Builds/VisualStudio2005/JuceDemoPlugin.vcproj

@@ -3972,6 +3972,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/audio plugin demo/Builds/VisualStudio2008/JuceDemoPlugin.vcproj

@@ -3972,6 +3972,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/audio plugin host/Builds/MacOSX/Plugin Host.xcodeproj/project.pbxproj

@@ -729,7 +729,6 @@
 		D313CF37B25D7FD313C4F336 = { isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = OpenGL.framework; path = System/Library/Frameworks/OpenGL.framework; sourceTree = SDKROOT; };
 		D314CF2F97529FCF35042263 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ActionBroadcaster.cpp"; path = "../../../../modules/juce_events/broadcasters/juce_ActionBroadcaster.cpp"; sourceTree = "SOURCE_ROOT"; };
 		D3CBC8D7CD32BDA0D37B4155 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_File.h"; path = "../../../../modules/juce_core/files/juce_File.h"; sourceTree = "SOURCE_ROOT"; };
-		D3D7C89A43B6A3591B776B1E = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = "juce_mac_WebBrowserComponent.mm"; path = "../../../../modules/juce_gui_extra/native/juce_mac_WebBrowserComponent.mm"; sourceTree = "SOURCE_ROOT"; };
 		D3EF8617364177AD4D247124 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_win32_WASAPI.cpp"; path = "../../../../modules/juce_audio_devices/native/juce_win32_WASAPI.cpp"; sourceTree = "SOURCE_ROOT"; };
 		D41728E9EB1522CCAC9AC7D4 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CameraDevice.h"; path = "../../../../modules/juce_video/capture/juce_CameraDevice.h"; sourceTree = "SOURCE_ROOT"; };
 		D41EA1E6002DE5DEB9A07CAA = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ComponentPeer.h"; path = "../../../../modules/juce_gui_basics/windows/juce_ComponentPeer.h"; sourceTree = "SOURCE_ROOT"; };
@@ -748,7 +747,6 @@
 		D950776EDF9260DB0278C132 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_SliderPropertyComponent.cpp"; path = "../../../../modules/juce_gui_basics/properties/juce_SliderPropertyComponent.cpp"; sourceTree = "SOURCE_ROOT"; };
 		D9857957A36A3081186D53FC = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_android_CameraDevice.cpp"; path = "../../../../modules/juce_video/native/juce_android_CameraDevice.cpp"; sourceTree = "SOURCE_ROOT"; };
 		DA918320EF4057DF54FF8909 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ActionBroadcaster.h"; path = "../../../../modules/juce_events/broadcasters/juce_ActionBroadcaster.h"; sourceTree = "SOURCE_ROOT"; };
-		DBD90FC88456B62308AD9AD0 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = "juce_events.mm"; path = "../../../../modules/juce_events/juce_events.mm"; sourceTree = "SOURCE_ROOT"; };
 		DC30BDE9C4EE1FD059D06902 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_AudioDataConverters.cpp"; path = "../../../../modules/juce_audio_basics/buffers/juce_AudioDataConverters.cpp"; sourceTree = "SOURCE_ROOT"; };
 		DD8E5D0C88FA2C287F824357 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = GraphEditorPanel.h; path = ../../Source/GraphEditorPanel.h; sourceTree = "SOURCE_ROOT"; };
 		DE0C4CC1CE545BAD009786E4 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_AudioFormatWriter.cpp"; path = "../../../../modules/juce_audio_formats/format/juce_AudioFormatWriter.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -762,6 +760,7 @@
 		CBE270C197A66B22EEE54D9C = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_PluginListComponent.cpp"; path = "../../../../modules/juce_audio_processors/scanning/juce_PluginListComponent.cpp"; sourceTree = "SOURCE_ROOT"; };
 		CF67033CFC21C0060B538042 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ReferenceCountedObject.h"; path = "../../../../modules/juce_core/memory/juce_ReferenceCountedObject.h"; sourceTree = "SOURCE_ROOT"; };
 		D06DA3FA113EAB0CCF8D7A64 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_Singleton.h"; path = "../../../../modules/juce_core/memory/juce_Singleton.h"; sourceTree = "SOURCE_ROOT"; };
+		D3D7C89A43B6A3591B776B1E = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = "juce_mac_WebBrowserComponent.mm"; path = "../../../../modules/juce_gui_extra/native/juce_mac_WebBrowserComponent.mm"; sourceTree = "SOURCE_ROOT"; };
 		D4DA13E7A12FB69C072D0FA7 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ShapeButton.cpp"; path = "../../../../modules/juce_gui_basics/buttons/juce_ShapeButton.cpp"; sourceTree = "SOURCE_ROOT"; };
 		D52F4E0C637B4685217CBEB4 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_Decibels.h"; path = "../../../../modules/juce_audio_basics/effects/juce_Decibels.h"; sourceTree = "SOURCE_ROOT"; };
 		D7433453EBB3700D2805FF42 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_XmlDocument.h"; path = "../../../../modules/juce_core/xml/juce_XmlDocument.h"; sourceTree = "SOURCE_ROOT"; };
@@ -769,6 +768,7 @@
 		DA9E577DC37123CEA41C4A71 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_WaitableEvent.h"; path = "../../../../modules/juce_core/threads/juce_WaitableEvent.h"; sourceTree = "SOURCE_ROOT"; };
 		DAF7C72A4348C33364BB654C = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CallbackMessage.h"; path = "../../../../modules/juce_events/messages/juce_CallbackMessage.h"; sourceTree = "SOURCE_ROOT"; };
 		DB246262CA669F13ABDDB33E = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_InterprocessConnectionServer.cpp"; path = "../../../../modules/juce_events/interprocess/juce_InterprocessConnectionServer.cpp"; sourceTree = "SOURCE_ROOT"; };
+		DBD90FC88456B62308AD9AD0 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = "juce_events.mm"; path = "../../../../modules/juce_events/juce_events.mm"; sourceTree = "SOURCE_ROOT"; };
 		DCACD63912A2F68687ED06CF = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_MemoryInputStream.h"; path = "../../../../modules/juce_core/streams/juce_MemoryInputStream.h"; sourceTree = "SOURCE_ROOT"; };
 		DD53156BA824889FD7DF52D2 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ThreadPool.cpp"; path = "../../../../modules/juce_core/threads/juce_ThreadPool.cpp"; sourceTree = "SOURCE_ROOT"; };
 		DD68D7D44BC0B2CA5DCE8532 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ChangeBroadcaster.h"; path = "../../../../modules/juce_events/broadcasters/juce_ChangeBroadcaster.h"; sourceTree = "SOURCE_ROOT"; };
@@ -801,6 +801,7 @@
 		E29B93C0B9DC9CDD74245891 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_Justification.h"; path = "../../../../modules/juce_graphics/placement/juce_Justification.h"; sourceTree = "SOURCE_ROOT"; };
 		E35BB0E91C10C1441FB5129C = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ApplicationCommandTarget.h"; path = "../../../../modules/juce_gui_basics/commands/juce_ApplicationCommandTarget.h"; sourceTree = "SOURCE_ROOT"; };
 		E3CCF4D578B2D6C84A9A5303 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_PixelFormats.h"; path = "../../../../modules/juce_graphics/colour/juce_PixelFormats.h"; sourceTree = "SOURCE_ROOT"; };
+		E43B7009E2DF76E52C806EFA = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		E47E7E5220482A860A16C572 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_Range.h"; path = "../../../../modules/juce_core/maths/juce_Range.h"; sourceTree = "SOURCE_ROOT"; };
 		E4A23878398A810023FB91B6 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_AudioFormatReaderSource.h"; path = "../../../../modules/juce_audio_formats/format/juce_AudioFormatReaderSource.h"; sourceTree = "SOURCE_ROOT"; };
 		E4C80E4A79A8DBD69192B3DF = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_PreferencesPanel.cpp"; path = "../../../../modules/juce_gui_extra/misc/juce_PreferencesPanel.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -1778,7 +1779,8 @@
 				03E77F398E412903532DB5DA,
 				F38F7E5ADE3BB633A529778A,
 				3E3FCE89FD34F370753351B7,
-				FC99246683913F59070B7966 ); name = "code_editor"; sourceTree = "<group>"; };
+				FC99246683913F59070B7966,
+				E43B7009E2DF76E52C806EFA ); name = "code_editor"; sourceTree = "<group>"; };
 		1585F98304BC113EA3FBA591 = { isa = PBXGroup; children = (
 				C322BCF5FF89EDA223FD1F9F,
 				1959937B4B2FC11B9AC53566 ); name = documents; sourceTree = "<group>"; };

extras/audio plugin host/Builds/VisualStudio2005/Plugin Host.vcproj

@@ -4015,6 +4015,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/audio plugin host/Builds/VisualStudio2008/Plugin Host.vcproj

@@ -4015,6 +4015,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/example projects/Builds/MacOSX/HelloWorld.xcodeproj/project.pbxproj

@@ -309,6 +309,7 @@
 		71EA769D5757AB3B16E5E6E6 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_InterprocessConnectionServer.cpp"; path = "../../../../modules/juce_events/interprocess/juce_InterprocessConnectionServer.cpp"; sourceTree = "SOURCE_ROOT"; };
 		732EB7C9CF8E817B7239DBA1 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_TreeView.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_TreeView.h"; sourceTree = "SOURCE_ROOT"; };
 		7402FA767F4FD9CCD3DD1578 = { isa = PBXFileReference; lastKnownFileType = file; name = "juce_module_info"; path = "../../../../modules/juce_graphics/juce_module_info"; sourceTree = "SOURCE_ROOT"; };
+		744475CFFBA91FDA03DDF659 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		74AB579ADDC7C355C5704A16 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_AppleRemote.h"; path = "../../../../modules/juce_gui_extra/misc/juce_AppleRemote.h"; sourceTree = "SOURCE_ROOT"; };
 		75228523BAA6F029F315E6F1 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ComboBox.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_ComboBox.h"; sourceTree = "SOURCE_ROOT"; };
 		754A3F88F55E0FB7F179F2E3 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ColourSelector.cpp"; path = "../../../../modules/juce_gui_extra/misc/juce_ColourSelector.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -1310,7 +1311,8 @@
 				08C730BD0BD4755B55E99BAC,
 				C7D88B24A0DC64FDD1EE28FD,
 				937C0CF5DC3FFF6161D7776D,
-				7BB92A7F312D9DA6AA53FAC3 ); name = "code_editor"; sourceTree = "<group>"; };
+				7BB92A7F312D9DA6AA53FAC3,
+				744475CFFBA91FDA03DDF659 ); name = "code_editor"; sourceTree = "<group>"; };
 		7A7F0DF75E9B01FE9A9470B2 = { isa = PBXGroup; children = (
 				1667021AB391F5D9B82B186D,
 				2CBD98BBA9322461A3ED0D05 ); name = documents; sourceTree = "<group>"; };

extras/example projects/Builds/VisualStudio2005/HelloWorld.vcproj

@@ -3053,6 +3053,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/example projects/Builds/VisualStudio2008/HelloWorld.vcproj

@@ -3053,6 +3053,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/example projects/Builds/iOS/HelloWorld.xcodeproj/project.pbxproj

@@ -307,6 +307,7 @@
 		71EA769D5757AB3B16E5E6E6 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_InterprocessConnectionServer.cpp"; path = "../../../../modules/juce_events/interprocess/juce_InterprocessConnectionServer.cpp"; sourceTree = "SOURCE_ROOT"; };
 		732EB7C9CF8E817B7239DBA1 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_TreeView.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_TreeView.h"; sourceTree = "SOURCE_ROOT"; };
 		7402FA767F4FD9CCD3DD1578 = { isa = PBXFileReference; lastKnownFileType = file; name = "juce_module_info"; path = "../../../../modules/juce_graphics/juce_module_info"; sourceTree = "SOURCE_ROOT"; };
+		744475CFFBA91FDA03DDF659 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		74AB579ADDC7C355C5704A16 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_AppleRemote.h"; path = "../../../../modules/juce_gui_extra/misc/juce_AppleRemote.h"; sourceTree = "SOURCE_ROOT"; };
 		75228523BAA6F029F315E6F1 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ComboBox.h"; path = "../../../../modules/juce_gui_basics/widgets/juce_ComboBox.h"; sourceTree = "SOURCE_ROOT"; };
 		754A3F88F55E0FB7F179F2E3 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ColourSelector.cpp"; path = "../../../../modules/juce_gui_extra/misc/juce_ColourSelector.cpp"; sourceTree = "SOURCE_ROOT"; };
@@ -1310,7 +1311,8 @@
 				08C730BD0BD4755B55E99BAC,
 				C7D88B24A0DC64FDD1EE28FD,
 				937C0CF5DC3FFF6161D7776D,
-				7BB92A7F312D9DA6AA53FAC3 ); name = "code_editor"; sourceTree = "<group>"; };
+				7BB92A7F312D9DA6AA53FAC3,
+				744475CFFBA91FDA03DDF659 ); name = "code_editor"; sourceTree = "<group>"; };
 		7A7F0DF75E9B01FE9A9470B2 = { isa = PBXGroup; children = (
 				1667021AB391F5D9B82B186D,
 				2CBD98BBA9322461A3ED0D05 ); name = documents; sourceTree = "<group>"; };

extras/static library/Builds/VisualStudio2008/juce.vcproj

@@ -3977,6 +3977,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/static library/Builds/VisualStudio2010/juce.vcxproj

@@ -1561,6 +1561,7 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeEditorComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeTokeniser.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_ActiveXControlComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_NSViewComponent.h"/>

extras/static library/Builds/VisualStudio2010/juce.vcxproj.filters

@@ -2601,6 +2601,9 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h">
       <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h">
+      <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h">
       <Filter>Juce Modules\juce_gui_extra\documents</Filter>
     </ClInclude>

extras/the jucer/Builds/MacOSX/Jucer.xcodeproj/project.pbxproj

@@ -188,6 +188,7 @@
 		2F2AA6FA49DC6B7A0072CBCB = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ColourGradient.cpp"; path = "../../../../modules/juce_graphics/colour/juce_ColourGradient.cpp"; sourceTree = "SOURCE_ROOT"; };
 		2F4E5CDCCD9B5ADEC3088057 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_Random.h"; path = "../../../../modules/juce_core/maths/juce_Random.h"; sourceTree = "SOURCE_ROOT"; };
 		2FC7C92E56BC1FBAB4CB34B4 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "jucer_PaintRoutinePanel.cpp"; path = "../../src/ui/jucer_PaintRoutinePanel.cpp"; sourceTree = "SOURCE_ROOT"; };
+		2FCCDD8C932EB4457DD6918D = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_CPlusPlusCodeTokeniserFunctions.h"; path = "../../../../modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h"; sourceTree = "SOURCE_ROOT"; };
 		30186699B45E27F46CA3DC89 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_ButtonPropertyComponent.cpp"; path = "../../../../modules/juce_gui_basics/properties/juce_ButtonPropertyComponent.cpp"; sourceTree = "SOURCE_ROOT"; };
 		30815C6C668E6F8C5730D339 = { isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = "juce_MACAddress.cpp"; path = "../../../../modules/juce_core/network/juce_MACAddress.cpp"; sourceTree = "SOURCE_ROOT"; };
 		3115152416B738D95BAA7865 = { isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "juce_ActionListener.h"; path = "../../../../modules/juce_events/broadcasters/juce_ActionListener.h"; sourceTree = "SOURCE_ROOT"; };
@@ -1570,7 +1571,8 @@
 				96E86A2DFBF465DADACD626F,
 				FF87A4F25EAFD358975B4133,
 				2A4B8EC4F1F0087DCC7D26FC,
-				05782738F3F2430B66A3CD87 ); name = "code_editor"; sourceTree = "<group>"; };
+				05782738F3F2430B66A3CD87,
+				2FCCDD8C932EB4457DD6918D ); name = "code_editor"; sourceTree = "<group>"; };
 		B8C35D3BF86EF68A2863BE2A = { isa = PBXGroup; children = (
 				F6F724542F5201858B8145DC,
 				EE27FB132BB31A2D365263E3 ); name = documents; sourceTree = "<group>"; };

extras/the jucer/Builds/VisualStudio2008/Jucer.vcproj

@@ -3220,6 +3220,7 @@
             </FileConfiguration>
           </File>
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+          <File RelativePath="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
         </Filter>
         <Filter Name="documents">
           <File RelativePath="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.cpp">

extras/the jucer/Builds/VisualStudio2010/Jucer.vcxproj

@@ -1313,6 +1313,7 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeEditorComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CodeTokeniser.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h"/>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_ActiveXControlComponent.h"/>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\embedding\juce_NSViewComponent.h"/>

extras/the jucer/Builds/VisualStudio2010/Jucer.vcxproj.filters

@@ -2289,6 +2289,9 @@
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniser.h">
       <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\..\modules\juce_gui_extra\code_editor\juce_CPlusPlusCodeTokeniserFunctions.h">
+      <Filter>Juce Modules\juce_gui_extra\code_editor</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\..\modules\juce_gui_extra\documents\juce_FileBasedDocument.h">
       <Filter>Juce Modules\juce_gui_extra\documents</Filter>
     </ClInclude>

modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniser.cpp

@@ -23,355 +23,8 @@
   ==============================================================================
 */
 
-namespace CppTokeniser
-{
-    static bool isIdentifierStart (const juce_wchar c) noexcept
-    {
-        return CharacterFunctions::isLetter (c)
-                || c == '_' || c == '@';
-    }
-
-    static bool isIdentifierBody (const juce_wchar c) noexcept
-    {
-        return CharacterFunctions::isLetterOrDigit (c)
-                || c == '_' || c == '@';
-    }
-
-    static bool isReservedKeyword (String::CharPointerType token, const int tokenLength) noexcept
-    {
-        static const char* const keywords2Char[] =
-            { "if", "do", "or", "id", 0 };
-
-        static const char* const keywords3Char[] =
-            { "for", "int", "new", "try", "xor", "and", "asm", "not", 0 };
-
-        static const char* const keywords4Char[] =
-            { "bool", "void", "this", "true", "long", "else", "char",
-              "enum", "case", "goto", "auto", 0 };
-
-        static const char* const keywords5Char[] =
-            {  "while", "bitor", "break", "catch", "class", "compl", "const", "false",
-                "float", "short", "throw", "union", "using", "or_eq", 0 };
-
-        static const char* const keywords6Char[] =
-            { "return", "struct", "and_eq", "bitand", "delete", "double", "extern",
-              "friend", "inline", "not_eq", "public", "sizeof", "static", "signed",
-              "switch", "typeid", "wchar_t", "xor_eq", 0};
-
-        static const char* const keywords7Char[] =
-            { "default", "mutable", "private", "typedef", "nullptr", "virtual", 0 };
-
-        static const char* const keywordsOther[] =
-            { "noexcept", "const_cast", "continue", "explicit", "namespace",
-              "operator", "protected", "register", "reinterpret_cast", "static_cast",
-              "template", "typename", "unsigned", "volatile", "constexpr",
-              "@implementation", "@interface", "@end", "@synthesize", "@dynamic", "@public",
-              "@private", "@property", "@protected", "@class", 0 };
-
-        const char* const* k;
-
-        switch (tokenLength)
-        {
-            case 2:     k = keywords2Char; break;
-            case 3:     k = keywords3Char; break;
-            case 4:     k = keywords4Char; break;
-            case 5:     k = keywords5Char; break;
-            case 6:     k = keywords6Char; break;
-            case 7:     k = keywords7Char; break;
-
-            default:
-                if (tokenLength < 2 || tokenLength > 16)
-                    return false;
-
-                k = keywordsOther;
-                break;
-        }
-
-        for (int i = 0; k[i] != 0; ++i)
-            if (token.compare (CharPointer_ASCII (k[i])) == 0)
-                return true;
-
-        return false;
-    }
-
-    static int parseIdentifier (CodeDocument::Iterator& source) noexcept
-    {
-        int tokenLength = 0;
-        String::CharPointerType::CharType possibleIdentifier [100];
-        String::CharPointerType possible (possibleIdentifier);
-
-        while (isIdentifierBody (source.peekNextChar()))
-        {
-            const juce_wchar c = source.nextChar();
-
-            if (tokenLength < 20)
-                possible.write (c);
-
-            ++tokenLength;
-        }
-
-        if (tokenLength > 1 && tokenLength <= 16)
-        {
-            possible.writeNull();
-
-            if (isReservedKeyword (String::CharPointerType (possibleIdentifier), tokenLength))
-                return CPlusPlusCodeTokeniser::tokenType_keyword;
-        }
-
-        return CPlusPlusCodeTokeniser::tokenType_identifier;
-    }
-
-    static bool skipNumberSuffix (CodeDocument::Iterator& source)
-    {
-        const juce_wchar c = source.peekNextChar();
-        if (c == 'l' || c == 'L' || c == 'u' || c == 'U')
-            source.skip();
-
-        if (CharacterFunctions::isLetterOrDigit (source.peekNextChar()))
-            return false;
-
-        return true;
-    }
-
-    static bool isHexDigit (const juce_wchar c) noexcept
-    {
-        return (c >= '0' && c <= '9')
-                || (c >= 'a' && c <= 'f')
-                || (c >= 'A' && c <= 'F');
-    }
-
-    static bool parseHexLiteral (CodeDocument::Iterator& source) noexcept
-    {
-        if (source.nextChar() != '0')
-            return false;
-
-        juce_wchar c = source.nextChar();
-        if (c != 'x' && c != 'X')
-            return false;
-
-        int numDigits = 0;
-        while (isHexDigit (source.peekNextChar()))
-        {
-            ++numDigits;
-            source.skip();
-        }
-
-        if (numDigits == 0)
-            return false;
-
-        return skipNumberSuffix (source);
-    }
-
-    static bool isOctalDigit (const juce_wchar c) noexcept
-    {
-        return c >= '0' && c <= '7';
-    }
-
-    static bool parseOctalLiteral (CodeDocument::Iterator& source) noexcept
-    {
-        if (source.nextChar() != '0')
-            return false;
-
-        if (! isOctalDigit (source.nextChar()))
-             return false;
-
-        while (isOctalDigit (source.peekNextChar()))
-            source.skip();
-
-        return skipNumberSuffix (source);
-    }
-
-    static bool isDecimalDigit (const juce_wchar c) noexcept
-    {
-        return c >= '0' && c <= '9';
-    }
-
-    static bool parseDecimalLiteral (CodeDocument::Iterator& source) noexcept
-    {
-        int numChars = 0;
-        while (isDecimalDigit (source.peekNextChar()))
-        {
-            ++numChars;
-            source.skip();
-        }
-
-        if (numChars == 0)
-            return false;
-
-        return skipNumberSuffix (source);
-    }
-
-    static bool parseFloatLiteral (CodeDocument::Iterator& source) noexcept
-    {
-        int numDigits = 0;
-
-        while (isDecimalDigit (source.peekNextChar()))
-        {
-            source.skip();
-            ++numDigits;
-        }
-
-        const bool hasPoint = (source.peekNextChar() == '.');
-
-        if (hasPoint)
-        {
-            source.skip();
-
-            while (isDecimalDigit (source.peekNextChar()))
-            {
-                source.skip();
-                ++numDigits;
-            }
-        }
-
-        if (numDigits == 0)
-            return false;
-
-        juce_wchar c = source.peekNextChar();
-        const bool hasExponent = (c == 'e' || c == 'E');
-
-        if (hasExponent)
-        {
-            source.skip();
-
-            c = source.peekNextChar();
-            if (c == '+' || c == '-')
-                source.skip();
-
-            int numExpDigits = 0;
-            while (isDecimalDigit (source.peekNextChar()))
-            {
-                source.skip();
-                ++numExpDigits;
-            }
+#include "juce_CPlusPlusCodeTokeniserFunctions.h"
 
-            if (numExpDigits == 0)
-                return false;
-        }
-
-        c = source.peekNextChar();
-        if (c == 'f' || c == 'F')
-            source.skip();
-        else if (! (hasExponent || hasPoint))
-            return false;
-
-        return true;
-    }
-
-    static int parseNumber (CodeDocument::Iterator& source)
-    {
-        const CodeDocument::Iterator original (source);
-
-        if (parseFloatLiteral (source))
-            return CPlusPlusCodeTokeniser::tokenType_float;
-
-        source = original;
-
-        if (parseHexLiteral (source))
-            return CPlusPlusCodeTokeniser::tokenType_integer;
-
-        source = original;
-
-        if (parseOctalLiteral (source))
-            return CPlusPlusCodeTokeniser::tokenType_integer;
-
-        source = original;
-
-        if (parseDecimalLiteral (source))
-            return CPlusPlusCodeTokeniser::tokenType_integer;
-
-        source = original;
-        return CPlusPlusCodeTokeniser::tokenType_error;
-    }
-
-    static void skipQuotedString (CodeDocument::Iterator& source) noexcept
-    {
-        const juce_wchar quote = source.nextChar();
-
-        for (;;)
-        {
-            const juce_wchar c = source.nextChar();
-
-            if (c == quote || c == 0)
-                break;
-
-            if (c == '\\')
-                source.skip();
-        }
-    }
-
-    static void skipComment (CodeDocument::Iterator& source) noexcept
-    {
-        bool lastWasStar = false;
-
-        for (;;)
-        {
-            const juce_wchar c = source.nextChar();
-
-            if (c == 0 || (c == '/' && lastWasStar))
-                break;
-
-            lastWasStar = (c == '*');
-        }
-    }
-
-    static void skipPreprocessorLine (CodeDocument::Iterator& source) noexcept
-    {
-        bool lastWasBackslash = false;
-
-        for (;;)
-        {
-            const juce_wchar c = source.peekNextChar();
-
-            if (c == '"')
-            {
-                skipQuotedString (source);
-                continue;
-            }
-
-            if (c == '/')
-            {
-                CodeDocument::Iterator next (source);
-                next.skip();
-                const juce_wchar c2 = next.peekNextChar();
-
-                if (c2 == '/' || c2 == '*')
-                    return;
-            }
-
-            if (c == 0)
-                break;
-
-            if (c == '\n' || c == '\r')
-            {
-                source.skipToEndOfLine();
-
-                if (lastWasBackslash)
-                    skipPreprocessorLine (source);
-
-                break;
-            }
-
-            lastWasBackslash = (c == '\\');
-            source.skip();
-        }
-    }
-
-    static void skipIfNextCharMatches (CodeDocument::Iterator& source, const juce_wchar c) noexcept
-    {
-        if (source.peekNextChar() == c)
-            source.skip();
-    }
-
-    static void skipIfNextCharMatches (CodeDocument::Iterator& source,
-                                       const juce_wchar c1, const juce_wchar c2) noexcept
-    {
-        const juce_wchar c = source.peekNextChar();
-
-        if (c == c1 || c == c2)
-            source.skip();
-    }
-}
 
 //==============================================================================
 CPlusPlusCodeTokeniser::CPlusPlusCodeTokeniser() {}
@@ -379,144 +32,7 @@ CPlusPlusCodeTokeniser::~CPlusPlusCodeTokeniser() {}
 
 int CPlusPlusCodeTokeniser::readNextToken (CodeDocument::Iterator& source)
 {
-    using namespace CppTokeniser;
-    int result = CPlusPlusCodeTokeniser::tokenType_error;
-    source.skipWhitespace();
-
-    const juce_wchar firstChar = source.peekNextChar();
-
-    switch (firstChar)
-    {
-    case 0:
-        source.skip();
-        break;
-
-    case '0':
-    case '1':
-    case '2':
-    case '3':
-    case '4':
-    case '5':
-    case '6':
-    case '7':
-    case '8':
-    case '9':
-    case '.':
-        result = parseNumber (source);
-
-        if (result == CPlusPlusCodeTokeniser::tokenType_error)
-        {
-            source.skip();
-
-            if (firstChar == '.')
-                result = CPlusPlusCodeTokeniser::tokenType_punctuation;
-        }
-
-        break;
-
-    case ',':
-    case ';':
-    case ':':
-        source.skip();
-        result = CPlusPlusCodeTokeniser::tokenType_punctuation;
-        break;
-
-    case '(':
-    case ')':
-    case '{':
-    case '}':
-    case '[':
-    case ']':
-        source.skip();
-        result = CPlusPlusCodeTokeniser::tokenType_bracket;
-        break;
-
-    case '"':
-    case '\'':
-        skipQuotedString (source);
-        result = CPlusPlusCodeTokeniser::tokenType_string;
-        break;
-
-    case '+':
-        result = CPlusPlusCodeTokeniser::tokenType_operator;
-        source.skip();
-        skipIfNextCharMatches (source, '+', '=');
-        break;
-
-    case '-':
-        source.skip();
-        result = parseNumber (source);
-
-        if (result == CPlusPlusCodeTokeniser::tokenType_error)
-        {
-            result = CPlusPlusCodeTokeniser::tokenType_operator;
-            skipIfNextCharMatches (source, '-', '=');
-        }
-        break;
-
-    case '*':
-    case '%':
-    case '=':
-    case '!':
-        result = CPlusPlusCodeTokeniser::tokenType_operator;
-        source.skip();
-        skipIfNextCharMatches (source, '=');
-        break;
-
-    case '/':
-        result = CPlusPlusCodeTokeniser::tokenType_operator;
-        source.skip();
-
-        if (source.peekNextChar() == '=')
-        {
-            source.skip();
-        }
-        else if (source.peekNextChar() == '/')
-        {
-            result = CPlusPlusCodeTokeniser::tokenType_comment;
-            source.skipToEndOfLine();
-        }
-        else if (source.peekNextChar() == '*')
-        {
-            source.skip();
-            result = CPlusPlusCodeTokeniser::tokenType_comment;
-            skipComment (source);
-        }
-
-        break;
-
-    case '?':
-    case '~':
-        source.skip();
-        result = CPlusPlusCodeTokeniser::tokenType_operator;
-        break;
-
-    case '<':
-    case '>':
-    case '|':
-    case '&':
-    case '^':
-        source.skip();
-        result = CPlusPlusCodeTokeniser::tokenType_operator;
-        skipIfNextCharMatches (source, firstChar);
-        skipIfNextCharMatches (source, '=');
-        break;
-
-    case '#':
-        result = CPlusPlusCodeTokeniser::tokenType_preprocessor;
-        skipPreprocessorLine (source);
-        break;
-
-    default:
-        if (isIdentifierStart (firstChar))
-            result = parseIdentifier (source);
-        else
-            source.skip();
-
-        break;
-    }
-
-    return result;
+    return CppTokeniserFunctions::readNextToken (source);
 }
 
 CodeEditorComponent::ColourScheme CPlusPlusCodeTokeniser::getDefaultColourScheme()
@@ -552,5 +68,5 @@ CodeEditorComponent::ColourScheme CPlusPlusCodeTokeniser::getDefaultColourScheme
 
 bool CPlusPlusCodeTokeniser::isReservedKeyword (const String& token) noexcept
 {
-    return CppTokeniser::isReservedKeyword (token.getCharPointer(), token.length());
+    return CppTokeniserFunctions::isReservedKeyword (token.getCharPointer(), token.length());
 }

modules/juce_gui_extra/code_editor/juce_CPlusPlusCodeTokeniserFunctions.h

@@ -0,0 +1,528 @@
+/*
+  ==============================================================================
+
+   This file is part of the JUCE library - "Jules' Utility Class Extensions"
+   Copyright 2004-11 by Raw Material Software Ltd.
+
+  ------------------------------------------------------------------------------
+
+   JUCE can be redistributed and/or modified under the terms of the GNU General
+   Public License (Version 2), as published by the Free Software Foundation.
+   A copy of the license is included in the JUCE distribution, or can be found
+   online at www.gnu.org/licenses.
+
+   JUCE is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+   A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  ------------------------------------------------------------------------------
+
+   To release a closed-source product which uses JUCE, commercial licenses are
+   available: visit www.rawmaterialsoftware.com/juce for more information.
+
+  ==============================================================================
+*/
+
+/** Some basic functions for simple tokenising of C++ code. */
+struct CppTokeniserFunctions
+{
+    static bool isIdentifierStart (const juce_wchar c) noexcept
+    {
+        return CharacterFunctions::isLetter (c)
+                || c == '_' || c == '@';
+    }
+
+    static bool isIdentifierBody (const juce_wchar c) noexcept
+    {
+        return CharacterFunctions::isLetterOrDigit (c)
+                || c == '_' || c == '@';
+    }
+
+    static bool isReservedKeyword (String::CharPointerType token, const int tokenLength) noexcept
+    {
+        static const char* const keywords2Char[] =
+            { "if", "do", "or", "id", 0 };
+
+        static const char* const keywords3Char[] =
+            { "for", "int", "new", "try", "xor", "and", "asm", "not", 0 };
+
+        static const char* const keywords4Char[] =
+            { "bool", "void", "this", "true", "long", "else", "char",
+              "enum", "case", "goto", "auto", 0 };
+
+        static const char* const keywords5Char[] =
+            {  "while", "bitor", "break", "catch", "class", "compl", "const", "false",
+                "float", "short", "throw", "union", "using", "or_eq", 0 };
+
+        static const char* const keywords6Char[] =
+            { "return", "struct", "and_eq", "bitand", "delete", "double", "extern",
+              "friend", "inline", "not_eq", "public", "sizeof", "static", "signed",
+              "switch", "typeid", "wchar_t", "xor_eq", 0};
+
+        static const char* const keywords7Char[] =
+            { "default", "mutable", "private", "typedef", "nullptr", "virtual", 0 };
+
+        static const char* const keywordsOther[] =
+            { "noexcept", "const_cast", "continue", "explicit", "namespace",
+              "operator", "protected", "register", "reinterpret_cast", "static_cast",
+              "template", "typename", "unsigned", "volatile", "constexpr",
+              "@implementation", "@interface", "@end", "@synthesize", "@dynamic", "@public",
+              "@private", "@property", "@protected", "@class", 0 };
+
+        const char* const* k;
+
+        switch (tokenLength)
+        {
+            case 2:     k = keywords2Char; break;
+            case 3:     k = keywords3Char; break;
+            case 4:     k = keywords4Char; break;
+            case 5:     k = keywords5Char; break;
+            case 6:     k = keywords6Char; break;
+            case 7:     k = keywords7Char; break;
+
+            default:
+                if (tokenLength < 2 || tokenLength > 16)
+                    return false;
+
+                k = keywordsOther;
+                break;
+        }
+
+        for (int i = 0; k[i] != 0; ++i)
+            if (token.compare (CharPointer_ASCII (k[i])) == 0)
+                return true;
+
+        return false;
+    }
+
+    template<class Iterator>
+    static int parseIdentifier (Iterator& source) noexcept
+    {
+        int tokenLength = 0;
+        String::CharPointerType::CharType possibleIdentifier [100];
+        String::CharPointerType possible (possibleIdentifier);
+
+        while (isIdentifierBody (source.peekNextChar()))
+        {
+            const juce_wchar c = source.nextChar();
+
+            if (tokenLength < 20)
+                possible.write (c);
+
+            ++tokenLength;
+        }
+
+        if (tokenLength > 1 && tokenLength <= 16)
+        {
+            possible.writeNull();
+
+            if (isReservedKeyword (String::CharPointerType (possibleIdentifier), tokenLength))
+                return CPlusPlusCodeTokeniser::tokenType_keyword;
+        }
+
+        return CPlusPlusCodeTokeniser::tokenType_identifier;
+    }
+
+    template<class Iterator>
+    static bool skipNumberSuffix (Iterator& source)
+    {
+        const juce_wchar c = source.peekNextChar();
+        if (c == 'l' || c == 'L' || c == 'u' || c == 'U')
+            source.skip();
+
+        if (CharacterFunctions::isLetterOrDigit (source.peekNextChar()))
+            return false;
+
+        return true;
+    }
+
+    static bool isHexDigit (const juce_wchar c) noexcept
+    {
+        return (c >= '0' && c <= '9')
+                || (c >= 'a' && c <= 'f')
+                || (c >= 'A' && c <= 'F');
+    }
+
+    template<class Iterator>
+    static bool parseHexLiteral (Iterator& source) noexcept
+    {
+        if (source.nextChar() != '0')
+            return false;
+
+        juce_wchar c = source.nextChar();
+        if (c != 'x' && c != 'X')
+            return false;
+
+        int numDigits = 0;
+        while (isHexDigit (source.peekNextChar()))
+        {
+            ++numDigits;
+            source.skip();
+        }
+
+        if (numDigits == 0)
+            return false;
+
+        return skipNumberSuffix (source);
+    }
+
+    static bool isOctalDigit (const juce_wchar c) noexcept
+    {
+        return c >= '0' && c <= '7';
+    }
+
+    template<class Iterator>
+    static bool parseOctalLiteral (Iterator& source) noexcept
+    {
+        if (source.nextChar() != '0')
+            return false;
+
+        if (! isOctalDigit (source.nextChar()))
+            return false;
+
+        while (isOctalDigit (source.peekNextChar()))
+            source.skip();
+
+        return skipNumberSuffix (source);
+    }
+
+    static bool isDecimalDigit (const juce_wchar c) noexcept
+    {
+        return c >= '0' && c <= '9';
+    }
+
+    template<class Iterator>
+    static bool parseDecimalLiteral (Iterator& source) noexcept
+    {
+        int numChars = 0;
+        while (isDecimalDigit (source.peekNextChar()))
+        {
+            ++numChars;
+            source.skip();
+        }
+
+        if (numChars == 0)
+            return false;
+
+        return skipNumberSuffix (source);
+    }
+
+    template<class Iterator>
+    static bool parseFloatLiteral (Iterator& source) noexcept
+    {
+        int numDigits = 0;
+
+        while (isDecimalDigit (source.peekNextChar()))
+        {
+            source.skip();
+            ++numDigits;
+        }
+
+        const bool hasPoint = (source.peekNextChar() == '.');
+
+        if (hasPoint)
+        {
+            source.skip();
+
+            while (isDecimalDigit (source.peekNextChar()))
+            {
+                source.skip();
+                ++numDigits;
+            }
+        }
+
+        if (numDigits == 0)
+            return false;
+
+        juce_wchar c = source.peekNextChar();
+        const bool hasExponent = (c == 'e' || c == 'E');
+
+        if (hasExponent)
+        {
+            source.skip();
+
+            c = source.peekNextChar();
+            if (c == '+' || c == '-')
+                source.skip();
+
+            int numExpDigits = 0;
+            while (isDecimalDigit (source.peekNextChar()))
+            {
+                source.skip();
+                ++numExpDigits;
+            }
+
+            if (numExpDigits == 0)
+                return false;
+        }
+
+        c = source.peekNextChar();
+        if (c == 'f' || c == 'F')
+            source.skip();
+        else if (! (hasExponent || hasPoint))
+            return false;
+
+        return true;
+    }
+
+    template<class Iterator>
+    static int parseNumber (Iterator& source)
+    {
+        const Iterator original (source);
+
+        if (parseFloatLiteral (source))
+            return CPlusPlusCodeTokeniser::tokenType_float;
+
+        source = original;
+
+        if (parseHexLiteral (source))
+            return CPlusPlusCodeTokeniser::tokenType_integer;
+
+        source = original;
+
+        if (parseOctalLiteral (source))
+            return CPlusPlusCodeTokeniser::tokenType_integer;
+
+        source = original;
+
+        if (parseDecimalLiteral (source))
+            return CPlusPlusCodeTokeniser::tokenType_integer;
+
+        source = original;
+        return CPlusPlusCodeTokeniser::tokenType_error;
+    }
+
+    template<class Iterator>
+    static void skipQuotedString (Iterator& source) noexcept
+    {
+        const juce_wchar quote = source.nextChar();
+
+        for (;;)
+        {
+            const juce_wchar c = source.nextChar();
+
+            if (c == quote || c == 0)
+                break;
+
+            if (c == '\\')
+                source.skip();
+        }
+    }
+
+    template<class Iterator>
+    static void skipComment (Iterator& source) noexcept
+    {
+        bool lastWasStar = false;
+
+        for (;;)
+        {
+            const juce_wchar c = source.nextChar();
+
+            if (c == 0 || (c == '/' && lastWasStar))
+                break;
+
+            lastWasStar = (c == '*');
+        }
+    }
+
+    template<class Iterator>
+    static void skipPreprocessorLine (Iterator& source) noexcept
+    {
+        bool lastWasBackslash = false;
+
+        for (;;)
+        {
+            const juce_wchar c = source.peekNextChar();
+
+            if (c == '"')
+            {
+                skipQuotedString (source);
+                continue;
+            }
+
+            if (c == '/')
+            {
+                Iterator next (source);
+                next.skip();
+                const juce_wchar c2 = next.peekNextChar();
+
+                if (c2 == '/' || c2 == '*')
+                    return;
+            }
+
+            if (c == 0)
+                break;
+
+            if (c == '\n' || c == '\r')
+            {
+                source.skipToEndOfLine();
+
+                if (lastWasBackslash)
+                    skipPreprocessorLine (source);
+
+                break;
+            }
+
+            lastWasBackslash = (c == '\\');
+            source.skip();
+        }
+    }
+
+    template<class Iterator>
+    static void skipIfNextCharMatches (Iterator& source, const juce_wchar c) noexcept
+    {
+        if (source.peekNextChar() == c)
+            source.skip();
+    }
+
+    template<class Iterator>
+    static void skipIfNextCharMatches (Iterator& source, const juce_wchar c1, const juce_wchar c2) noexcept
+    {
+        const juce_wchar c = source.peekNextChar();
+
+        if (c == c1 || c == c2)
+            source.skip();
+    }
+
+    template<class Iterator>
+    static int readNextToken (Iterator& source)
+    {
+        int result = CPlusPlusCodeTokeniser::tokenType_error;
+        source.skipWhitespace();
+
+        const juce_wchar firstChar = source.peekNextChar();
+
+        switch (firstChar)
+        {
+        case 0:
+            source.skip();
+            break;
+
+        case '0':
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9':
+        case '.':
+            result = parseNumber (source);
+
+            if (result == CPlusPlusCodeTokeniser::tokenType_error)
+            {
+                source.skip();
+
+                if (firstChar == '.')
+                    result = CPlusPlusCodeTokeniser::tokenType_punctuation;
+            }
+
+            break;
+
+        case ',':
+        case ';':
+        case ':':
+            source.skip();
+            result = CPlusPlusCodeTokeniser::tokenType_punctuation;
+            break;
+
+        case '(':
+        case ')':
+        case '{':
+        case '}':
+        case '[':
+        case ']':
+            source.skip();
+            result = CPlusPlusCodeTokeniser::tokenType_bracket;
+            break;
+
+        case '"':
+        case '\'':
+            skipQuotedString (source);
+            result = CPlusPlusCodeTokeniser::tokenType_string;
+            break;
+
+        case '+':
+            result = CPlusPlusCodeTokeniser::tokenType_operator;
+            source.skip();
+            skipIfNextCharMatches (source, '+', '=');
+            break;
+
+        case '-':
+            source.skip();
+            result = parseNumber (source);
+
+            if (result == CPlusPlusCodeTokeniser::tokenType_error)
+            {
+                result = CPlusPlusCodeTokeniser::tokenType_operator;
+                skipIfNextCharMatches (source, '-', '=');
+            }
+            break;
+
+        case '*':
+        case '%':
+        case '=':
+        case '!':
+            result = CPlusPlusCodeTokeniser::tokenType_operator;
+            source.skip();
+            skipIfNextCharMatches (source, '=');
+            break;
+
+        case '/':
+            result = CPlusPlusCodeTokeniser::tokenType_operator;
+            source.skip();
+
+            if (source.peekNextChar() == '=')
+            {
+                source.skip();
+            }
+            else if (source.peekNextChar() == '/')
+            {
+                result = CPlusPlusCodeTokeniser::tokenType_comment;
+                source.skipToEndOfLine();
+            }
+            else if (source.peekNextChar() == '*')
+            {
+                source.skip();
+                result = CPlusPlusCodeTokeniser::tokenType_comment;
+                skipComment (source);
+            }
+
+            break;
+
+        case '?':
+        case '~':
+            source.skip();
+            result = CPlusPlusCodeTokeniser::tokenType_operator;
+            break;
+
+        case '<':
+        case '>':
+        case '|':
+        case '&':
+        case '^':
+            source.skip();
+            result = CPlusPlusCodeTokeniser::tokenType_operator;
+            skipIfNextCharMatches (source, firstChar);
+            skipIfNextCharMatches (source, '=');
+            break;
+
+        case '#':
+            result = CPlusPlusCodeTokeniser::tokenType_preprocessor;
+            skipPreprocessorLine (source);
+            break;
+
+        default:
+            if (isIdentifierStart (firstChar))
+                result = parseIdentifier (source);
+            else
+                source.skip();
+
+            break;
+        }
+
+        return result;
+    }
+};