setStyleMask isn't supported on Mac OS X 10.4, so the window will need to be taken down and recreated and the content view moved to the new window in that case.

See this for an example:
https://svn.blender.org/svnroot/bf-blender/trunk/blender/intern/ghost/intern/GHOST_WindowCocoa.mm

/**
 * @note Fullscreen switch is not actual fullscreen with display capture. As this capture removes all OS X window manager features.
 * Instead, the menu bar and the dock are hidden, and the window is made borderless and enlarged.
 * Thus, process switch, exposé, spaces, ... still work in fullscreen mode
 */
GHOST_TSuccess GHOST_WindowCocoa::setState(GHOST_TWindowState state)
{
	GHOST_ASSERT(getValid(), "GHOST_WindowCocoa::setState(): window invalid")
    switch (state) {
		case GHOST_kWindowStateMinimized:
            [m_window miniaturize:nil];
            break;
		case GHOST_kWindowStateMaximized:
			[m_window zoom:nil];
			break;
		
		case GHOST_kWindowStateFullScreen:
			if (!m_fullScreen)
			{
				NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
			
				//This status change needs to be done before Cocoa call to enter fullscreen mode
				//to give window delegate hint not to forward its deactivation to ghost wm that doesn't know view/window difference
				m_fullScreen = true;

#ifdef MAC_OS_X_VERSION_10_6
				//10.6 provides Cocoa functions to autoshow menu bar, and to change a window style
				//Hide menu & dock if needed
				if ([[m_window screen] isEqual:[[NSScreen screens] objectAtIndex:0]])
				{
					[NSApp setPresentationOptions:(NSApplicationPresentationHideDock | NSApplicationPresentationAutoHideMenuBar)];
				}
				//Make window borderless and enlarge it
				[m_window setStyleMask:NSBorderlessWindowMask];
				[m_window setFrame:[[m_window screen] frame] display:YES];
				[m_window makeFirstResponder:m_openGLView];
#else
				//With 10.5, we need to create a new window to change its style to borderless
				//Hide menu & dock if needed
				if ([[m_window screen] isEqual:[[NSScreen screens] objectAtIndex:0]])
				{
					//Cocoa function in 10.5 does not allow to set the menu bar in auto-show mode [NSMenu setMenuBarVisible:NO];
					//One of the very few 64bit compatible Carbon function
					SetSystemUIMode(kUIModeAllHidden,kUIOptionAutoShowMenuBar);
				}
				//Create a fullscreen borderless window
				CocoaWindow *tmpWindow = [[CocoaWindow alloc]
										  initWithContentRect:[[m_window screen] frame]
										  styleMask:NSBorderlessWindowMask
										  backing:NSBackingStoreBuffered
										  defer:YES];
				//Copy current window parameters
				[tmpWindow setTitle:[m_window title]];
				[tmpWindow setRepresentedFilename:[m_window representedFilename]];
				[tmpWindow setReleasedWhenClosed:NO];
				[tmpWindow setAcceptsMouseMovedEvents:YES];
				[tmpWindow setDelegate:[m_window delegate]];
				[tmpWindow setSystemAndWindowCocoa:[m_window systemCocoa] windowCocoa:this];
				[tmpWindow registerForDraggedTypes:[NSArray arrayWithObjects:NSFilenamesPboardType,
												   NSStringPboardType, NSTIFFPboardType, nil]];
				
				//Assign the openGL view to the new window
				[tmpWindow setContentView:m_openGLView];
				
				//Show the new window
				[tmpWindow makeKeyAndOrderFront:m_openGLView];
				//Close and release old window
				[m_window setDelegate:nil]; // To avoid the notification of "window closed" event
				[m_window close];
				[m_window release];
				m_window = tmpWindow;
#endif
			
				//Tell WM of view new size
				m_systemCocoa->handleWindowEvent(GHOST_kEventWindowSize, this);
				
				[pool drain];
				}
			break;
		case GHOST_kWindowStateNormal:
        default:
			NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
			if (m_fullScreen)
			{
				m_fullScreen = false;

				//Exit fullscreen
#ifdef MAC_OS_X_VERSION_10_6
				//Show again menu & dock if needed
				if ([[m_window screen] isEqual:[NSScreen mainScreen]])
				{
					[NSApp setPresentationOptions:NSApplicationPresentationDefault];
				}
				//Make window normal and resize it
				[m_window setStyleMask:(NSTitledWindowMask | NSClosableWindowMask | NSMiniaturizableWindowMask | NSResizableWindowMask)];
				[m_window setFrame:[[m_window screen] visibleFrame] display:YES];
				//TODO for 10.6 only : window title is forgotten after the style change
				[m_window makeFirstResponder:m_openGLView];
#else
				//With 10.5, we need to create a new window to change its style to borderless
				//Show menu & dock if needed
				if ([[m_window screen] isEqual:[NSScreen mainScreen]])
				{
					//Cocoa function in 10.5 does not allow to set the menu bar in auto-show mode [NSMenu setMenuBarVisible:YES];
					SetSystemUIMode(kUIModeNormal, 0); //One of the very few 64bit compatible Carbon function
				}
				//Create a fullscreen borderless window
				CocoaWindow *tmpWindow = [[CocoaWindow alloc]
										  initWithContentRect:[[m_window screen] frame]
													styleMask:(NSTitledWindowMask | NSClosableWindowMask | NSMiniaturizableWindowMask | NSResizableWindowMask)
													  backing:NSBackingStoreBuffered
														defer:YES];
				//Copy current window parameters
				[tmpWindow setTitle:[m_window title]];
				[tmpWindow setRepresentedFilename:[m_window representedFilename]];
				[tmpWindow setReleasedWhenClosed:NO];
				[tmpWindow setAcceptsMouseMovedEvents:YES];
				[tmpWindow setDelegate:[m_window delegate]];
				[tmpWindow setSystemAndWindowCocoa:[m_window systemCocoa] windowCocoa:this];
				[tmpWindow registerForDraggedTypes:[NSArray arrayWithObjects:NSFilenamesPboardType,
												   NSStringPboardType, NSTIFFPboardType, nil]];
				//Forbid to resize the window below the blender defined minimum one
				[tmpWindow setContentMinSize:NSMakeSize(320, 240)];
				
				//Assign the openGL view to the new window
				[tmpWindow setContentView:m_openGLView];
				
				//Show the new window
				[tmpWindow makeKeyAndOrderFront:nil];
				//Close and release old window
				[m_window setDelegate:nil]; // To avoid the notification of "window closed" event
				[m_window close];
				[m_window release];
				m_window = tmpWindow;
#endif
			
				//Tell WM of view new size
				m_systemCocoa->handleWindowEvent(GHOST_kEventWindowSize, this);
			}
            else if ([m_window isMiniaturized])
				[m_window deminiaturize:nil];
			else if ([m_window isZoomed])
				[m_window zoom:nil];
			[pool drain];
            break;
    }

    return GHOST_kSuccess;
}