Like I said, I really don't know. I just saved all render states, reset, apply librocket states (That's what I called naive approach and slow). Also, that was exactly 1 year ago blush . Bellow is the C++ code I used to test the lib (Before creating a plugin, I test it using Win App) if that can help you (I think that only the states that are used by librocket and 3dgs need to be changed).

PS : As I said, it is only a dirty code I used to test if a 3dgs plugin can be created using librocket wink .

Code:
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <d3d9.h>
#include <d3dx9.h>
#include <string>

#include <var.h>
#include <adll.h>

#include <Rocket\Core.h>
#include <Rocket\Controls.h>
#include <Rocket\Debugger.h>

#include "RenderInterfaceDirectX.h"
#include "Input.h"
#include "InputWin32.h"

// Engine vars
ENGINE_VARS *ev = NULL;

LPDIRECT3DDEVICE9 g_pd3dDevice = NULL;

Rocket::Core::Context *context = NULL;             

IDirect3DStateBlock9* pStateBlock = NULL;
IDirect3DStateBlock9* pStateBlock2 = NULL;

// System Interface
class GSSystemInterface : public Rocket::Core::SystemInterface
{
public:
	virtual float GetElapsedTime(); 
};

float GSSystemInterface::GetElapsedTime()
{
	return v(total_ticks) * 0.16;
}

// Load default fonts
void LoadFonts()
{
	Rocket::Core::String font_names[5];
	font_names[0] = "Delicious-Roman.otf";
	font_names[1] = "Delicious-Italic.otf";
	font_names[2] = "Delicious-Bold.otf";
	font_names[3] = "Delicious-BoldItalic.otf";
	font_names[4] = "Comica BD.ttf";

	for (int i = 0; i < sizeof(font_names) / sizeof(Rocket::Core::String); i++)
	{
		Rocket::Core::FontDatabase::LoadFontFace(Rocket::Core::String("assets/") + font_names[i]);
	}
}

// Set renderStates for rocket
void SetRenderStates()
{
	// Get screen size
	VECTOR screen_size;
	vec_set(&screen_size, &v(screen_size));
	
	// Save State block before
	// Save current stateblock
	g_pd3dDevice->CreateStateBlock( D3DSBT_ALL, &pStateBlock2);
	//Restore state block
	pStateBlock->Apply();
	
	// Set up an orthographic projection.
	D3DXMATRIX projection;
	D3DXMatrixOrthoOffCenterLH(&projection, 0, screen_size.x, screen_size.y, 0, -1, 1);
	g_pd3dDevice->SetTransform(D3DTS_PROJECTION, &projection);

	// Switch to clockwise culling instead of counter-clockwise culling; Rocket generates counter-clockwise geometry,
	// so you can either reverse the culling mode when Rocket is rendering, or reverse the indices in the render
	// interface.
	//g_pd3dDevice->SetRenderState(D3DRS_CULLMODE, D3DCULL_CW);

	// Enable alpha-blending for Rocket.
	g_pd3dDevice->SetRenderState(D3DRS_ALPHABLENDENABLE, TRUE);
	g_pd3dDevice->SetRenderState(D3DRS_SRCBLEND, D3DBLEND_BOTHSRCALPHA);
	g_pd3dDevice->SetRenderState(D3DRS_DESTBLEND, D3DBLEND_BOTHINVSRCALPHA);

	// Set up the texture stage states for the diffuse texture.
	g_pd3dDevice->SetTextureStageState(0, D3DTSS_COLOROP, D3DTOP_MODULATE);
	g_pd3dDevice->SetTextureStageState(0, D3DTSS_COLORARG1, D3DTA_TEXTURE);
	g_pd3dDevice->SetTextureStageState(0, D3DTSS_COLORARG2, D3DTA_DIFFUSE);
	g_pd3dDevice->SetTextureStageState(0, D3DTSS_ALPHAOP, D3DTOP_SELECTARG1);
	g_pd3dDevice->SetTextureStageState(0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE);

	g_pd3dDevice->SetSamplerState(0, D3DSAMP_ADDRESSU, D3DTADDRESS_CLAMP);
	g_pd3dDevice->SetSamplerState(0, D3DSAMP_ADDRESSV, D3DTADDRESS_CLAMP);

	// Disable lighting for Rocket.
	g_pd3dDevice->SetRenderState(D3DRS_LIGHTING, FALSE);

	//g_pd3dDevice->SetRenderState(D3DRS_FILLMODE, 2);
}

// Event Handlers Declaration
// Store Original message handler
WNDPROC OriginalHandler = NULL;
// Create a neww handler for rocket
LRESULT CALLBACK AckRocketEventHandler(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam)
{
	// Process Input
	InputWin32::ProcessWindowsEvent(msg, wParam, lParam);
	// Call OriginalHandler
	return OriginalHandler(hWnd, msg, wParam, lParam);
}

VOID WINAPI ColorFill (D3DXVECTOR4* pOut, const D3DXVECTOR2* pTexCoord, 
const D3DXVECTOR2* pTexelSize, LPVOID pData)
{
    //*pOut = D3DXVECTOR4(pTexCoord->x, pTexCoord->y, pTexCoord->x + pTexCoord->y, 1.0f);
	*pOut = D3DXVECTOR4(pTexCoord->x, pTexCoord->y, pTexCoord->x + pTexCoord->y, 1.0f);
}

// WinMain
INT WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCMd)
{
	// Open engine
	ev = engine_open("");

	// Create Rendering engine
	engine_frame();
	
	//level_load(NULL);
	vec_set((VECTOR *)&v(screen_color), vector(255, 0, 0));
	//level_load(NULL);
	//v(camera).size_x = 100;
	//v(camera).size_y = 100;
	
	// Get DirectX interface
	g_pd3dDevice = (LPDIRECT3DDEVICE9)draw_begin();
	if (g_pd3dDevice == NULL)
	{
		return -1;
	}
	// Save current stateblock
	g_pd3dDevice->CreateStateBlock( D3DSBT_ALL, &pStateBlock );

	// Test level load
	v(logo) = 1;
	//level_load(NULL);
	//ENTITY *ent = ent_create("CUBE.MDL", vector(0, 0, 0), NULL);
	//ent->x += 100;
	//v(fps_max) = 60;
	// Display panel
	PANEL *pan = pan_create("", 0);
	pan->flags = SHOW;
	BMAP *bmp = bmap_createblack((v(screen_size).x * 0.5f), (v(screen_size).y * 0.5f), 32);
	pan->bmap = bmp;
	bmap_fill(bmp, (COLOR *)vector(0, 0, 255), 100);

	level_load("");

	var frames = 0;
	var fps = 0;
	pan_setdigits(pan, 0, 10, 200, "Variable acknex = %5.0f", NULL, 16, &fps);

	// Initialize librocket
	GSSystemInterface sysInterface;
	Rocket::Core::SetSystemInterface(&sysInterface);

	RenderInterfaceDirectX directx_renderer((LPDIRECT3D9)NULL, g_pd3dDevice);
	Rocket::Core::SetRenderInterface(&directx_renderer);
	
	// Init Rocket
	Rocket::Core::Initialise();
	// Init controls
	Rocket::Controls::Initialise();

	// Initialize keymap
	InputWin32::Initialise();
	// Bind event handler
	OriginalHandler = (WNDPROC) *ev->on_message;
	*ev->on_message = (EVENT)AckRocketEventHandler;

	// define a suited vertex struct
	typedef struct VERTEX_FLAT { 
		float x,y,z; 
		float rhw; 
		D3DCOLOR color; 
	} VERTEX_FLAT;
	#define D3DFVF_FLAT (D3DFVF_XYZRHW | D3DFVF_DIFFUSE)

	// define the three corner vertices
   VERTEX_FLAT v[3];
   v[0].x = 10.0; v[0].y = 10.0; v[0].color = 0xFFFF00FF; // the red corner
   v[1].x = 310.0; v[1].y = 10.0; v[1].color = 0xFF0000FF; // the blue corner
   v[2].x = 10.0; v[2].y = 310.0; v[2].color = 0xFF00FFFF; // the green corner
   v[0].z = v[1].z = v[2].z = 0.0; // z buffer - paint over everything
   v[0].rhw = v[1].rhw = v[2].rhw = 1.0; // no perspective

	// Engine loop
	while(engine_frame())
	{
		g_pd3dDevice = (LPDIRECT3DDEVICE9)draw_begin();
		SetRenderStates();

		fps = 0.9*fps + 0.1 / v(time_frame); 

		//ent->pan += 5 * v(time_step);

		if (context != NULL)
		{
			context->Update();
			context->Render();

			// Restore stateBlock
			pStateBlock2->Apply();
		}
		else
		{
			// Initialize context
			// CReate the main Rocket context and set it to the input layer
			VECTOR screen_size;
			vec_set(&screen_size, &v(screen_size));
			//(int)screen_size.x, (int)screen_size.y)
			context = Rocket::Core::CreateContext("main", Rocket::Core::Vector2i(screen_size.x, screen_size.y));
			
			/*TCHAR buff[255];
			wsprintf(buff, TEXT("Sizes = %d, %d"), context->GetDimensions().x, context->GetDimensions().y);
			MessageBox(NULL, buff, TEXT("Screen sizes"), MB_OK);*/

			Rocket::Debugger::Initialise(context);
			Rocket::Debugger::SetVisible(true);
			Input::SetContext(context);

			// Load Core fonts
			LoadFonts();
			// Load and show the tutorial document.
			Rocket::Core::ElementDocument* document = context->LoadDocument("hassets/window.rml");
			if (document != NULL)
			{
				document->Show();
				document->RemoveReference();
			}
		}
	}

	// Close acknex engine
	engine_close();

	// Return success
	return 0;
}