NFAA (Normal Filter AA)Post-Processing Anti-Aliasing

// Post-Process Anti-Aliasing Filter Demo by rojart 03.11.2010
// http://www.opserver.de/ubb7/ubbthreads.php?ubb=newpost&Board=27
// OBGEv2 port by Dracusis aka Cameron Owen
// Version 1.1 2010/10/29
// Converted to lite-c by rojart
// Based on Styves implimentation at GemeDev.net
// http://www.gamedev.net/community/forums/topic.asp?topic_id=580517

#include <default.c>
#define PRAGMA_PATH "%EXE_DIR%\samples"

var filterStrength = 1.0;
var filterSpread = 3.0;
var debug = 0;
var useColor = 0;
var RenderHalfScreen = 1;

FONT* A20b = "Arial#20b";
VIEW* view_nfaa = { layer=2; flags=PROCESS_TARGET; }
MATERIAL* mtl_nfaa = {effect = "NFAA.fx";}
PANEL* pInfo;

function on_1_event() {while (key_1){wait (1);}	RenderHalfScreen = !RenderHalfScreen;}
function on_2_event() {while (key_2){wait (1);} useColor = !useColor; uu1 = useColor;}
function on_3_event() {while (key_3){wait (1);} debug = !debug; uu2 = debug;}
function on_4_event() {while (key_4){wait (1);} toggle(pInfo, SHOW);}

function main() {
	vec_set(sky_color,vector(255,120,120));
	video_window(NULL,NULL,2,NULL);
	video_set(1280, 720, 32, 2);
	mouse_mode = 4;
	//sun_light = 100;
	level_load ("small.hmp");
	vec_set(camera.x, vector(0,0,50));
	
	pInfo = pan_create(NULL,0);
	pan_setdigits(pInfo,0,1,1,"NFAA - Post-Process Anti-Aliasing Filter", A20b, 0, 0);
	pan_setdigits(pInfo,0,1,30,"Strength = %.1f", A20b, 1, filterStrength);
	pan_setdigits(pInfo,0,1,70,"filSpread = %.1f", A20b, 1, filterSpread);
	pan_setdigits(pInfo,0,1,110,"[1] = Render half of the image Off/On", A20b, 0, 0);
	pan_setdigits(pInfo,0,1,150,"[2] = Use Color On/Off", A20b, 0, 0);
	pan_setdigits(pInfo,0,1,190,"[3] = Debug On/Off", A20b, 0, 0);
	pan_setdigits(pInfo,0,1,230,"[4] = Panel Info Off/On", A20b, 0, 0);
	pan_setslider(pInfo,0,130,30,bmap_fill(bmap_createblack(100,20,32),COLOR_WHITE,50),bmap_fill(bmap_createblack(20,18,32),COLOR_WHITE,90),0.5,1.5,filterStrength);	
	pan_setslider(pInfo,0,130,70,bmap_fill(bmap_createblack(100,20,32),COLOR_WHITE,50),bmap_fill(bmap_createblack(20,18,32),COLOR_WHITE,90),2,5,filterSpread);
	pan_setcolor(pInfo,1,1,COLOR_GREEN);
	pInfo.pos_y = 110;
	set(pInfo, OUTLINE|SHOW);
	
	you = ent_create(SPHERE_MDL, vector(500,0,100), 0);
	vec_fill(you.scale_x, 12);set(you,LIGHT);
	you.material = mat_metal;vec_set(you.blue,vector(0,255,255));

	view_nfaa.material = mtl_nfaa;	
	camera.stage = view_nfaa;

	def_move();
	def_debug();
	
	while(1) {	
		while (RenderHalfScreen) {
			draw_text("NFAA - Disabled!",900,10,COLOR_RED); 
			draw_line(nullvector,NULL,100);
			draw_line(nullvector,COLOR_RED,100);
			def_box(screen_size.x/2,2,screen_size.x-2,screen_size.y-2,COLOR_RED);
			draw_line(nullvector,NULL,100);
			draw_line(nullvector,COLOR_WHITE,100);
			wait(1);
		}
		wait(1);
	}
}

///////////////////////////////////////////////////////////////////////////
////////////////////////// NORMAL Filter AA SHADER ////////////////////////
///////////////////////////////////////////////////////////////////////////

// The Normal Filter Anti Aliasing (NFAA) shader attempts to reduce obvious
// alising by searching for contrasting luminosity changes in the final 
// render image. It then builds a normal displament map to apply a 
// per-pixel blur filter in high contrast alised areas.

// Based on Styves implimentation at GemeDev.net
// http://www.gamedev.net/community/forums/topic.asp?topic_id=580517

// OBGEv2 port by Dracusis aka Cameron Owen
// Version 1.1 2010/10/29
// Converted by rojart

///////////////////////////////////////////////////
// EDIT THE VALUES BELOW TO CUSTOMISE THE FILTER //
///////////////////////////////////////////////////

// Filter Strength Adjusts the overall power of the filter. 
// Values in the range of 0.5 to 1.5 should provide good results without
// blurring the overal image too much. Anything higher will also likely
// cause ugly blocky or spikey artifacts.
// Default Value = 1.0;

float filterStrength_var = 1.0;

// Filter Spread controls how large an area the filter tries to sample
// and fix aliasing within. This has a direct relationship to the angle
// of lines the filter can smooth well. A 45 degree line will be perfectly
// alised with a spread of 2.0, steeper lines will need higher 
// values. The tradeoff for setting a high spread value is the overall 
// softness of the image. Values between 2.0 and 5.0 work best.
// Default Value = 3.0;

float filterSpread_var = 3.0;

// This adjusts whether or not the filter works on pixel color or pixel
// luminance. Using pixel color gives better results with lower 
// contrasting aliasing areas but can over soften the whole scene a bit. 
// Set to false to use pixel luminance method if you're only concerned 
// with high contrast aliasing or if you find the colour version 
// makes everything too blurry.
// Default Value = true;

float useColor_var = 0;

// Set Debug to true to see the normal map used to apply the blur filter.
// Use this with the OBSE.ini bRenderHalfScreen=1 setting to get a feel
// for how the filter works. Default Value = false;

float debug_var = 0;

float RenderHalfScreen_var;

// DO NOT EDIT VALUES PAST THIS POINT

float2   rcpres = .001;
float4x4 matProj;
Texture TargetMap;

sampler FrameSampler = sampler_state {
	texture = <TargetMap>;
	AddressU = CLAMP;
	AddressV = CLAMP;
	MINFILTER = LINEAR;
	MAGFILTER = LINEAR;
	MIPFILTER = LINEAR;
};

// VERTEX SHADER

struct VSOUT{float4 vertPos : POSITION;float2 UVCoord : TEXCOORD0;};
struct VSIN{float4 vertPos : POSITION0;float2 UVCoord : TEXCOORD0;};
VSOUT FrameVS(VSIN IN)
{
	VSOUT OUT = (VSOUT)0.0f;	// initialize to zero, avoid complaints.
	OUT.vertPos = IN.vertPos;
	OUT.UVCoord = IN.UVCoord;
	return OUT;
}

// PIXEL SHADER

// Luminance Conversion 

float GetColorLuminance( float3 i_vColor ) {
	//return dot(i_vColor, float3(1.f, 1.f, 1.f));
	return dot( i_vColor, float3( 0.2126f, 0.7152f, 0.0722f ) );
}

float2 findContrastByLuminance(float2 XYCoord) {
	// Normal offsets, scale by filter spread
	float2 upOffset = float2( 0, rcpres.y ) * filterSpread_var;
	float2 rightOffset = float2( rcpres.x, 0 ) * filterSpread_var;
	float topHeight         = GetColorLuminance( tex2D( FrameSampler, XYCoord + upOffset              ).rgb );
	float bottomHeight      = GetColorLuminance( tex2D( FrameSampler, XYCoord - upOffset              ).rgb );
	float rightHeight       = GetColorLuminance( tex2D( FrameSampler, XYCoord + rightOffset           ).rgb );
	float leftHeight        = GetColorLuminance( tex2D( FrameSampler, XYCoord - rightOffset           ).rgb );
	float leftTopHeight     = GetColorLuminance( tex2D( FrameSampler, XYCoord - rightOffset + upOffset).rgb );
	float leftBottomHeight  = GetColorLuminance( tex2D( FrameSampler, XYCoord - rightOffset - upOffset).rgb );
	float rightBottomHeight = GetColorLuminance( tex2D( FrameSampler, XYCoord + rightOffset + upOffset).rgb );
	float rightTopHeight    = GetColorLuminance( tex2D( FrameSampler, XYCoord + rightOffset - upOffset).rgb );
	// Normal map creation
	float sum0 = rightTopHeight    + bottomHeight + leftTopHeight;
	float sum1 = leftBottomHeight  + topHeight    + rightBottomHeight;
	float sum2 = leftTopHeight     + rightHeight  + leftBottomHeight;
	float sum3 = rightBottomHeight + leftHeight   + rightTopHeight;
	// Subtract the opposite sample set for final vectors
	float vec1 = (sum0 - sum1) * filterStrength_var;
	float vec2 = (sum3 - sum2) * filterStrength_var;
	float2 Vectors = float2(vec1,vec2);
	return  Vectors;
}

float2 findContrastByColor(float2 XYCoord) {
	// Normal offsets, scale by filter spread
	float2 upOffset = float2( 0, rcpres.y ) * filterSpread_var;
	float2 rightOffset = float2( rcpres.x, 0 ) * filterSpread_var;
	float3 topHeight         = tex2D( FrameSampler, XYCoord + upOffset              ).rgb;
	float3 bottomHeight      = tex2D( FrameSampler, XYCoord - upOffset              ).rgb;
	float3 rightHeight       = tex2D( FrameSampler, XYCoord + rightOffset           ).rgb;
	float3 leftHeight        = tex2D( FrameSampler, XYCoord - rightOffset           ).rgb;
	float3 leftTopHeight     = tex2D( FrameSampler, XYCoord - rightOffset + upOffset).rgb;
	float3 leftBottomHeight  = tex2D( FrameSampler, XYCoord - rightOffset - upOffset).rgb;
	float3 rightBottomHeight = tex2D( FrameSampler, XYCoord + rightOffset + upOffset).rgb;
	float3 rightTopHeight    = tex2D( FrameSampler, XYCoord + rightOffset - upOffset).rgb;
	// Normal map creation
	float3 sum0 = rightTopHeight    + bottomHeight + leftTopHeight;
	float3 sum1 = leftBottomHeight  + topHeight    + rightBottomHeight;
	float3 sum2 = leftTopHeight     + rightHeight  + leftBottomHeight;
	float3 sum3 = rightBottomHeight + leftHeight   + rightTopHeight;
	// Subtract the opposite sample set for final vectors
	float vec1 = length(sum0 - sum1) * filterStrength_var;
	float vec2 = length(sum3 - sum2) * filterStrength_var;
	float2 Vectors = float2(vec1,vec2);
	return  Vectors;
}

float4 NormalAAPS(VSOUT IN,float2 Tex : TEXCOORD0) : COLOR0 {
	filterStrength_var += (filterSpread_var/2);
	float2 Vectors;	
	// Find Contrast By Colour
	if (useColor_var) { 
		Vectors = findContrastByColor(IN.UVCoord.xy);			
		// Find Contrast By Luminance
		} else {
		Vectors = findContrastByLuminance(IN.UVCoord.xy);
	}	
	float filterClamp = filterStrength_var/filterSpread_var;
	Vectors.xy = clamp(Vectors, -float2(filterClamp,filterClamp), float2(filterClamp,filterClamp));
	float2 Normal = float2(Vectors.x,Vectors.y) * rcpres;
	//Normal.xy *= 2;
	float4 Scene0 = tex2D( FrameSampler, IN.UVCoord.xy );
	float4 Scene1 = tex2D( FrameSampler, IN.UVCoord.xy + Normal.xy );
	float4 Scene2 = tex2D( FrameSampler, IN.UVCoord.xy - Normal.xy );
	float4 Scene3 = tex2D( FrameSampler, IN.UVCoord.xy + float2(Normal.x, -Normal.y) );
	float4 Scene4 = tex2D( FrameSampler, IN.UVCoord.xy - float2(Normal.x, -Normal.y) );
	// Final color
	float4 o_Color = (Scene0 + Scene1 + Scene2 + Scene3 + Scene4) * 0.2;
	// Debug Output
	if (debug_var) {
		o_Color .xyz = normalize(float3(Vectors.x, Vectors.y, 1) * 0.1 + 0.1);
	}
	// postprocess only the half of the image
	if (RenderHalfScreen_var) {
		if (Tex.x > 0.5) o_Color = tex2D( FrameSampler, Tex.xy);
	}
	return o_Color;
}

// RENDER PASSES

technique t0 {
	pass p0 {
		AlphaBlendEnable = false;
		CullMode = None;
		VertexShader = compile vs_3_0 FrameVS();
		PixelShader = compile ps_3_0 NormalAAPS();
	}
}