I have a gaming framework with an renderer interface. Those support DX8, DX9 and latest, DX11. Both DX8 and DX9 use fixed function pipeline, while DX11 obviously uses shaders. I've got most of the parts working fine, as in I can switch renderers and notice almost no difference. The most advanced features are 2 directional lights with a single texture 
My last problem is lighting; albeit there's documentation on the D3D lighting model I still can't get the behaviour right. My mistake shows most prominently in the dark side opposite the lights. I'm pretty sure the ambient calculation is off, but that one's supposed to be the most simple one and should be hard to get wrong.
Interestingly I've been searching high and low, and have yet to find a resource that shows how to build a HLSL shader where diffuse, ambient and specular are used together with material properties. I've got various shaders for all the variations I'm supporting. I stepped through the shader with the graphics debugger, but the calculation seems to do what I want. I'm just not sure the formula is correct.
This one should suffice though, it's doing two directional lights, texture modulated with vertex color and a normal. Maybe someone can spot one (or more mistakes). And yes, this is in the vertex shader and I'm aware lighting will be as "bad" as in fixed function; that's my goal currently.
// A constant buffer that stores the three basic column-major matrices for composing geometry.
cbuffer ModelViewProjectionConstantBuffer : register(b0)
{
matrix model;
matrix view;
matrix projection;
matrix ortho2d;
};
struct DirectionLight
{
float3 Direction;
float PaddingL1;
float4 Ambient;
float4 Diffuse;
float4 Specular;
};
cbuffer LightsConstantBuffer : register( b1 )
{
float4 Ambient;
float3 EyePos;
float PaddingLC1;
DirectionLight Light[8];
};
struct Material
{
float4 MaterialEmissive;
float4 MaterialAmbient;
float4 MaterialDiffuse;
float4 MaterialSpecular;
float MaterialSpecularPower;
float3 MaterialPadding;
};
cbuffer MaterialConstantBuffer : register( b2 )
{
Material _Material;
};
// Per-vertex data used as input to the vertex shader.
struct VertexShaderInput
{
float3 pos : POSITION;
float3 normal : NORMAL;
float4 color : COLOR0;
float2 tex : TEXCOORD0;
};
// Per-pixel color data passed through the pixel shader.
struct PixelShaderInput
{
float4 pos : SV_POSITION;
float2 tex : TEXCOORD0;
float4 color : COLOR0;
};
// Simple shader to do vertex processing on the GPU.
PixelShaderInput main(VertexShaderInput input)
{
PixelShaderInput output;
float4 pos = float4( input.pos, 1.0f );
// Transform the vertex position into projected space.
pos = mul(pos, model);
pos = mul(pos, view);
pos = mul(pos, projection);
output.pos = pos;
// pass texture coords
output.tex = input.tex;
// Calculate the normal vector against the world matrix only.
//set required lighting vectors for interpolation
float3 normal = mul( input.normal, ( float3x3 )model );
normal = normalize( normal );
float4 ambientEffect = Ambient;
float4 diffuseEffect = float4( 0, 0, 0, 0 );
float4 specularEffect = float4( 0, 0, 0, 0 );
for ( int i = 0; i < 2; ++i )
{
// Invert the light direction for calculations.
float3 lightDir = -Light[i].Direction;
float lightFactor = max( dot( lightDir, input.normal ), 0 );
ambientEffect += Light[i].Ambient * _Material.MaterialAmbient;
diffuseEffect += saturate( Light[i].Diffuse * dot( normal, lightDir ) );// * _Material.MaterialDiffuse;
//specularEffect += Light.Specular * dot( normal, halfangletolight ) * _Material.MaterialSpecularPower;
}
specularEffect *= _Material.MaterialSpecular;
//ambientEffect.w = 1.0;
ambientEffect = normalize( ambientEffect );
/*
Ambient effect: (L1.ambient + L2.ambient) * object ambient color
Diffuse effect: (L1.diffuse * Dot(VertexNormal, Light1.Direction) + L2.diffuse * Dot(VertexNormal, Light2.Direction)) * object diffuse color
Specular effect: (L1.specular * Dot(VertexNormal, HalfAngleToLight1) * Object specular reflection power + L2.specular * Dot(VertexNormal, HalfAngleToLight2) * Object specular reflection power ) * object specular color
Resulting color = Ambient effect + diffuse effect + specular effect*/
float4 totalFactor = ambientEffect + diffuseEffect + specularEffect;
totalFactor.w = 1.0;
output.color = input.color * totalFactor;
return output;
}
