I'm trying to implement that shader using GLSL. The problem is I get very strange results.

I'm not sure If I compute the stuff correctly.

Here is the ogre material

material Chassis 
{
    technique
    {

    pass standard
    {
    cull_software back

    scene_blend zero one
    }
        pass psssm
        {


      cull_software front 
      scene_blend src_alpha one_minus_src_alpha

            vertex_program_ref reflection_cube_specularmap_normalmap_vs100 
            {
                param_named_auto modelViewProjectionMatrix worldviewproj_matrix
                param_named_auto normalMatrix inverse_transpose_world_matrix
                param_named_auto modelView worldview_matrix
                param_named_auto camera_world_position camera_position
                param_named_auto inverse_projection_matrix inverse_projection_matrix
        param_named_auto  projection_matrix projection_matrix
        param_named_auto  p_InverseModelView inverse_worldview_matrix
            }
            fragment_program_ref reflection_cube_specularmap_normalmap_fs100
            {

            }

        }
    }
}

Here is the vertex shader

#version 140
#define lowp
#define mediump
#define highp

in vec4 vertex;
in vec3 normal;


uniform mat4 normalMatrix;
uniform mat4 modelViewProjectionMatrix;
uniform mat4 modelView;
uniform vec3 camera_world_position;
uniform mat4 projection_matrix;
uniform mat4 inverse_projection_matrix;
void main()
{       
     vec4 pos = modelViewProjectionMatrix * vertex;
   mat4 modelView = inverse_projection_matrix * modelViewProjectionMatrix;

   vec4 norm   =   inverse(transpose(modelView)) * vec4(normal, 0.0);
   vec2 offset =   vec2( norm.x * projection_matrix[0][0], norm.y * projection_matrix[1][1] );

     pos.xy += offset * pos.z * 0.18;
     gl_Position = pos;
} 

This is cool times ten.

When I was doing it, the research was vague and hard to get answers. Edge detection attempts were a failure. I ended up doing it in a two pass fashion. On the first, my shader pushed the geometry toward the camera which had the effect of enlarging the selected geometry. The fragment shader simply rendered with the selection color I was using with depth write turned off. Then on the second pass the item was rendered again as normal. The effect looked like this.

The problem with this approach was perspective. The vertex shader required a bias to maintain a one pixel silhouette for object distance less than ~24 units relative to the field of view, etc. being used, that I applied to the vertex position before applying projection.

edit: I see you're doing something similar but instead pushing geometry along the normal. If I remember correctly, I got the same exploding effect and ditched the approach.

Can you post your shader please ?

I don't know what I'm doing wrong that I get that exploding effect. 

I just copied that shader from unity, and it works very well in Unity.. 

Would you also explain the math behind the shader that I have failed to do ? I don't get it.

probably won't help much. I try to leave myself reference links close to where I use them as code comments. I just checked and it looks like the page I was linking has faded from existence. Looks like I left my attempt of using the normal, but commented out. If it helps, I'd be surprised. 

pulled from an older monogame project.

edit: no, I take that back...the commented out block has nothing to do with adjusting position along normal. sorry.

// reference : http://drwalton.github.io/2013/10/18/silhouette-rendering.html

#if OPENGL
	#define SV_POSITION POSITION
	#define VS_SHADERMODEL vs_3_0
	#define PS_SHADERMODEL ps_3_0
#else
	#define VS_SHADERMODEL vs_4_0_level_9_1
	#define PS_SHADERMODEL ps_4_0_level_9_1
#endif

matrix WorldViewProjection;
float3 ObjDirection;
float  ObjDistance;

struct VertexShaderInput
{
	float4 Position : SV_POSITION;
};

struct VertexShaderOutput
{
	float4 Position : SV_POSITION;
};


VertexShaderOutput OutlineVS(in VertexShaderInput input)
{
	VertexShaderOutput output = (VertexShaderOutput)0;

	// zBias method 
	// scale outline offset by distance
	// just do it in hard steps...smoothed looked worse
	float zBias;
	if (ObjDistance <= 24)
	{
		if (ObjDistance < 6)              // 0 --> 6 = 64
			zBias = ObjDistance / 64;
		else if (ObjDistance < 12)        // 6 --> 12 = 32  (objDist - 6) = 0.0001-->6 : 64 --> 32 (factor of 5.333)
			zBias = ObjDistance / 32;     // (64 - ((ObjDistance - 6) * 2)); // 32;
		else                              // 12 --> 24 = 16 (objDist - 12) = 0.0001-->12 : 32 --> 16 (factor of 1.333)
			zBias = ObjDistance / 16;     // (32 - ((ObjDistance - 12) * 0.2)); // 16;
	}
	else                                  // > 32 = 10
		zBias = ObjDistance / 10;
	
	// push vertex toward the camera 
	float4 Pos = input.Position - (float4(ObjDirection, 0) * zBias);
	output.Position = mul(Pos, WorldViewProjection);

	return output;
}

float4 OutlinePS(VertexShaderOutput input) : COLOR
{
	return float4(1, 0.761, 0.5, 0.3);
}


/*
VertexShaderOutput FillVS(in VertexShaderInput input)
{
	VertexShaderOutput output = (VertexShaderOutput)0;
	output.Position = mul(input.Position, WorldViewProjection);
	output.Normal = input.Normal;
	return output;
}

float4 FillPS(VertexShaderOutput input) : COLOR
{
	float4 ddir = float4(ObjDirection, 0);
	//float3 OdotN = dot(-ObjDirection, input.Normal.xyz);
	//float i = 1;//saturate(OdotN);
	float4 diffuse = saturate(dot(-ddir, input.Normal));
	return diffuse;

	//return diffuse;// float4(0, 0, 0, 1);
}
*/

technique OutlineMesh
{
	pass P0
	{
		VertexShader = compile VS_SHADERMODEL OutlineVS();
		PixelShader = compile PS_SHADERMODEL OutlinePS();
	}
	/*pass P1
	{
		VertexShader = compile VS_SHADERMODEL FillVS();
		PixelShader = compile PS_SHADERMODEL FillPS();
	}*/
};

Thinking about this more, a possible fix for the exploding mesh would be to have averaged normals pre computed at connected geometry. This may reduce or remove the separating faces when the silhouette is being established. It's a shame to have an extra buffer for just an editor feature, but it's the best I can come up with other than what I ended up doing with pushing geometry towards the camera.

edit: Looking at your shader, maybe try doing your offset in model space then apply the viewProjection.