Im doing post processing which involves drawing on quads. Im doing distortion. Like a haze from a fire. 

First, draw a distortion map to a quad.

Next draw a normal scene to a quad.

Then composite them together in the shader.

Code is below the picture to see what im talking about.

As in the picture, I turned the box so it would be in front of the haze. Shouldn't see the haze. but it is still poking through. How should I solve this problem?

//CUT OUT MASK

        md3dImmediateContext->OMSetRenderTargets(1, &first.mRenderTargetView, first.GetDSV());
	md3dImmediateContext->ClearRenderTargetView( first.GetRTV(), reinterpret_cast<const float*>(&Colors::Black));
	
        md3dImmediateContext->ClearDepthStencilView(first.GetDSV(), D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);

	md3dImmediateContext->IASetInputLayout(InputLayouts::Basic32);
        md3dImmediateContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
 
	UINT stride = sizeof(Vertex::Basic32);
        UINT offset = 0;
 
	XMMATRIX view  = XMLoadFloat4x4(&mView);
	XMMATRIX proj  = XMLoadFloat4x4(&mProj);
	XMMATRIX viewProj = view*proj;

	// Set per frame constants.
	Effects::BasicFX->SetDirLights(mDirLights);
	Effects::BasicFX->SetEyePosW(mEyePosW);
 
	ID3DX11EffectTechnique* activeTech = Effects::DistortionFX->cutout;

    D3DX11_TECHNIQUE_DESC techDesc;
    activeTech->GetDesc( &techDesc );
    for(UINT p = 0; p < techDesc.Passes; ++p)
    {
		md3dImmediateContext->IASetVertexBuffers(0, 1, &mBoxVB, &stride, &offset);
		md3dImmediateContext->IASetIndexBuffer(mBoxIB, DXGI_FORMAT_R32_UINT, 0);

		// Draw the box.
		XMMATRIX world = XMLoadFloat4x4(&mBoxWorld);
		XMMATRIX worldInvTranspose = MathHelper::InverseTranspose(world);
		XMMATRIX worldViewProj = world*view*proj;

		Effects::DistortionFX->SetWorldViewProj(worldViewProj);
		Effects::DistortionFX->SetDiffuseMap2(mDiffuseMapSRV2);

		activeTech->GetPassByIndex(p)->Apply(0, md3dImmediateContext);
		md3dImmediateContext->DrawIndexed(mBoxIndexCount, mBoxIndexOffset, mBoxVertexOffset);
    }
	md3dImmediateContext->OMSetRenderTargets(1, &mRenderTargetView, mDepthStencilView);
	md3dImmediateContext->ClearRenderTargetView( mRenderTargetView, reinterpret_cast<const float*>(&Colors::Black));
   
	md3dImmediateContext->ClearDepthStencilView(mDepthStencilView, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);
	



	md3dImmediateContext->OMSetRenderTargets(1, &second.mRenderTargetView, second.GetDSV());
	md3dImmediateContext->ClearRenderTargetView( second.GetRTV(), reinterpret_cast<const float*>(&Colors::LightSteelBlue));

	md3dImmediateContext->ClearDepthStencilView(second.GetDSV(), D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);

//NORMAL SCENE TO QUAD


	md3dImmediateContext->IASetInputLayout(InputLayouts::Basic32);
        md3dImmediateContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
 
	 stride = sizeof(Vertex::Basic32);
     offset = 0;
 
	 view  = XMLoadFloat4x4(&mView);
	 proj  = XMLoadFloat4x4(&mProj);
	 viewProj = view*proj;

	// Set per frame constants.
	Effects::BasicFX->SetDirLights(mDirLights);
	Effects::BasicFX->SetEyePosW(mEyePosW);
 
	 activeTech = Effects::BasicFX->Light2TexTech;

    
	activeTech->GetDesc( &techDesc );
    for(UINT p = 0; p < techDesc.Passes; ++p)
    {
		md3dImmediateContext->IASetVertexBuffers(0, 1, &mBoxVB2, &stride, &offset);
		md3dImmediateContext->IASetIndexBuffer(mBoxIB2, DXGI_FORMAT_R32_UINT, 0);

		// Draw the box.
		XMMATRIX world = XMLoadFloat4x4(&mBoxWorld);
		world *= XMMatrixTranslation(-2,0,0);
		XMMATRIX worldInvTranspose = MathHelper::InverseTranspose(world);
		XMMATRIX worldViewProj = world*view*proj;

		Effects::BasicFX->SetWorld(world);
		Effects::BasicFX->SetWorldInvTranspose(worldInvTranspose);
		Effects::BasicFX->SetWorldViewProj(worldViewProj);
		Effects::BasicFX->SetTexTransform(XMLoadFloat4x4(&mTexTransform));
		Effects::BasicFX->SetMaterial(mBoxMat);
		Effects::BasicFX->SetDiffuseMap(mDiffuseMapSRV);

		activeTech->GetPassByIndex(p)->Apply(0, md3dImmediateContext);
		md3dImmediateContext->DrawIndexed(mBoxIndexCount, mBoxIndexOffset, mBoxVertexOffset);
    }

	md3dImmediateContext->OMSetRenderTargets(1, &mRenderTargetView, mDepthStencilView);
	md3dImmediateContext->ClearRenderTargetView( mRenderTargetView, reinterpret_cast<const float*>(&Colors::Black));
 
	md3dImmediateContext->ClearDepthStencilView(mDepthStencilView, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);
	//COMPOSITE TEH QUADS TOGETHING FILTERING OUT THE HAZE AND NORMAL IMAGE
	DrawCompositeQuad();
	HR(mSwapChain->Present(0, 0));

And the shader

/************* Resources *************/
static const float ZeroCorrection = 0.5f / 255.0f;


    float4x4 gWorldViewProj;

cbuffer CBufferPerObjectComposite
{
    float DisplacementScale = 3.0f;
	float haze = 0.0f;
}



Texture2D SceneTexture;
Texture2D DistortionMap;

SamplerState TrilinearSampler
{
    Filter = MIN_MAG_MIP_LINEAR;
    AddressU = CLAMP;
    AddressV = CLAMP;
};


struct VertexIn
{
	float3 PosL    : POSITION;
	float3 NormalL : NORMAL;
	float2 Tex     : TEXCOORD;
};

struct VertexOut
{
	float4 PosH : SV_POSITION;
	float2 Tex  : TEXCOORD;
};

/************* Cutout *************/

VertexOut VS(VertexIn vin)
{
   VertexOut vout;

	vout.PosH = mul(float4(vin.PosL, 1.0f), gWorldViewProj);
	vout.Tex  = vin.Tex;
	
	return vout;
}

float4 PS(VertexOut pin) : SV_Target
{
    float2 displacement = DistortionMap.Sample(TrilinearSampler, pin.Tex).xy;

    return float4(displacement.xy, 0, 1);
}

/************* Compositing *************/

VertexOut VS2(VertexIn vin)
{
   VertexOut vout;

	vout.PosH = mul(float4(vin.PosL, 1.0f), gWorldViewProj);
	vout.Tex  = vin.Tex;
	
	return vout;
}

float4 PS2(VertexOut pin) : SV_Target
{
    float4 OUT = (float4)0;

    float2 displacement = DistortionMap.Sample(TrilinearSampler, pin.Tex).xy;
   
	if (displacement.x == 0 && displacement.y == 0)
	{
		OUT = SceneTexture.Sample(TrilinearSampler, pin.Tex);
	}
	else
	{
		displacement -= 0.5f + ZeroCorrection;
		OUT = SceneTexture.Sample(TrilinearSampler, pin.Tex + (cos(haze) * displacement));
	}

    return OUT;
}



/************* Techniques *************/

technique11 displacement_cutout
{
    pass p0
    {
        SetVertexShader(CompileShader(vs_5_0, VS()));
        SetGeometryShader(NULL);
        SetPixelShader(CompileShader(ps_5_0, PS()));
    }
}

technique11 distortion_composite
{
    pass p0
    {
        SetVertexShader(CompileShader(vs_5_0, VS2()));
        SetGeometryShader(NULL);
        SetPixelShader(CompileShader(ps_5_0, PS2()));
    }
}

-do=add#.url

Shouldn't see the haze

why? because of occlusion? if that's what you want, then take the occlusion from the depth buffer into account in your composition shader

Ok, so depth is needed. I need something specific. Anyone? I tried OMSetDepthStencilState with DepthEnable to true but still the haze is in front of box. here is my composite function if needed.

DrawCompositeQuad()
{


	md3dImmediateContext->RSSetViewports(1, &mScreenViewport);



	UINT stride = sizeof(Vertex::Basic32);
    UINT offset = 0;

	md3dImmediateContext->IASetInputLayout(InputLayouts::Basic32);
    md3dImmediateContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
	md3dImmediateContext->IASetVertexBuffers(0, 1, &mScreenQuadVB, &stride, &offset);
	md3dImmediateContext->IASetIndexBuffer(mScreenQuadIB, DXGI_FORMAT_R32_UINT, 0);
	
	// Scale and shift quad to lower-right corner.
	XMMATRIX world(
		1.0f, 0.0f, 0.0f, 0.0f,
		0.0f, 1.0f, 0.0f, 0.0f,
		0.0f, 0.0f, 1.0f, 0.0f,
		0.0f, 0.0f, 0.0f, 1.0f);

	ID3DX11EffectTechnique* tech = Effects::DistortionFX->composite;

	D3DX11_TECHNIQUE_DESC techDesc;

	tech->GetDesc( &techDesc );
	for(UINT p = 0; p < techDesc.Passes; ++p)
    {
		Effects::DistortionFX->SetWorldViewProj(world);
		Effects::DistortionFX->SetDiffuseMap(second.GetSRV()); //normal scene quad
		Effects::DistortionFX->SetDiffuseMap2(first.GetSRV()); //cutout quad
		Effects::DistortionFX->SetHaze(mTimer.TotalTime());
		tech->GetPassByIndex(p)->Apply(p, md3dImmediateContext);
		md3dImmediateContext->DrawIndexed(6, 0, 0);
    }
}

If you want both objects to occlude each other in the same 3d space, you'll have to reuse the depth buffer for both objects, and not clear it in between. Suppose for a given pixel, the box is at depth value 0.3, and your haze effect should be invisible because it is at depth 0.6, then the 0.3 has to be in the depth buffer while drawing the haze quad, so the 0.6 result from the haze shader can be compared to it, and ignored. If you clear the depth buffer, the depth occlusion information gets erased.

edit: I think I misunderstood. I guess the goal of the cutout pass is to add distortion to some texture. But that happens for all pixels covered by the geometry you draw during this pass. And you are drawing the box in this pass, while during the 'normal scene' pass you draw the box with a different transformation. So it seems these two passes are basically unrelated.

If I understand correctly, you should first draw the normal scene box. Then keep the depth buffer and view-projection matrix, and draw whatever haze geometry you want for the cutout pass. Then only the visible haze pixels will have received distortion, so only those pixels will be distorted during the composition pass.

Alright, running good.