Hello everyone. I cannot apply anti-aliasing to shadows in Directx9. I successfully applied it to many points in the game, but I could not manage to do this in the shadows. I guess it's because I'm a newbie. I would be glad if you help me in this regard.

Creating a device:

int CGraphicDevice::Create(HWND hWnd, int iHres, int iVres, bool Windowed, int /*iBit*/, int iReflashRate)
{
	int iRet = CREATE_OK;
	Destroy();

	ms_iWidth	= iHres;
	ms_iHeight	= iVres;

	ms_hWnd		= hWnd;
	ms_hDC		= GetDC(hWnd);
	ms_lpd3d	= Direct3DCreate9(D3D_SDK_VERSION);

	if (!ms_lpd3d)
		return CREATE_NO_DIRECTX;

	if (!ms_kD3DDetector.Build(*ms_lpd3d, EL3D_ConfirmDevice))
		return CREATE_ENUM;

	if (!ms_kD3DDetector.Find(800, 600, 32, TRUE, &ms_iD3DModeInfo, &ms_iD3DDevInfo, &ms_iD3DAdapterInfo))
		return CREATE_DETECT;

	std::string stDevList;
	ms_kD3DDetector.GetString(&stDevList);

	D3D_CAdapterInfo * pkD3DAdapterInfo = ms_kD3DDetector.GetD3DAdapterInfop(ms_iD3DAdapterInfo);
	if (!pkD3DAdapterInfo)
	{
		Tracenf("adapter %d is EMPTY", ms_iD3DAdapterInfo);
		return CREATE_DETECT;
	}

	if (__IsInDriverBlackList(*pkD3DAdapterInfo))
	{
		iRet |= CREATE_BAD_DRIVER;
		__WarningMessage(hWnd, CREATE_BAD_DRIVER);
	}

	D3D_SModeInfo * pkD3DModeInfo = pkD3DAdapterInfo->GetD3DModeInfop(ms_iD3DDevInfo, ms_iD3DModeInfo);		
	if (!pkD3DModeInfo)
	{
		Tracenf("device %d, mode %d is EMPTY", ms_iD3DDevInfo, ms_iD3DModeInfo);
		return CREATE_DETECT;
	}

	D3DADAPTER_IDENTIFIER9& rkD3DAdapterId=pkD3DAdapterInfo->GetIdentifier();
	if (Windowed &&
		strnicmp(rkD3DAdapterId.Driver, "3dfx", 4)==0 &&
		22 == pkD3DAdapterInfo->GetDesktopD3DDisplayModer().Format)
	{
		return CREATE_FORMAT;
	}

	if (pkD3DModeInfo->m_dwD3DBehavior==D3DCREATE_SOFTWARE_VERTEXPROCESSING)
	{
		iRet |= CREATE_NO_TNL;

		// DISABLE_NOTIFY_NOT_SUPPORT_TNL_MESSAGE
		//__WarningMessage(hWnd, CREATE_NO_TNL);
		// END_OF_DISABLE_NOTIFY_NOT_SUPPORT_TNL_MESSAGE
	}

	std::string stModeInfo;
	pkD3DModeInfo->GetString(&stModeInfo);

	//Tracen(stModeInfo.c_str());

	int ErrorCorrection = 0;

RETRY:
	ZeroMemory(&ms_d3dPresentParameter, sizeof(ms_d3dPresentParameter));
	
	ms_d3dPresentParameter.Windowed							= Windowed;
	ms_d3dPresentParameter.BackBufferWidth					= iHres;
	ms_d3dPresentParameter.BackBufferHeight					= iVres;
	ms_d3dPresentParameter.hDeviceWindow					= hWnd;
	ms_d3dPresentParameter.BackBufferCount					= m_uBackBufferCount;
	ms_d3dPresentParameter.SwapEffect						= D3DSWAPEFFECT_DISCARD;

	if (Windowed)
	{
		ms_d3dPresentParameter.BackBufferFormat				= pkD3DAdapterInfo->GetDesktopD3DDisplayModer().Format;
	}
	else
	{
		ms_d3dPresentParameter.BackBufferFormat				= pkD3DModeInfo->m_eD3DFmtPixel;
		ms_d3dPresentParameter.FullScreen_RefreshRateInHz	= iReflashRate;
	}

	ms_d3dPresentParameter.Flags							= D3DPRESENTFLAG_DISCARD_DEPTHSTENCIL; // D3DPRESENTFLAG_LOCKABLE_BACKBUFFER
	ms_d3dPresentParameter.EnableAutoDepthStencil			= TRUE;
	ms_d3dPresentParameter.AutoDepthStencilFormat			= pkD3DModeInfo->m_eD3DFmtDepthStencil;
	for (uint8_t i = 0; i <= D3DMULTISAMPLE_16_SAMPLES; i++)
	{
		const auto multisamplelevel = static_cast<D3DMULTISAMPLE_TYPE>(i);
		const auto result = ms_lpd3d->CheckDeviceMultiSampleType(ms_iD3DAdapterInfo, D3DDEVTYPE_HAL, ms_d3dPresentParameter.BackBufferFormat, Windowed, multisamplelevel, 0);

		if (SUCCEEDED(result))
		{
			TraceError("multisamplelevel == %u", multisamplelevel);
			ms_d3dPresentParameter.MultiSampleType = multisamplelevel;
		}
		ms_d3dPresentParameter.MultiSampleQuality = 0;
	}

	ms_dwD3DBehavior = pkD3DModeInfo->m_dwD3DBehavior;

	if (FAILED(ms_hLastResult = ms_lpd3d->CreateDevice(
				ms_iD3DAdapterInfo,
				D3DDEVTYPE_HAL,
				hWnd,
				pkD3DModeInfo->m_dwD3DBehavior,
				&ms_d3dPresentParameter,
				&ms_lpd3dDevice)))
	{
		switch (ms_hLastResult)
		{
			case D3DERR_INVALIDCALL:
				Tracen("IDirect3DDevice.CreateDevice - ERROR D3DERR_INVALIDCALL\nThe method call is invalid. For example, a method's parameter may have an invalid value.");					
				break;
			case D3DERR_NOTAVAILABLE:
				Tracen("IDirect3DDevice.CreateDevice - ERROR D3DERR_NOTAVAILABLE\nThis device does not support the queried technique. ");
				break;
			case D3DERR_OUTOFVIDEOMEMORY:
				Tracen("IDirect3DDevice.CreateDevice - ERROR D3DERR_OUTOFVIDEOMEMORY\nDirect3D does not have enough display memory to perform the operation");
				break;
			default:
				Tracenf("IDirect3DDevice.CreateDevice - ERROR %d", ms_hLastResult);
				break;
		}

		if (ErrorCorrection)
			return CREATE_DEVICE;

		iReflashRate = 0;
		++ErrorCorrection;
		iRet = CREATE_REFRESHRATE;
		goto RETRY;
	}

	// Check DXT Support Info
	if(ms_lpd3d->CheckDeviceFormat(
				ms_iD3DAdapterInfo, 
				D3DDEVTYPE_HAL,
				ms_d3dPresentParameter.BackBufferFormat,
				0,
				D3DRTYPE_TEXTURE,
				D3DFMT_DXT1) == D3DERR_NOTAVAILABLE)
	{
		ms_bSupportDXT = false;
	}

	if(ms_lpd3d->CheckDeviceFormat(
				ms_iD3DAdapterInfo, 
				D3DDEVTYPE_HAL,
				ms_d3dPresentParameter.BackBufferFormat,
				0,
				D3DRTYPE_TEXTURE,
				D3DFMT_DXT3) == D3DERR_NOTAVAILABLE)
	{
		ms_bSupportDXT = false;
	}

	if(ms_lpd3d->CheckDeviceFormat(
				ms_iD3DAdapterInfo, 
				D3DDEVTYPE_HAL,
				ms_d3dPresentParameter.BackBufferFormat,
				0,
				D3DRTYPE_TEXTURE,
				D3DFMT_DXT5) == D3DERR_NOTAVAILABLE)
	{
		ms_bSupportDXT = false;
	}	

	if (FAILED((ms_hLastResult = ms_lpd3dDevice->GetDeviceCaps(&ms_d3dCaps))))
	{
		Tracenf("IDirect3DDevice.GetDeviceCaps - ERROR %d", ms_hLastResult);
		return CREATE_GET_DEVICE_CAPS2;
	}

	if (!Windowed)
		SetWindowPos(hWnd, HWND_TOPMOST, 0, 0, iHres, iVres, SWP_SHOWWINDOW);

	ms_lpd3dDevice->GetViewport(&ms_Viewport);

	m_pStateManager = new CStateManager(ms_lpd3dDevice);

	D3DXCreateMatrixStack(0, &ms_lpd3dMatStack);
	ms_lpd3dMatStack->LoadIdentity();

	ms_ptVS	= CreatePTStreamVertexShader();
	ms_pntVS = CreatePNTStreamVertexShader();
	ms_pnt2VS = CreatePNT2StreamVertexShader();

	D3DXMatrixIdentity(&ms_matIdentity);
	D3DXMatrixIdentity(&ms_matView);
	D3DXMatrixIdentity(&ms_matProj);
	D3DXMatrixIdentity(&ms_matInverseView);
	D3DXMatrixIdentity(&ms_matInverseViewYAxis);
	D3DXMatrixIdentity(&ms_matScreen0);
	D3DXMatrixIdentity(&ms_matScreen1);
	D3DXMatrixIdentity(&ms_matScreen2);

	ms_matScreen0._11 = 1;
	ms_matScreen0._22 = -1;	

	ms_matScreen1._41 = 1;
	ms_matScreen1._42 = 1;

	ms_matScreen2._11 = (float) iHres / 2;
	ms_matScreen2._22 = (float) iVres / 2;
	
	D3DXCreateSphere(ms_lpd3dDevice, 1.0f, 32, 32, &ms_lpSphereMesh, NULL);
	D3DXCreateCylinder(ms_lpd3dDevice, 1.0f, 1.0f, 1.0f, 8, 8, &ms_lpCylinderMesh, NULL);

	ms_lpd3dDevice->Clear(0L, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, 0xff000000, 1.0f, 0);

	if (!__CreateDefaultIndexBufferList())
		return false;

	if (!__CreatePDTVertexBufferList())
		return false;
	
	DWORD dwTexMemSize = GetAvailableTextureMemory();

	if (dwTexMemSize < 64 * 1024 * 1024)
		ms_isLowTextureMemory = true;
	else
		ms_isLowTextureMemory = false;

	if (dwTexMemSize > 100 * 1024 * 1024)
		ms_isHighTextureMemory = true;
	else
		ms_isHighTextureMemory = false;

	if (ms_d3dCaps.TextureAddressCaps & D3DPTADDRESSCAPS_BORDER)
		GRAPHICS_CAPS_CAN_NOT_TEXTURE_ADDRESS_BORDER=false;
	else
		GRAPHICS_CAPS_CAN_NOT_TEXTURE_ADDRESS_BORDER=true;

	//D3DADAPTER_IDENTIFIER8& rkD3DAdapterId=pkD3DAdapterInfo->GetIdentifier();
	if (strnicmp(rkD3DAdapterId.Driver, "SIS", 3) == 0)
	{
		GRAPHICS_CAPS_CAN_NOT_DRAW_LINE = true;
		GRAPHICS_CAPS_CAN_NOT_DRAW_SHADOW = true;
		GRAPHICS_CAPS_HALF_SIZE_IMAGE = true;
		ms_isLowTextureMemory = true;
	}
	else if (strnicmp(rkD3DAdapterId.Driver, "3dfx", 4) == 0)
	{
		GRAPHICS_CAPS_CAN_NOT_DRAW_SHADOW = true;
		GRAPHICS_CAPS_HALF_SIZE_IMAGE = true;
		ms_isLowTextureMemory = true;
	}

	return (iRet);
}

Create Shadow:

void CMapOutdoor::CreateCharacterShadowTexture()
{
	recreate = false;
	ReleaseCharacterShadowTexture();

	if (IsLowTextureMemory())
		SetShadowTextureSize(250);

	m_ShadowMapViewport.X = 1;
	m_ShadowMapViewport.Y = 1;
	m_ShadowMapViewport.Width = m_wShadowMapSize - 2;
	m_ShadowMapViewport.Height = m_wShadowMapSize - 2;
	m_ShadowMapViewport.MinZ = 0.0f;
	m_ShadowMapViewport.MaxZ = 1.0f;

	ms_lpd3dDevice->CreateDepthStencilSurface(m_wShadowMapSize, m_wShadowMapSize, D3DFMT_D16, D3DMULTISAMPLE_8_SAMPLES, 0, TRUE, &m_lpCharacterShadowMapDepthSurface, NULL);
	ms_lpd3dDevice->CreateTexture(m_wShadowMapSize, m_wShadowMapSize, 1, D3DUSAGE_RENDERTARGET, D3DFMT_R5G6B5, D3DPOOL_DEFAULT, &m_lpCharacterShadowMapTexture, NULL);
	m_lpCharacterShadowMapTexture->GetSurfaceLevel(0, &m_lpCharacterShadowMapRenderTargetSurface);
	
}

Begin Shadow:

bool CMapOutdoor::BeginRenderCharacterShadowToTexture()
{
	
	CCamera* pCurrentCamera = CCameraManager::Instance().GetCurrentCamera();
	STATEMANAGER.SetRenderState(D3DRS_MULTISAMPLEANTIALIAS, TRUE); // don't work.
	if (recreate)
		CreateCharacterShadowTexture();

	D3DXMATRIX matLightView, matLightProj;
	D3DXVECTOR3 v3Target = pCurrentCamera->GetTarget();

	D3DXVECTOR3 v3Eye(v3Target.x - 1.732f * 1250.0f,
					  v3Target.y - 1250.0f,
					  v3Target.z + 2.0f * 1.732f * 1250.0f);

	D3DXMatrixLookAtRH(&matLightView,
					   &v3Eye,
					   &v3Target,
					   &D3DXVECTOR3(0.0f, 0.0f, 1.0f));


	D3DXMatrixOrthoRH(&matLightProj, 2550.0f, 2550.0f, 1.0f, 15000.0f);
	STATEMANAGER.SaveTransform(D3DTS_VIEW, &matLightView);
	STATEMANAGER.SaveTransform(D3DTS_PROJECTION, &matLightProj);
	dwLightEnable = STATEMANAGER.GetRenderState(D3DRS_LIGHTING);
	STATEMANAGER.SetRenderState(D3DRS_LIGHTING, TRUE);

	STATEMANAGER.SaveRenderState(D3DRS_TEXTUREFACTOR, 0xFF808080);
	STATEMANAGER.SetTextureStageState(0, D3DTSS_COLOROP, D3DTOP_SELECTARG1);
	STATEMANAGER.SetTextureStageState(0, D3DTSS_COLORARG1, D3DTA_TFACTOR);
	STATEMANAGER.SetTextureStageState(0, D3DTSS_ALPHAOP, D3DTOP_DISABLE);
	STATEMANAGER.SetTextureStageState(1, D3DTSS_COLOROP, D3DTOP_DISABLE);
	
	ms_lpd3dDevice->GetDepthStencilSurface(&m_lpBackupDepthSurface);
	ms_lpd3dDevice->GetRenderTarget(0, &m_lpBackupRenderTargetSurface);
	ms_lpd3dDevice->SetRenderTarget(0, m_lpCharacterShadowMapRenderTargetSurface);
	ms_lpd3dDevice->SetDepthStencilSurface(m_lpCharacterShadowMapDepthSurface);
	ms_lpd3dDevice->Clear(0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, 0xFFFFFFFF, 1.0f, 0);

	ms_lpd3dDevice->GetViewport(&m_BackupViewport);
	ms_lpd3dDevice->SetViewport(&m_ShadowMapViewport);
	return true;
}

Anti-aliasing won't work on shadow maps, you need to use PCF or other more advanced shadow technique (VSM, ESM, EVSM). AA might work on shadow volumes, but those are obsolete. Temporal AA might help some, but you would still need PCF to produce soft shadows.

@Aressera Thank you very much. Now I understand..

I tried to do the methods you mentioned with external shaders such as .fx, but I could not import and adapt these files. If you have knowledge, can you give me a detailed example on how to transfer and run .fx files? Unfortunately, I am very new to Directx and I am having a hard time.

You will probably find it easier and better documented to use the APIs from the past 17 years. DX9 was the 2003-2006 API.

The newer, better shadow effects require newer systems.

Re-creating the graphics from two decades ago is something you can do, like people who want to work with classic cars from 70 years ago, but as you are a beginner it is generally better to use today's technology.

frob said:
but as you are a beginner it is generally better to use today's technology.

Easier said than done if todays APIs are as low level as DX12 / Vulkan.
But i agree and would recommend DX11, the latest high level API which still gets good driver support.
Looking at new Intel GPUs we see older APIs are only emulated and performance isn't ideal. I expect similar decisions from Qualcomm or ImgTech, both aiming to enter PC platform soon.

However, i've found a tutorial for shadows in DX9 in this book: https://www.realtimerendering.com/resources/shaderx/Introductions_and_Tutorials_with_DirectX_9.pdf

The shaders look like assembly language. I can't remember this old stuff, but guess it's pretty cumbersome to work with. Porting the whole project to DX11 might be the better choice.

@frob @JoeJ Firstly, thank you.

Yes, you are right in what you say. But considering I'm too newbie to even run a .fx file, you can imagine how difficult it could be for me to port DX9 to DX11.

I'm working on a game project powered by C++ and it was using directx8. Since it is very similar to Directx9, I did not have much difficulty in upgrading it. However, upgrading from Directx9 to 10 or 11 has big differences and is almost impossible for me. Therefore, I try to develop as much as possible in directx9. Yesterday I managed to include the .fx file into the project, but it had no effect on the game. I couldn't figure out why.

Codes I use:

LPD3DXBUFFER pErrors = nullptr;
LPD3DXEFFECT	pEffect;

HRESULT hrr = D3DXCreateEffectFromFile(
	ms_lpd3dDevice, 
	LPCSTR("testShader.fx"),
	nullptr,
	nullptr,
	D3DXSHADER_DEBUG,
	nullptr,
	&pEffect,
	&pErrors
);

if (FAILED(hrr))
{
	if (pErrors)
	{
		OutputDebugStringA((char*)pErrors->GetBufferPointer());
		pErrors->Release();
	}
}
pEffect->SetTechnique("Tech1");

//render codes -- beginscene
UINT pass;
pEffect->Begin(&pass, 0);
pEffect->BeginPass(0);
pEffect->EndPass();

// endscene
pEffect->End();