I'm trying to do mouse picking and actually it works but when I scale the object and click that point on the image it picks wrong one, this happens only after scaling an object and I'm sure matrices are correct, any idea?
DirectX::XMFLOAT4X4 projection;
DirectX::XMStoreFloat4x4(&projection, Graphics::projectionMatrix);
float viewX = ((2.0f * Input::GetMouseX()) / MainWindow::GetDisplayResolution().width - 1.0f) / projection(0,0);
float viewY = (1.0f - (2.0f * Input::GetMouseY()) / MainWindow::GetDisplayResolution().height) / projection(1,1);
DirectX::XMMATRIX inverseMV = DirectX::XMMatrixInverse(nullptr,DirectX::XMMatrixTranspose(entity->Renderer_->MV_Matrix)); //MV is column order
DirectX::XMVECTOR rayOrigin = DirectX::XMVector3TransformCoord(DirectX::XMVectorSet(0.0f,0.0f,0.0f,1.0f), inverseMV);
DirectX::XMVECTOR rayDirection = DirectX::XMVector3Normalize(DirectX::XMVector3TransformNormal(DirectX::XMVectorSet(viewX, viewY, 1.0f, 0.0f), inverseMV));
for (const auto& subMesh : entity->Renderer_->mesh->GetSubMeshes())
{
float distance = 0.0f;
if (subMesh.AABB.Intersects(rayOrigin, rayDirection, distance))
{
for (unsigned int i = 0; i < subMesh.GetIndexCount(); i += 3)
{
DirectX::XMVECTOR p1 = DirectX::XMLoadFloat3(&subMesh.positions[subMesh.indices[i]]);
DirectX::XMVECTOR p2 = DirectX::XMLoadFloat3(&subMesh.positions[subMesh.indices[i + 1]]);
DirectX::XMVECTOR p3 = DirectX::XMLoadFloat3(&subMesh.positions[subMesh.indices[i + 2]]);
if (DirectX::TriangleTests::Intersects(rayOrigin, rayDirection, p1, p2, p3, distance) && distance < minDistance)
{
minDistance = distance;
*pickedEntity = entity;
}
}
}
}