I've followed this tutorial for a custom ray-obb intersection test, but I'm having some trouble with my implementation. I'm not even sure what the behavior is like; from one viewpoint the intersection tests positive all over the screen, but when I move right half a unit it tests false almost everywhere. I'm pretty sure I'm passing in the correct ray, and that the problem lies in the intersection test function. Here it is:
bool TestRayOBBIntersect(glm::vec3 rayOrigin, glm::vec3 rayDir, glm::vec3 aabbMin, glm::vec3 aabbMax, glm::mat4 ModelMat, float& intersectDistance){
float tMinX = 0.0f, tMinY = 0.0f, tMinZ = 0.0f;
float tMaxX = 100000.0f, tMaxY = 100000.0f, tMaxZ = 100000.0f;
float xDist, yDist, zDist;
glm::vec3 obbPos = glm::vec3(ModelMat[3].x, ModelMat[3].y, ModelMat[3].z);
glm::vec3 delta = obbPos - rayOrigin;
// X axis
glm::vec3 xaxis = glm::vec3(ModelMat[0].x, ModelMat[0].y, ModelMat[0].z);
float e = glm::dot(xaxis, delta);
float f = glm::dot(rayDir, xaxis);
float t1 = (e+aabbMin.x)/f;
float t2 = (e+aabbMax.x)/f;
if (t1 > t2){
float w = t1; t1 = t2; t2 = w;
}
if (t2 < tMaxX) tMaxX = t2;
if (t1 > tMinX) tMinX = t1;
if (tMaxX < tMinX) return false;
xDist = t1;// Distance to the nearest intersection in x
// Y axis
glm::vec3 yaxis = glm::vec3(ModelMat[1].x, ModelMat[1].y, ModelMat[1].z);
float g = glm::dot(yaxis, delta);
float h = glm::dot(rayDir, yaxis);
float t3 = (g+aabbMin.y)/h;
float t4 = (g+aabbMax.y)/h;
if (t3 > t4){
float w = t3; t3 = t4; t4 = w;
}
if (t4 < tMaxY) tMaxY = t4;
if (t3 > tMinY) tMinY = t3;
if (tMaxY < tMinY) return false;
yDist = t3;// Distance to the nearest intersection in y
// Z axis
glm::vec3 zaxis = glm::vec3(ModelMat[2].x, ModelMat[2].y, ModelMat[2].z);
float i = glm::dot(zaxis, delta);
float j = glm::dot(rayDir, zaxis);
float t5 = (i+aabbMin.z)/j;
float t6 = (i+aabbMax.z)/j;
if (t5 > t6){
float w = t5; t5 = t6; t6 = w;
}
if (t6 < tMaxZ) tMaxZ = t6;
if (t5 > tMinZ) tMinZ = t5;
if (tMaxZ < tMinZ) return false;
zDist = t5;// Distance to the nearest intersection in z
intersectDistance = sqrt((xDist*xDist + yDist*yDist + zDist*zDist));// Total distance to nearest intersection
return true;
}
Also, is this an OBB intersection test, or an AABB intersection test? I'd assume the rotation is present in the model mat, but the source code on the tutorial page has references to an AABB.