Hi ,
I am trying to do object picking.I calculate AABB and do ray - AABB intersaction.
When the object has position(0,0,0) everything works , but when I change the position the ray - AABB intersaction is alwayes true.

This is how I calculate the ray :

float pointX = (2.0f * localX / width - 1.0f) / shaderData.projection._11;
float pointY = (-2.0f * localY / height + 1.0f) / shaderData.projection._22;

XMVECTOR dir = XMVectorSet(pointX, pointY, 1.0f, 0.0f);

XMVECTOR det = XMMatrixDeterminant(view);
XMMATRIX viewInv = XMMatrixInverse(&det, view);

dir = XMVector3TransformNormal(dir, viewInv);

XMStoreFloat3((XMFLOAT3*)&pickRayDirection, dir);

And for every object I do this :

XMFLOAT4X4 mat = ent->GetWorldMatrix();
XMMATRIX world = XMLoadFloat4x4((XMFLOAT4X4*)&mat);
XMVECTOR det = XMMatrixDeterminant(world);

XMMATRIX invWorld = XMMatrixInverse(&det, world);
Vector3 org = cam->GetPosition();
Vector3 dir = cam->GetPickingRayDirection();
XMVECTOR orgInv = XMVector3TransformCoord(XMLoadFloat3((XMFLOAT3*)&org), invWorld);
XMVECTOR dirInv = XMVector3TransformNormal(XMLoadFloat3((XMFLOAT3*)&dir), invWorld);

dirInv = XMVector3Normalize(dirInv);
XMStoreFloat3((XMFLOAT3*)&org, orgInv);
XMStoreFloat3((XMFLOAT3*)&dir, dirInv);

Somewhere I saw that they get the translation matrix of the object , multiply it by the world matrix of the object and then inverse it.

Isn't the inverse of the translation done in the inverse world matrix ?!?!

I can apply any rotation and scale to the object and it will work , but if I change the position it does not work .

Tx

*EDIT :

I fixed it ! I just had to transpose the world matrix before I inverse it wow...

this is how i do it (note that mat4 is for DirectX matrix order whatever that is)

inline vec3 GetDirectionFromScreen(mat4 modelview, TSpecCamera * FPP_CAM,  int x, int y, int sw, int sh, float fov, float z_near, float z_far)
{

mat4 mvm = modelview;
mvm.Transpose();
vec3 dirX, dirY;
	dirX.x = mvm.m[0];
	dirX.y = mvm.m[4];
	dirX.z = mvm.m[8];

	dirY.x =	mvm.m[1];
	dirY.y =	mvm.m[5];
	dirY.z =	mvm.m[9];

	float aspect = float(SCREEN_WIDTH) / float(SCREEN_HEIGHT);
	float a = fov / 2.0;
float cotangent = 1.0 / tan( a * imopi );

float ax = z_near / cotangent;

float screen_w = 2.0*ax;

float screen_h = screen_w;// * yratio;

screen_w = screen_w * aspect;

float scr_coord_x = float(x) / float(sw);
float scr_coord_y = float(sh - y) / float(sh);


vec3 dir = FPP_CAM->ReturnFrontVector();

//move to lower left corner
vec3 start_pos = (FPP_CAM->pos + dir * z_near) + (-dirX * (screen_w / 2.0)) + (-dirY * (screen_h/2.0));

vec3 start = start_pos + (dirX * (screen_w * scr_coord_x)) + (dirY * (screen_h * scr_coord_y));

return Normalize( vectorAB(FPP_CAM->pos, start) );
}

fov is actually fovy (up down angle)

now this function returns a normalized vector, you use it like rayA = EYE_POSITION; rayB = rayA + GetDirectionFromScreen(MODEL_MATRIX * VIEW_MATRIX)*1000.0;

where MODEL_MATRIX is identity...

also x,y are screen pixel coords and float scr_coord_y = float(sh - y) / float(sh); is for opengl compilant thing since its 0,0 is at bottom left by default

another thing is to properly calculate aabb for each object

so you loop through vertices and do something like this:

 void  CalculateMinMax()
{

T minx = 1000000000;
T miny = 1000000000;
T minz = 1000000000;


T maxx = -1000000000;
T maxy = -1000000000;
T maxz = -1000000000;
int i;
for (i = 0; i < header.LENGTH; i++) {
if (minx > AOS[i].v.x) minx = AOS[i].v.x;
if (miny > AOS[i].v.y) miny = AOS[i].v.y;
if (minz > AOS[i].v.z) minz = AOS[i].v.z;


if (maxx < AOS[i].v.x) maxx = AOS[i].v.x;
if (maxy < AOS[i].v.y) maxy = AOS[i].v.y;
if (maxz < AOS[i].v.z) maxz = AOS[i].v.z;

}


MIN.x = minx;
MIN.y = miny;
MIN.z = minz;

MAX.x = maxx;
MAX.y = maxy;
MAX.z = maxz;

}

but theres a catch, you should always first move objects center to 0,0,0 point then calculate min and max

then to do proper ray aabb box intersection you define object position and you need to add the position to the rayPolyogn or rayBox intersection algorithm (additionally if you have a rotation for the aabb then you need to add the rotation too) - i dont know if i made that even clear, but you cant test against object vertices without adding objects position to the checked polygon verts. (by polyogn i mean a face of an AABB)

Hi Cat ,

It is interesting that you transpose the modelView matrix.Do you transpose the view matrix before you pass it to GetDirectionFromScreen ?

And dont you think that GetDirectionFromScreen is too big to be inlined ;)

Anyway I think that my AABB calculation is correct and my ray calculation is correct , I just can't put the ray and the AABB in the same space :)

both model matrix and view matrix are not transposed, i transpose them in that function only because i didint want to think where is transposed mvm.m[4] etc.

for that purpose all of those mats are in order like this:

void Translate(T x, T y, T z)
{
	Matrix44<T> tran(                 1, 0, 0, x,
					  0, 1, 0, y,
					  0, 0, 1, z,
					  0, 0, 0, 1);
	(*this) = (*this) * tran;
}

i suggest you write explanation for every line of your code and tell me what you think it does, so i can verify that

mine does something liek that

from modelviewmatrix (not the object model matrix so thats why its identity)

i get transpose to eaisly get direction vectors of camera

becasue for transposed matrix in this case its [i only do that because i didin't have anywhere in code the spec for the normal matrix but i had this so i just transpose and use this below ;p)

			AIR_MATRIX[0] = rr.x; //right
			AIR_MATRIX[1] = rr.y;
			AIR_MATRIX[2] = rr.z;
			AIR_MATRIX[3] = 0.0;
			AIR_MATRIX[4] = ru.x; //up
			AIR_MATRIX[5] = ru.y;
			AIR_MATRIX[6] = ru.z;
			AIR_MATRIX[7] = 0.0;
			AIR_MATRIX[8]  = -rf.x; //front
			AIR_MATRIX[9]  = -rf.y;
			AIR_MATRIX[10] = -rf.z;
			AIR_MATRIX[11] = 0.0;
			AIR_MATRIX[12] =  0.0;  //point better not use it
			AIR_MATRIX[13] =  0.0;
			AIR_MATRIX[14] =  0.0;
			AIR_MATRIX[15] = 1.0;

more over i compute height - pixel_y for opengl compilant thing since i move from lower left corner of screen

so now i have eye position, right direction, up direction, and front direction

i need aspect ration and field of view (i pass vertical field of view in this case not horizontal)

this tells me the size of the z_near quad in view pyramid

now i go to the bottom left corner of that

then knowing the screen coord and screen size i compute the exact point on z_near plane

ray vector is then eye to this calculated point then you need to normalize it and multiply by some factor.

well do as you want, here you have complete solution.

Do not invert matrices unless you really need to during the game.

Make sure that you perform all the computations in the same space. Do all of it in world space just to make sure.

Not too different of your code, this is how I compute a ray in world space assuming DirectX (position = mouse position in client coordinates):

inline CVector3 ScreenToWorld(const CCamera& camera, const D3D11_VIEWPORT& screen, const CVector2& screenPos) {
	// Rewind (Screen->NDC->Projection->World)
	// To NDC.
	float scaleX = 0.5f * screen.Width;
	float scaleY = 0.5f * screen.Height;

	CVector3 ndcPos;
	ndcPos.x = (screenPos.x - scaleX) / scaleX;
	ndcPos.y = -(screenPos.y - scaleY) / scaleY;
	ndcPos.z = 1.0f;

	// To clip.
	CVector3 clipPos;
	clipPos.x = ndcPos.x / camera.ProjMatrix()._11;
	clipPos.y = ndcPos.y / camera.ProjMatrix()._22;
	clipPos.z = ndcPos.z;

	// To world.
	CMatrix4x4 invView = camera.Orientation().CreateTransformationMatrix();

	CVector3 worldPos = invView * clipPos;

	return worldPos;
}

Or, for readability if you will:

inline CMatrix4x4 GetInvMatrix(const D3D11_VIEWPORT& screen) {
	CMatrix4x4 out;
	out._11 = 1.0f / (0.5f * screen.Width);
	out._12 = 0.0f;
	out._13 = 0.0f;
	out._14 = 0.0f;

	out._21 = 0.0f;
	out._22 = 1.0f / (-0.5f * screen.Height);
	out._23 = 0.0f;
	out._24 = 0.0f;

	out._31 = 0.0f;
	out._32 = 0.0f;
	out._33 = 1.0f / (screen.MaxDepth - screen.MinDepth);
	out._34 = 0.0f;

	out._41 = -out._11 * (screen.TopLeftX + 0.5f * screen.Width);
	out._42 = -out._22 * (screen.TopLeftY + 0.5f * screen.Height);
	out._43 = -out._33 * screen.MinDepth;
	out._44 = 1.0f;
	return out;
}

inline CVector3 ScreenToWorld(const CCamera& camera, const D3D11_VIEWPORT& screen, const CVector2& screenPos) {
	// Screen->Projection->View->World
	CMatrix4x4 invScreen = GetInvMatrix(screen);
        CMatrix4x4 invView = camera.Orientation().CreateTransformationMatrix();

	CVector3 projPos = invScreen * CVector3(screenPos.x, screenPos.y, screen.MaxDepth);
	CVector3 viewPos(projPos.x / camera.ProjMatrix()._11, projPos.y / camera.ProjMatrix()._22, projPos.z);
	CVector3 worldPos = invView * viewPos;
	
	return worldPos;
}

And that's how I do a Ray-AABB test (p = ray origin, d = ray direction):

// Christer Ericson, Real-Time Collision Detection (2005), p. 180.
// Correction: http://realtimecollisiondetection.net/books/rtcd/errata/
bool RayCast(const b3Vec3& p, const b3Vec3& d, r32 tmax, r32& tmin) const {
		// Test all three slabs.
		for (u32 i = 0; i < 3; i++) {
			if (b3Abs(d[i]) < B3_EPSILON) { //epsilon = very small value
				// The segment is parallel to slab. No hit if origin not within slab.
				if (p[i] < min[i] || p[i] > max[i]) {
					return false; 
				}
			}
			else {
				// Compute intersection t value of segment with near and far plane of slab.
				r32 ood = B3_ONE / d[i];
				r32 t1 = ood * (min[i] - p[i]);
				r32 t2 = ood * (max[i] - p[i]);

				// Make t1 be intersection with near plane, t2 with far plane.
				if (t1 > t2) {
					b3Swap(t1, t2);
				}

				// Compute the intersection of slab intersection intervals
				tmin = b3Max(tmin, t1); // Rather than: if (t1 > tmin) tmin = t1;
				tmax = b3Min(tmax, t2); // Rather than: if (t2 < tmax) tmax = t2;

				// Exit with no collision as soon as slab intersection becomes empty.
				if (tmin > tmax) { 
					return false; 
				}
			}
		}

		return true;
	}

I calculate the AABB in object space , so I tried to put the ray in object space (and it is not working if I change the position of the object) and I tried to put the AABB in world space and it is not working at all.

ray in object space I do - rayDir * invWorldMatrix(of the object) and rayOrg * invWorldMatrix(of the object)

AABB in world space I do - AABBmax * worldMatrix and AABBmin * worldMatrix ( I am not sure if the math here is correct)

Also when I calculate the ray when I do :

vec3 p = (normalizedMouseY,normalizedMouseY,1.0f);

p = inverseViewMatrix * p

Is p now a point or a direction ?