Hello! I'm making Mesh Collision Function.

But It has a important bug.

when two mesh's X Axis is same,

they are not colliding but returns 'true'

Here's the code!

bool TriangleCollision(Transform * transA, VEC * t10, VEC * t11, VEC * t12, Transform * transB, VEC * t20, VEC * t21, VEC * t22)
{
	VEC v0 = { t10->x,t10->y,t10->z };
	VEC v1 = { t11->x,t11->y,t11->z };
	VEC v2 = { t12->x,t12->y,t12->z };
	VEC u0 = { t20->x,t20->y,t20->z };
	VEC u1 = { t21->x,t21->y,t21->z };
	VEC u2 = { t22->x,t22->y,t22->z };
	VEC e1, e2;
	VEC n1, n2;
	VEC dd;
	//Vector2
	VEC isect1 = { 0,0,0 };
	VEC isect2 = { 0,0,0 };

	float du0, du1, du2, dv0, dv1, dv2, d1, d2;
	float du0du1, du0du2, dv0dv1, dv0dv2;
	float vp0, vp1, vp2;
	float up0, up1, up2;
	float bb, cc, max;

	short index;

	VecMinusVec(&v1, &v0, &e1);
	VecMinusVec(&v2, &v0, &e2);
	VecCross(&e1, &e2, &n1);
	d1 = -VecDot(&n1, &v0);

	du0 = VecDot(&n1, &u0) + d1;
	du1 = VecDot(&n1, &u1) + d1;
	du2 = VecDot(&n1, &u2) + d1;

	if (abs(du0) < FEPSILON) { du0 = 0.0f; }
	if (abs(du1) < FEPSILON) { du1 = 0.0f; }
	if (abs(du2) < FEPSILON) { du2 = 0.0f; }

	du0du1 = du0 * du1;
	du0du2 = du0 * du2;

	if (du0du1 > 0.0f && du0du2 > 0.0f)
		return false;

	VecMinusVec(&u1, &u0, &e1);
	VecMinusVec(&u2, &u0, &e2);
	VecCross(&e1, &e2, &n2);
	d2 = -VecDot(&n2, &u0);

	dv0 = VecDot(&n2, &v0) + d2;
	dv1 = VecDot(&n2, &v1) + d2;
	dv2 = VecDot(&n2, &v2) + d2;

	if (abs(dv0) < FEPSILON) { dv0 = 0.0f; }
	if (abs(dv1) < FEPSILON) { dv1 = 0.0f; }
	if (abs(dv2) < FEPSILON) { dv2 = 0.0f; }

	dv0dv1 = dv0 * dv1;
	dv0dv2 = dv0 * dv2;

	if (dv0dv1 > 0.0f && dv0dv2 > 0.0f)
		return false;

	VecCross(&n1, &n2, &dd);

	max = (float)abs(dd.x);
	index = 0;
	bb = (float)abs(dd.y);
	cc = (float)abs(dd.z);
	if (bb > max) { max = bb; index = 1; }
	if (cc > max) { max = cc; index = 2; }

	switch (index)
	{
	case 0:
		vp0 = v0.x;
		vp1 = v1.x;
		vp2 = v2.x;

		up0 = u0.x;
		up1 = u1.x;
		up2 = u2.x;
		break;
	case 1:
		vp0 = v0.y;
		vp1 = v1.y;
		vp2 = v2.y;

		up0 = u0.y;
		up1 = u1.y;
		up2 = u2.y;
		break;
	case 2:
		vp0 = v0.z;
		vp1 = v1.z;
		vp2 = v2.z;

		up0 = u0.z;
		up1 = u1.z;
		up2 = u2.z;
		break;
	}

	float a = 0; 
	float b = 0; 
	float c = 0;
	float x0 = 0;
	float x1 = 0;
	if (ComputeIntervals(vp0, vp1, vp2, dv0, dv1, dv2, dv0dv1, dv0dv2, a, b, c, x0, x1))
	{
		return TriTriCoplanar(n1, v0, v1, v2, u0, u1, u2);
	}

	float d = 0;
	float e = 0;
	float f = 0;
	float y0 = 0;
	float y1 = 0;
	if (ComputeIntervals(up0,up1,up2,du0,du1,du2,du0du1,du0du2,d,e,f,y0,y1))
	{
		return TriTriCoplanar(n1, v0, v1, v2, u0, u1, u2);
	}

	float xx, yy, xxyy, tmp;
	xx = x0 * x1;
	yy = y0 * y1;
	xxyy = xx * yy;

	tmp = a * xxyy;
	isect1.x = tmp + b * x1 * yy;
	isect1.y = tmp + c * x0 * yy;

	tmp = d * xxyy;
	isect2.x = tmp + e * xx * y1;
	isect2.y = tmp + f * xx * y0;

	Sort(isect1);
	Sort(isect2);

	return !(isect1.y < isect2.x || isect2.y < isect1.x);
}

void Sort(VEC& v)
{
	if (v.x > v.y)
	{
		float c;
		c = v.x;
		v.x = v.y;
		v.y = c;
	}
}

bool EdgeEdgeTest(VEC v0, VEC v1, VEC u0, VEC u1, int i0, int i1)
{
	float ax, ay, bx, by, cx, cy, e, d, f;
	float v0i0;
	float v1i0;
	float u0i0;
	float u1i0;
	if (i0 == 0)
	{
		v0i0 = v0.x;
		v1i0 = v1.x;
		u0i0 = u0.x;
		u1i0 = u1.x;
	}
	else if (i0 == 1)
	{
		v0i0 = v0.y;
		v1i0 = v1.y;
		u0i0 = u0.y;
		u1i0 = u1.y;
	}
	else if (i0 == 2)
	{
		v0i0 = v0.z;
		v1i0 = v1.z;
		u0i0 = u0.z;
		u1i0 = u1.z;
	}
	float v0i1;
	float v1i1;
	float u0i1;
	float u1i1;
	if (i1 == 0)
	{
		v0i1 = v0.x;
		v1i1 = v1.x;
		u0i1 = u0.x;
		u1i1 = u1.x;
	}
	else if (i1 == 1)
	{
		v0i1 = v0.y;
		v1i1 = v1.y;
		u0i1 = u0.y;
		u1i1 = u1.y;
	}
	else if (i1 == 2)
	{
		v0i1 = v0.z;
		v1i1 = v1.z;
		u0i1 = u0.z;
		u1i1 = u1.z;
	}
	ax = v1i0 - v0i0;
	ay = v1i1 - v0i1;

	bx = u0i0 - u1i0;
	by = u0i1 - u1i1;
	cx = v0i0 - u0i0;
	cy = v0i1 - u0i1;

	f = ay * bx - ax * by;
	d = by * cx - bx * cy;

	if ((f > 0 && d >= 0 && d <= 5) || (f < 0 && d <= 0 && d >= f))
	{
		e = ax * cy - ay * cx;
		if (f > 0)
		{
			if (e >= 0 && e <= f)
				return true;
		}
		else
		{
			if (e <= 0 && e >= f)
				return true;
		}
	}
	return false;
}

bool EdgeAgainstTriEdges(VEC v0, VEC v1, VEC u0, VEC u1, VEC u2, short i0, short i1)
{
	if (EdgeEdgeTest(v0, v1, u0, u1, i0, i1))
		return true;
	if (EdgeEdgeTest(v0, v1, u1, u2, i0, i1))
		return true;
	if (EdgeEdgeTest(v0, v1, u2, u0, i0, i1))
		return true;

	return false;
}

bool PtInTri(VEC v0, VEC u0, VEC u1, VEC u2, short i0, short i1)
{
	float a, b, c, d0, d1, d2;

	float v0i0;
	float u0i0;
	float u1i0;
	float u2i0;
	if (i0 == 0)
	{
		v0i0 = v0.x;
		u0i0 = u0.x;
		u1i0 = u1.x;
		u2i0 = u2.x;
	}
	else if (i0 == 1)
	{
		v0i0 = v0.y;
		u0i0 = u0.y;
		u1i0 = u1.y;
		u2i0 = u2.y;
	}
	else if (i0 == 2)
	{
		v0i0 = v0.z;
		u0i0 = u0.z;
		u1i0 = u1.z;
		u2i0 = u2.z;
	}
	float v0i1;
	float u0i1;
	float u1i1;
	float u2i1;
	if (i1 == 0)
	{
		v0i1 = v0.x;
		u0i1 = u0.x;
		u1i1 = u1.x;
		u2i1 = u2.x;
	}
	else if (i1 == 1)
	{
		v0i1 = v0.y;
		u0i1 = u0.y;
		u1i1 = u1.y;
		u2i1 = u2.y;
	}
	else if (i1 == 2)
	{
		v0i1 = v0.z;
		u0i1 = u0.z;
		u1i1 = u1.z;
		u2i1 = u2.z;
	}

	a = u1i1 - u0i1;
	b = -(u1i0 - u0i0);
	c = -a * u0i0 - b * u0i1;
	d0 = a * v0i0 + b * v0i1 + c;

	a = u2i1 - u1i1;
	b = -(u2i0 - u1i0);
	c = -a * u1i0 - b * u1i1;
	d1 = a * v0i0 + b * v0i1 + c;

	a = u0i1 - u2i1;
	b = -(u0i0 - u2i0);
	c = -a * u2i0 - b * u2i1;
	d2 = a * v0i0 + b * v0i1 + c;

	if (d0 * d1 > 0.0f)
	{
		if (d0 * d2 > 0.0f)
			return true;
	}
	return false;
}

bool TriTriCoplanar(VEC n, VEC v0, VEC v1, VEC v2, VEC u0, VEC u1, VEC u2)
{
	float a[3];
	short i0, i1;

	a[0] = abs(n.x);
	a[1] = abs(n.y);
	a[2] = abs(n.z);

	if (a[0] > a[1])
	{
		if (a[0] > a[2])
		{
			i0 = 1;
			i1 = 2;
		}
		else
		{
			i0 = 0;
			i1 = 1;
		}
	}
	else
	{
		if (a[2] > a[1])
		{
			i0 = 0;
			i1 = 1;
		}
		else
		{
			i0 = 0;
			i1 = 2;
		}
	}

	if (EdgeAgainstTriEdges(v0, v1, u0, u1, u2, i0, i1))
		return true;
	if (EdgeAgainstTriEdges(v1, v2, u0, u1, u2, i0, i1))
		return true;
	if (EdgeAgainstTriEdges(v2, v0, u0, u1, u2, i0, i1))
		return true;

	if (PtInTri(v0, u0, u1, u2, i0, i1))
		return true;
	if (PtInTri(u0, v0, v1, v2, i0, i1))
		return true;

	return false;
}

bool ComputeIntervals(float vv0, float vv1, float vv2, float d0, float d1, float d2, float d0d1, float d0d2, float& a, float& b, float& c, float& x0, float& x1)
{
	if (d0d1 > 0.0f)
	{
		a = vv2; b = (vv0 - vv2) * d2; c = (vv1 - vv2) * d2; x0 = d2 - d0; x1 = d2 - d1;
	}
	else if (d0d2 > 0.0f)
	{
		a = vv1; b = (vv0 - vv1) * d1; c = (vv2 - vv1) * d1; x0 = d1 - d0; x1 = d1 - d2;
	}
	else if (d1 * d2 > 0.0 || d0 != 0.0f)
	{
		a = vv0; b = (vv1 - vv0) * d0; c = (vv2 - vv0) * d0; x0 = d0 - d1; x1 = d0 - d2;
	}
	else if (d1 != 0.0f)
	{
		a = vv1; b = (vv0 - vv1) * d1; c = (vv2 - vv1) * d1; x0 = d1 - d0; x1 = d1 - d2;
	}
	else if (d2 != 0.0f)
	{
		a = vv2; b = (vv0 - vv2) * d2; c = (vv1 - vv2) * d2; x0 = d2 - d0; x1 = d2 - d1;
	}
	else
		return true;

	return false;
}

That code seems difficult to debug visually (to me at least). If you're willing to entertain some tangential suggestions though, here are a few things that might help.

It looks like you're programming in C++, correct? If so, there are a lot of things you could do to make the code more concise, readable, and robust. Even in C, something like this:

VEC v0 = { t10->x,t10->y,t10->z };

Shouldn't be necessary. At the very least you could create a duplicate/copy function.

Regardless, under normal circumstances you should be able to do a direct assignment there. (Also, are v0, etc. ever modified? If not, is there any particular reason for making copies of the 't' arguments?) Among other things, each time you do it manually like this is an opportunity for errors to be introduced.

C++ also offers operator overloading, which allows you to e.g. replace this:

VecMinusVec(&v1, &v0, &e1);

With this:

e1 = v1 - v0;

The short variable names and the fact that they're often declared in advance and not at first use also make the code hard to follow. Lastly, correct use of 'const' can both help catch bugs and make code easier to read by letting the reader know whether or not a value is expected to change.

Orthogonal to all that, testing functions in isolation might also help. You have many support functions there, most of which do non-trivial work. A bug in any of them could cause you're higher-level tests to fail. I'd recommend working on one function at a time and not moving on until you're fairly confident each function is correct (be sure to test corner/edge cases as well).

Lastly, the debugger can be a valuable tool. One approach is to set up simple test cases where you know what the results should be, and then step through in the debugger to see if/where the actual results differ from what's expected.

1 hour ago, Zakwayda said:

Lastly, the debugger can be a valuable tool. One approach is to set up simple test cases where you know what the results should be, and then step through in the debugger to see if/where the actual results differ from what's expected.

QFE

If you have a case that you know returns the wrong answer, debug it, either by a debugger (the simplest, and a useful skill to develop if you don't know how to use a debugger), but even simple "printf" debugging would work. In the latter case, make a program that calls the routine exactly once, with the know bad case, sprinkle the routine with print statements what it calculates, run it, and study the output.