Pls help me. Is my collision detection wrong?

	static bool SphereCapsule(SphereCollider *A, CapsuleCollider *B, vector<Contact>& contacts)
	{
		float AX2 = SqDistPointSegment(B->GetCenter1(), B->GetCenter2(), A->GetCenter());
		float rS = A->GetRadius();
		float rC = B->GetRadius();

		float t;
		XMVECTOR X;
		ClosestPtPointSegment(A->GetCenter(), B->GetCenter1(), B->GetCenter2(), t, X);

		XMVECTOR CenterSeparation = X - A->GetCenter();
		float Separation = sqrt(AX2) - (rS + rC);
		bool testPart = (Separation <= 0.0f);

		if (testPart)
		{
			XMVECTOR Normal = XMVector3Normalize(CenterSeparation);
			XMVECTOR EndSurfPointA = A->GetCenter() + Normal * A->GetRadius();
			XMVECTOR EndSurfPointB = X - Normal * B->GetRadius();
			XMVECTOR ContactPoint = (EndSurfPointA + EndSurfPointB) / 2.0f;

			char cStr[2048];
			sprintf_s(cStr, "[COLLISION] [SPHERE, CAPSULE] Normal %.10f %.10f %.10f Point %.10f %.10f %.10f Separation %.10f", Normal.m128_f32[0], Normal.m128_f32[1], Normal.m128_f32[2], ContactPoint.m128_f32[0], ContactPoint.m128_f32[1], ContactPoint.m128_f32[2], Separation);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);

			Contact contact;
			contact.point = ContactPoint;
			contact.normal = Normal;
			contact.separation = (Separation);
			contact.contactID = 2;
			contacts.push_back(contact);
		}
		return testPart;
	}

	static bool CapsuleCapsule(CapsuleCollider *A, CapsuleCollider *B, vector<Contact>& contacts)
	{
		float s, t;
		XMVECTOR X1, X2;
		float X1X22 = ClosestPtSegmentSegment(A->GetCenter1(), A->GetCenter2(), B->GetCenter1(), B->GetCenter2(), s, t, X1, X2);

		XMVECTOR CenterSeparation = X2 - X1;
		float rA = A->GetRadius();
		float rB = B->GetRadius();
		float Separation = sqrt(X1X22) - (rA + rB);
		bool testPart = (Separation <= 0.0f);

		if (testPart)
		{
			XMVECTOR Normal = XMVector3Normalize(CenterSeparation);
			XMVECTOR EndSurfPointA = X1 + Normal * rA;
			XMVECTOR EndSurfPointB = X2 - Normal * rB;
			XMVECTOR ContactPoint = (EndSurfPointA + EndSurfPointB) / 2.0f;

			char cStr[2048];
			sprintf_s(cStr, "[COLLISION] [CAPSULE, CAPSULE] Normal %.10f %.10f %.10f Point %.10f %.10f %.10f Separation %.10f", Normal.m128_f32[0], Normal.m128_f32[1], Normal.m128_f32[2], ContactPoint.m128_f32[0], ContactPoint.m128_f32[1], ContactPoint.m128_f32[2], Separation);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);

			Contact contact;
			contact.point = ContactPoint;
			contact.normal = Normal;
			contact.separation = (Separation);
			contact.contactID = 4;
			contacts.push_back(contact);
		}
		return testPart;
	}

	static bool CapsulePlane(CapsuleCollider *A, PlaneCollider *B, vector<Contact>& contacts)
	{
		XMVECTOR CapsuleNormal = A->GetDirection(); 
		XMVECTOR LineEndOffset = CapsuleNormal * A->GetRadius(); 
		XMVECTOR CapBase = A->GetCenter1() - LineEndOffset; 
		XMVECTOR CapTip = A->GetCenter2() + LineEndOffset;

		float denom = XMVector3Dot(B->GetDirection(), CapsuleNormal).m128_f32[0];
		if (denom == 0.0f) denom += EPSILON;

		XMVECTOR p0 = XMVectorSet(0, 0, 0, 1); //hack for simplifying the eqns
		float t = XMVector3Dot(B->GetDirection(), (p0 - A->GetCenter1()) / abs(denom)).m128_f32[0];
		XMVECTOR CapsuleLine_Plane_Intersection = A->GetCenter1() + CapsuleNormal * t;

		XMVECTOR X = ClosestPointOnLineSegment(CapBase, CapTip, CapsuleLine_Plane_Intersection);
		float Separation = (XMVector3Dot(X, B->GetDirection()).m128_f32[0] - B->GetDistanceFromOrigin()) - A->GetRadius();
		bool test = (Separation <= 0.0f);

		if (test)
		{
			XMVECTOR Normal = XMVector3Normalize(B->GetDirection());
			XMVECTOR ContactPoint = X - Normal * A->GetRadius();

			char cStr[2048];
			sprintf_s(cStr, "[COLLISION] [CAPSULE, PLANE] Normal %.10f %.10f %.10f Point %.10f %.10f %.10f Separation %.10f", Normal.m128_f32[0], Normal.m128_f32[1], Normal.m128_f32[2], ContactPoint.m128_f32[0], ContactPoint.m128_f32[1], ContactPoint.m128_f32[2], Separation);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);

			Contact contact;
			contact.point = ContactPoint;
			contact.normal = Normal;
			contact.separation = (Separation);
			contact.contactID = 5;
			contacts.push_back(contact);
		}
		return test;
	}

I checked the sphere vs. capsule code and didn't see anything wrong. The only thing that I can think of is that you need two collision points, one on body A and the other on body B, rather than the average. Using the right collision points would be important mostly for applying angular impulses in collision response.

Hi Sir, I thank you for replying to my post, cos it seems nobody does that. According to Dirk Gregorius's Generating Robust Contacts for Physics Sim, I have to put the contact point between the two surfaces for Sphere-Capsule and Capsule-Capsule.

Could you please look into this thread also, just give me hint, I can work it out the rest. pls. https://gamedev.net/forums/topic/714834-my-objects-sink-through-the-floor/5458717/​

My friction somewhat is not working correctly, here's a video clip for your reference

Based on the video it looks like you might not be transforming contact points with the object rotation, or are using a wrong (i.e. inverse) rotation. You should try visualizing the contact points and normals you compute to ensure they match up with the other rendering.

Now I've got friction working. here's the latest code for applyimpulse and prestep. Apart from logging the code, I'm setting up to visualize the vectors. Could pls pls look at my code and tell if there's an error? pls

void Arbiter::PreStep(float inv_dt)
{
	char cStr[2048];

	for (int i = 0; i < contacts.size(); ++i)
	{
		Contact& c = contacts[i];
		{
			sprintf_s(cStr, "[ARBITER] Restituition %.10f Friction %.10f", restituition, friction);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}
		XMVECTOR ContactPos = c.point;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] ContactPos %.10f %.10f %.10f", ContactPos.m128_f32[0], ContactPos.m128_f32[1], ContactPos.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR rA = XMVector3Transform((ContactPos - body1->GetPosition()), body1->GetOrientation());
		XMVECTOR rB = XMVector3Transform((ContactPos - body2->GetPosition()), body2->GetOrientation());
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] rA %.10f %.10f %.10f rB %.10f %.10f %.10f", rA.m128_f32[0], rA.m128_f32[1], rA.m128_f32[2], rB.m128_f32[0], rB.m128_f32[1], rB.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR VelocityA = body1->GetMass() > 0 ? (body1->GetVelocityAtCM() + XMVector3Cross(body1->GetAngularVelocity(), rA)) : XMVectorSet(0,0,0,1);
		XMVECTOR VelocityB = body2->GetMass() > 0 ? (body2->GetVelocityAtCM() + XMVector3Cross(body2->GetAngularVelocity(), rB)) : XMVectorSet(0,0,0,1);
		XMVECTOR dVel = VelocityB - VelocityA;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] VelocityA %.10f VelocityB %.10f dVel %.10f %.10f %.10f", XMVector3Length(VelocityA).m128_f32[0], XMVector3Length(VelocityB).m128_f32[0], dVel.m128_f32[0], dVel.m128_f32[1], dVel.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR ContactNormal = c.normal;
		ContactNormal = XMVector3Transform(ContactNormal, body1->GetOrientation());

		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] ContactNormal %.10f %.10f %.10f", ContactNormal.m128_f32[0], ContactNormal.m128_f32[1], ContactNormal.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}
		float Val = XMVector3Dot((body2->GetPosition() - body1->GetPosition()), ContactNormal).m128_f32[0];
		if (Val < 0.0f)
			ContactNormal *= -1;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] Val %.10f ContactNormal %.10f %.10f %.10f", Val, ContactNormal.m128_f32[0], ContactNormal.m128_f32[1], ContactNormal.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}
		ContactNormal = XMVector3Normalize(ContactNormal);

		c.bias += -BAUMGARTE * inv_dt * min(0.0f, c.separation + LINEAR_SLOP); 
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] BG Bias %.10f", c.bias);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		float Cdot = XMVector3Dot(dVel, ContactNormal).m128_f32[0];
		if (Cdot < -VELOCITY_THRESHOLD)
			c.bias += -restituition * Cdot; 
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] Cdot %.10f BG Bias after restituition %.10f", Cdot, c.bias);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		float term1 =  body1->GetMass()>0.0f ? (1.0f / body1->GetMass()) : 0.0f;
		float term2 =  body2->GetMass()>0.0f ? (1.0f / body2->GetMass()) : 0.0f;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] Body1 inv mass %.10f Body2 inv mass %.10f", term1, term2);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR rnA = XMVector3Cross(rA, ContactNormal);
		XMVECTOR rnB = XMVector3Cross(rB, ContactNormal);
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] rnA %.10f %.10f %.10f rnB %.10f %.10f %.10f", rnA.m128_f32[0], rnA.m128_f32[1], rnA.m128_f32[2], rnB.m128_f32[0], rnB.m128_f32[1], rnB.m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		float K = term1 + term2 + XMVector3Dot(rnA, XMVector4Transform(rnA, body1->GetIInverse())).m128_f32[0] + XMVector3Dot(rnB, XMVector4Transform(rnB, body2->GetIInverse())).m128_f32[0];
		c.massNormal = K > 0.0f ? 1.0f / K : 0.0f;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] Mass Normal %.10f", c.massNormal);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		c.tangents[0] = XMVector3Orthogonal(ContactNormal);
		c.tangents[1] = XMVector3Cross(c.tangents[0], ContactNormal);;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] ContactTangent 0 %.10f %.10f %.10f ContactTangent 1 %.10f %.10f %.10f", c.tangents[0].m128_f32[0], c.tangents[0].m128_f32[1], c.tangents[0].m128_f32[2], c.tangents[1].m128_f32[0], c.tangents[1].m128_f32[1], c.tangents[1].m128_f32[2]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR rt1A = XMVector3Cross(rA, c.tangents[0]);
		XMVECTOR rt1B = XMVector3Cross(rB, c.tangents[0]);

		XMVECTOR rt2A = XMVector3Cross(rA, c.tangents[1]);
		XMVECTOR rt2B = XMVector3Cross(rB, c.tangents[1]);

		float K1 =	term1 + term2 + XMVector3Dot(rt1A, XMVector4Transform(rt1A, body1->GetIInverse())).m128_f32[0] + XMVector3Dot(rt1B, XMVector4Transform(rt1B, body2->GetIInverse())).m128_f32[0];
		float K2 = 	term1 + term2 + XMVector3Dot(rt2A, XMVector4Transform(rt2A, body1->GetIInverse())).m128_f32[0] + XMVector3Dot(rt2B, XMVector4Transform(rt2B, body2->GetIInverse())).m128_f32[0];
		c.massTangent[0] = K1 > 0.0f ? 1.0f / K1 : 0.0f;
		c.massTangent[1] = K2 > 0.0f ? 1.0f / K2 : 0.0f;
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] Mass Tangent 0 %.10f Mass Tangent 1 %.10f", c.massTangent[0], c.massTangent[1]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		XMVECTOR P = c.Pn * ContactNormal + c.Pt[0] * c.tangents[0] + c.Pt[1] * c.tangents[1];
		{
			sprintf_s(cStr, "[ARBITER] [PRESTEP] P %.10f %.10f %.10f Pn %.10f Pt[0] %.10f Pt[1] %.10f", P.m128_f32[0], P.m128_f32[1], P.m128_f32[2], c.Pn, c.Pt[0], c.Pt[1]);
			string str(cStr);
			RELog::GetInstance()->WriteMessage(str);
		}

		body1->AddImpulse(-P);
		body1->AddImpulsiveTorque(-XMVector3Cross(rA, P));
		body2->AddImpulse(P);
		body2->AddImpulsiveTorque(XMVector3Cross(rB, P));
	}
}

float Clamp(float a, float low, float high)
{
	return max(low, min(a, high));
}

void Arbiter::ApplyImpulse()
{
	char cStr[2048];

	for (int i = 0; i < contacts.size(); ++i)
	{
		Contact& c = contacts[i];

		{
			XMVECTOR ContactPos = c.point;

			XMVECTOR rA = XMVector3Transform((ContactPos - body1->GetPosition()), body1->GetOrientation());
			XMVECTOR rB = XMVector3Transform((ContactPos - body2->GetPosition()), body2->GetOrientation());

			XMVECTOR VelocityA = body1->GetMass() > 0 ? (body1->GetVelocityAtCM() + XMVector3Cross(body1->GetAngularVelocity(), rA)) : XMVectorSet(0,0,0,1);
			XMVECTOR VelocityB = body2->GetMass() > 0 ? (body2->GetVelocityAtCM() + XMVector3Cross(body2->GetAngularVelocity(), rB)) : XMVectorSet(0,0,0,1);
			XMVECTOR dVel = VelocityB - VelocityA;

			XMVECTOR ContactNormal = c.normal;
			ContactNormal = XMVector3Transform(ContactNormal, body1->GetOrientation());

			float Val = XMVector3Dot((body2->GetPosition() - body1->GetPosition()), ContactNormal).m128_f32[0];
			if (Val < 0.0f)
				ContactNormal *= -1;
				
			ContactNormal = XMVector3Normalize(ContactNormal);
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Normal Impulse Start");
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}

			float Cdot = XMVector3Dot(dVel, ContactNormal).m128_f32[0];
			float dPn = c.massNormal * (-Cdot + c.bias);
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Cdot %.10f Before dPn %.10f", Cdot, dPn);
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}

			float Pn0 = c.Pn;
			c.Pn = max(Pn0 + dPn, 0.0f);
			dPn = c.Pn - Pn0;
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] After dPn %.10f c.Pn %.10f", dPn, c.Pn);
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}

			XMVECTOR P = dPn * ContactNormal;
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] P %.10f %.10f %.10f", P.m128_f32[0], P.m128_f32[1], P.m128_f32[2]);
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}

			body1->AddImpulse(-P);
			body1->AddImpulsiveTorque(-XMVector3Cross(rA, P));
			body2->AddImpulse(P);
			body2->AddImpulsiveTorque(XMVector3Cross(rB, P));

			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Normal Impulse End");
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}
		}
		{
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Friction Impulse Start");
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}

			for (int z=0;z<2;z++)
			{
				XMVECTOR ContactPos = c.point;

				XMVECTOR rA = XMVector3Transform((ContactPos - body1->GetPosition()), body1->GetOrientation());
				XMVECTOR rB = XMVector3Transform((ContactPos - body2->GetPosition()), body2->GetOrientation());

				XMVECTOR VelocityA = body1->GetMass() > 0 ? (body1->GetVelocityAtCM() + XMVector3Cross(body1->GetAngularVelocity(), rA)) : XMVectorSet(0,0,0,1);
				XMVECTOR VelocityB = body2->GetMass() > 0 ? (body2->GetVelocityAtCM() + XMVector3Cross(body2->GetAngularVelocity(), rB)) : XMVectorSet(0,0,0,1);
				XMVECTOR dVel = VelocityB - VelocityA;

				float vt = XMVector3Dot(dVel, c.tangents[z]).m128_f32[0];
				float dPt = c.massTangent[z] * (-vt);
				{
					sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] vt %.10f Before dPt %.10f", vt, dPt);
					string str(cStr);
					RELog::GetInstance()->WriteMessage(str);
				}

				float maxPt = friction * c.Pn;
				{
					sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] maxPt %.10f c.Pn %.10f", maxPt, c.Pn);
					string str(cStr);
					RELog::GetInstance()->WriteMessage(str);
				}

				float oldTangentImpulse = c.Pt[z];
				c.Pt[z] = Clamp(oldTangentImpulse + dPt, -maxPt, maxPt);
				dPt = c.Pt[z] - oldTangentImpulse;
				{
					sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] After dPt %.10f", dPt);
					string str(cStr);
					RELog::GetInstance()->WriteMessage(str);
				}

				XMVECTOR Pt = dPt * c.tangents[z];
				{
					sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Pt %.10f %.10f %.10f", Pt.m128_f32[0], Pt.m128_f32[1], Pt.m128_f32[2]);
					string str(cStr);
					RELog::GetInstance()->WriteMessage(str);
				}

				body1->AddImpulse(-Pt);
				body1->AddImpulsiveTorque(-XMVector3Cross(rA, Pt));
				body2->AddImpulse(Pt);
				body2->AddImpulsiveTorque(XMVector3Cross(rB, Pt));
			}
			{
				sprintf_s(cStr, "[ARBITER] [APPLYIMPULSE] Friction Impulse End");
				string str(cStr);
				RELog::GetInstance()->WriteMessage(str);
			}
		}
	}
}

You can't just expect random people on the internet to devote their scarce time to debugging your code. I have plenty of my own to work on, plus a real job. Fundamentally, you are the only one who can figure out why your code doesn't work. You need to develop your debugging skills.

Anyway thanks. I too have a job. This is my side prj. I just wanted to check my steps in applyimpusle and prestep were correct.