For future reference/ if others ever want to try this out 🙂
I fixed the near plane, which is now a weighted average of the quads that are facing the camera. Here's the (not yet optimized) code:
std::vector<CR_VECTOR4> CMathHelper::CreateOccluderPlanes(const CR_VECTOR3 &pCameraPos, const CR_VECTOR3 *pCorners, const CR_VECTOR3 &pCenter, std::vector<CR_VECTOR3> &pPlanesSigns, std::vector<CR_VECTOR3> &pDebugPoints)
{
assert(pCorners != nullptr);
pPlanesSigns.clear();
std::vector<int> cornerIds = { 0, 1, 2, 3, 3, 2, 6, 7, 7, 6, 5, 4, 4, 5, 1, 0, 0, 3, 7, 4, 1, 5, 6, 2 }; // 6*4 = 24
float nearPlanesArea = 0.0f;
std::vector<float> nearQuadFactors;
std::vector<CR_VECTOR3> nearQuadNormals;
std::vector<float> nearQuadAreas;
std::set<CR_VECTOR2I> visibleEdges;
std::vector<CR_VECTOR4> planes;
std::vector<unsigned int> silhouetteCorners;
// find quads facing the camera
for(size_t verts=0;verts<cornerIds.size();verts+=4)
{
float factor = DotProductVec3(CalculateFaceNormal(pCorners[cornerIds[verts]], pCorners[cornerIds[verts+1]], pCorners[cornerIds[verts+2]]), pCameraPos - pCorners[cornerIds[verts+1]]);
if(factor > 0)
{
nearQuadFactors.emplace_back(factor);
nearQuadNormals.emplace_back(CalculateFaceNormal(pCorners[cornerIds[verts]], pCorners[cornerIds[verts+1]], pCorners[cornerIds[verts+2]]));
float quadArea = GetQuadArea(pCorners[cornerIds[verts]], pCorners[cornerIds[verts+1]], pCorners[cornerIds[verts+2]], pCorners[cornerIds[verts+3]]) * factor;
nearPlanesArea += quadArea;
nearQuadAreas.emplace_back(quadArea);
silhouetteCorners.emplace_back(cornerIds[verts]);
silhouetteCorners.emplace_back(cornerIds[verts+1]);
silhouetteCorners.emplace_back(cornerIds[verts+2]);
silhouetteCorners.emplace_back(cornerIds[verts+3]);
std::vector<CR_VECTOR2I> newEdges;
newEdges.emplace_back(CR_VECTOR2I(cornerIds[verts], cornerIds[verts+1]));
newEdges.emplace_back(CR_VECTOR2I(cornerIds[verts+1], cornerIds[verts+2]));
newEdges.emplace_back(CR_VECTOR2I(cornerIds[verts+2], cornerIds[verts+3]));
newEdges.emplace_back(CR_VECTOR2I(cornerIds[verts+3], cornerIds[verts]));
for(auto && edge : newEdges)
{
edge.LowToHigh();
auto it = visibleEdges.find(edge);
if(it != visibleEdges.end())
{
visibleEdges.erase(it);
}
else
{
visibleEdges.insert(edge);
}
}
}
}
if(silhouetteCorners.size() == 0)
{
return planes;
}
// calculate weighted near plane
CR_VECTOR3 nearFaceNormal;
for(size_t q=0;q<nearQuadAreas.size();++q)
{
nearFaceNormal += nearQuadNormals[q] * nearPlanesArea / (nearQuadAreas[q] * nearQuadFactors[q]);
}
nearFaceNormal *= -1.0f; // should face aways from camera
nearFaceNormal.Normalize();
// find furthest silhouette vertex
unsigned int furthestVertex = 0;
float maxDist = CoordToCoordDist(pCorners[silhouetteCorners[0]], pCameraPos);
for(int s=1;s<silhouetteCorners.size();++s)
{
float newDist = CoordToCoordDist(pCorners[silhouetteCorners[s]], pCameraPos);
if(newDist > maxDist)
{
furthestVertex = s;
}
}
float d = -DotProductVec3(pCorners[silhouetteCorners[furthestVertex]], nearFaceNormal);
planes.emplace_back(CR_VECTOR4(nearFaceNormal.x, nearFaceNormal.y, nearFaceNormal.z, d));
pPlanesSigns.emplace_back(GetSigns(nearFaceNormal));
// continue with 'sides' of the volume, based on the edges without duplicates
for(auto && tEdge : visibleEdges)
{
CR_VECTOR3 faceNormal = CalculateFaceNormal(pCameraPos, pCorners[tEdge.x], pCorners[tEdge.y]);
if(!TriangleFacingPoint(faceNormal, pCameraPos, pCenter))
{
faceNormal *= -1.0f;
}
faceNormal.Normalize();
float d = -DotProductVec3(pCameraPos, faceNormal);
planes.emplace_back(CR_VECTOR4(faceNormal.x, faceNormal.y, faceNormal.z, d));
pPlanesSigns.emplace_back(GetSigns(faceNormal));
}
return planes;
}
And a quick recording of the result: