Hi

I am trying to program shadow volumes and i stumbled upon an artifact which i can not find the cause for.

I generate the shadow volumes using a geometry shader with reversed extrusion (projecting the lightfacing triangles to infinity) and write the stencil buffer according to z-fail. The base of my code is the "lighting" chapter from learnopengl.com, where i extended the shader class to include geometry shader. I also modified the "lightingshader" to draw the ambient pass when "pass" is set to true and the diffuse/ specular pass when set to false. For easier testing i added a view controls to switch on/off the shadow volumes' color rendering or to change the cubes' position, i made the lightnumber controllable and changed the diffuse pass to render green for easier visualization of my problem.

The first picture shows the rendered scene for one point light, all cubes and the front cube's shadow volume is the only one created (intentional). Here, all is rendered as it should be with all lit areas green and all areas inside the shadow volume black (with the volume's sides blended over).

If i now turn on the shadow volumes for all the other cubes, we get a bit of a mess, but its also obvious that some areas that were in shadow before are now erroneously lit (for example the first cube to the right from the originaly shadow volumed cube). From my testing the areas erroneously lit are the ones where more than one shadow volume marks the area as shadowed.

To check if a wrong stencil buffer value caused this problem i decided to change the stencil function for the diffuse pass to only render if the stencil is equal to 2. As i repeated this approach with different values for the stencil function i found out that if i set the value equal to 1 or any other uneven value the lit and shadowed areas are inverted and if i set it to 0 or any other even value i get the results shown above.

This lead me to believe that the value and thus the stencil buffer values may be clamped to [0,1] which would also explain the artifact, because twice in shadow would equal in no shadow at all, but from what i found on the internet and from what i tested with

GLint stencilSize = 0;
glGetFramebufferAttachmentParameteriv(GL_DRAW_FRAMEBUFFER,
    GL_STENCIL, GL_FRAMEBUFFER_ATTACHMENT_STENCIL_SIZE, &stencilSize);

my stencilsize is 8 bit, which should be values within [0,255].

Does anyone know what might be the cause for this artifact or the confusing results with other stencil functions?

// [the following code includes all used gl* functions, other parts are due to readability partialy excluded]

// glfw: initialize and configure
// ------------------------------
glfwInit();
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 4);
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);

// glfw window creation
// --------------------
GLFWwindow* window = glfwCreateWindow(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", NULL, NULL);
if (window == NULL)
{
  cout << "Failed to create GLFW window" << endl;
  glfwTerminate();
  return -1;
}
glfwMakeContextCurrent(window);
glfwSetFramebufferSizeCallback(window, framebuffer_size_callback);
glfwSetCursorPosCallback(window, mouse_callback);
glfwSetScrollCallback(window, scroll_callback);

// tell GLFW to capture our mouse
glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_DISABLED);

// glad: load all OpenGL function pointers
// ---------------------------------------
if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress))
{
  cout << "Failed to initialize GLAD" << endl;
  return -1;
}
// ====================================================================================================
// window and functions are set up
// ====================================================================================================

// configure global opengl state
// -----------------------------
glEnable(GL_DEPTH_TEST);
glEnable(GL_CULL_FACE);

// build and compile our shader program [...]
// set up vertex data (and buffer(s)) and configure vertex attributes [...]
// shader configuration [...]

// render loop
// ===========
while (!glfwWindowShouldClose(window))
{
  // input processing and fps calculation[...]

  // render
  // ------
  glClearColor(0.1f, 0.1f, 0.1f, 1.0f);
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

  glDepthMask(GL_TRUE);																//enable depth writing
  glDepthFunc(GL_LEQUAL);																//avoid z-fighting
  //draw ambient component into color and depth buffer

  view = camera.GetViewMatrix();
  projection = glm::perspective(glm::radians(camera.Zoom), (float)SCR_WIDTH / (float)SCR_HEIGHT, 0.1f, 100.0f);
  // setting up lighting shader for ambient pass [...]

  // render the cubes
  glBindVertexArray(cubeVAO);
  for (unsigned int i = 0; i < 10; i++)
  {
    //position cube [...]
    glDrawArrays(GL_TRIANGLES, 0, 36);
  }

  //------------------------------------------------------------------------------------------------------------------------
  glDepthMask(GL_FALSE);															//disable depth writing
  glEnable(GL_BLEND);
  glBlendFunc(GL_ONE, GL_ONE);														//additive blending
  glEnable(GL_STENCIL_TEST);

  //setting up shadowShader and lightingShader [...]

  for (int light = 0; light < lightsused; light++)
  {
    glDepthFunc(GL_LESS);
    glClear(GL_STENCIL_BUFFER_BIT);
    //configure stencil ops for front- and backface to write according to z-fail
    glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_DECR_WRAP, GL_KEEP);			//-1 for front-facing
    glStencilOpSeparate(GL_BACK, GL_KEEP, GL_INCR_WRAP, GL_KEEP);			//+1 for back-facing
    glStencilFunc(GL_ALWAYS, 0, GL_TRUE);											//stencil test always passes
    if(hidevolumes)
      glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);						//disable writing to the color buffer
    glDisable(GL_CULL_FACE);
    glEnable(GL_DEPTH_CLAMP);															//necessary to render SVs into infinity

    //draw SV-------------------
    shadowShader.use();
    shadowShader.setInt("lightnr", light);

    int nr;
    if (onecaster)
      nr = 1;
    else
      nr = 10;

    for (int i = 0; i < nr; i++)
    {
      //position cube[...]
      glDrawArrays(GL_TRIANGLES, 0, 36);
    }
    //--------------------------
    glDisable(GL_DEPTH_CLAMP);
    glEnable(GL_CULL_FACE);

    glStencilFunc(GL_EQUAL, 0, GL_TRUE);							//stencil test passes for ==0 so only for non shadowed areas
    glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);							//keep stencil values for illumination
    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);				//enable writing to the color buffer
    glDepthFunc(GL_LEQUAL);											//avoid z-fighting

    //draw diffuse and specular pass
    lightingShader.use();
    lightingShader.setInt("lightnr", light);

    // render the cubes
    for (unsigned int i = 0; i < 10; i++)
    {
      //position cube[...]
      glDrawArrays(GL_TRIANGLES, 0, 36);
    }
  }

  glDisable(GL_BLEND);
  glDepthMask(GL_TRUE);																//enable depth writing
  glDisable(GL_STENCIL_TEST);
  //------------------------------------------------------------------------------------------------------------------------

  // also draw the lamp object(s) [...]

  // glfw: swap buffers and poll IO events (keys pressed/released, mouse moved etc.)
  // -------------------------------------------------------------------------------
  glfwSwapBuffers(window);
  glfwP
}

// optional: de-allocate all resources once they've outlived their purpose:
// ------------------------------------------------------------------------
glDeleteVertexArrays(1, &cubeVAO);
glDeleteVertexArrays(1, &lightVAO);
glDeleteBuffers(1, &VBO);

// glfw: terminate, clearing all previously allocated GLFW resources.
// ------------------------------------------------------------------
glfwTerminate();
return 0;

I figured out what caused the stencil function to interpret the value in the stencil buffer as binary.

void StencilFunc(enum func, int ref, uint mask);

I set mask to GL_TRUE, as i saw in a later part of the tutorial, thinking that this would result in 0xFF as mask, but it instead set it to 0x01, which caused the binary behavior.