My ultimate aim is to determine a sequence of (stencil) operations so that regardless of the order in which objects are rendered I can end up with the visible (non-occluded) part of any object. To this end, I have written some code where I am displaying two cubes one in front of the other. The smaller one is blocking partially the larger one. What I want to retrieve is the part of the larger cube that is visible. At the moment I am not able to accomplish this. The code is written in python using PyOpengl.

The code shown here is used to create two renderings and save them on disk. The first rendering shows the distance from the camera and the second one is just the stencil buffer. I generate both renderings side by side in order to make sure that my work is okay.

Here is the code I am using (except for the shader class which is a generic class that handles compilation and passing data to a shader),

from OpenGL.GL import *
from glfw.GLFW import *
from glfw import _GLFWwindow as GLFWwindow
import glm
from shader_m import Shader
import ctypes
import numpy as np

# settings
SCR_WIDTH = 800
SCR_HEIGHT = 800

def main() -> int:

   # glfw: initialize and configure
   # ------------------------------
   glfwInit()
   glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3)
   glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3)
   glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE)

   # glfw window creation
   # --------------------
   window = glfwCreateWindow(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", None, None)
   if (window == None):

       print("Failed to create GLFW window")
       glfwTerminate()
       return -1

   glfwMakeContextCurrent(window)

   # configure global opengl state
   # -----------------------------
   glEnable(GL_DEPTH_TEST)
   glEnable(GL_STENCIL_TEST)

   # build and compile shaders
   # -------------------------
   shaderDepth = Shader("2.stencil_depth.vs", "2.stencil_depth.fs")

   # set up vertex data (and buffer(s)) and configure vertex attributes
   # ------------------------------------------------------------------
   cubeVertices = glm.array(glm.float32,
       # positions          # texture Coords
       -0.5, -0.5, -0.5,  0.0, 0.0,
        0.5, -0.5, -0.5,  1.0, 0.0,
        0.5,  0.5, -0.5,  1.0, 1.0,
        0.5,  0.5, -0.5,  1.0, 1.0,
       -0.5,  0.5, -0.5,  0.0, 1.0,
       -0.5, -0.5, -0.5,  0.0, 0.0,

       -0.5, -0.5,  0.5,  0.0, 0.0,
        0.5, -0.5,  0.5,  1.0, 0.0,
        0.5,  0.5,  0.5,  1.0, 1.0,
        0.5,  0.5,  0.5,  1.0, 1.0,
       -0.5,  0.5,  0.5,  0.0, 1.0,
       -0.5, -0.5,  0.5,  0.0, 0.0,

       -0.5,  0.5,  0.5,  1.0, 0.0,
       -0.5,  0.5, -0.5,  1.0, 1.0,
       -0.5, -0.5, -0.5,  0.0, 1.0,
       -0.5, -0.5, -0.5,  0.0, 1.0,
       -0.5, -0.5,  0.5,  0.0, 0.0,
       -0.5,  0.5,  0.5,  1.0, 0.0,

        0.5,  0.5,  0.5,  1.0, 0.0,
        0.5,  0.5, -0.5,  1.0, 1.0,
        0.5, -0.5, -0.5,  0.0, 1.0,
        0.5, -0.5, -0.5,  0.0, 1.0,
        0.5, -0.5,  0.5,  0.0, 0.0,
        0.5,  0.5,  0.5,  1.0, 0.0,

       -0.5, -0.5, -0.5,  0.0, 1.0,
        0.5, -0.5, -0.5,  1.0, 1.0,
        0.5, -0.5,  0.5,  1.0, 0.0,
        0.5, -0.5,  0.5,  1.0, 0.0,
       -0.5, -0.5,  0.5,  0.0, 0.0,
       -0.5, -0.5, -0.5,  0.0, 1.0,

       -0.5,  0.5, -0.5,  0.0, 1.0,
        0.5,  0.5, -0.5,  1.0, 1.0,
        0.5,  0.5,  0.5,  1.0, 0.0,
        0.5,  0.5,  0.5,  1.0, 0.0,
       -0.5,  0.5,  0.5,  0.0, 0.0,
       -0.5,  0.5, -0.5,  0.0, 1.0)

   # cube VAO
   cubeVAO = glGenVertexArrays(1)
   cubeVBO = glGenBuffers(1)
   glBindVertexArray(cubeVAO)
   glBindBuffer(GL_ARRAY_BUFFER, cubeVBO)
   glBufferData(GL_ARRAY_BUFFER, cubeVertices.nbytes, cubeVertices.ptr, GL_STATIC_DRAW)
   glEnableVertexAttribArray(0)
   glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * glm.sizeof(glm.float32), None)
   glEnableVertexAttribArray(1)
   glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * glm.sizeof(glm.float32), ctypes.c_void_p(3 * glm.sizeof(glm.float32)))
   glBindVertexArray(0)

   # shader configuration
   # --------------------
   shaderDepth.use()

   # create framebuffers
   fbo = create_framebuffer()

   # bind the framebuffer
   glBindFramebuffer(GL_FRAMEBUFFER, fbo)

   # define basic viewing parameters
   camera_zoom = 45
   camera_position = glm.vec3(0.0, 0.0, 4.0)
   camera_front = glm.vec3(0.0, 0.0, -1.0)
   camera_up = glm.vec3(0., 1., 0)
   view = glm.lookAt(camera_position, camera_position + camera_front, camera_up)
   projection = glm.perspective(glm.radians(camera_zoom), SCR_WIDTH / SCR_HEIGHT, 0.1, 100.0)  
       
   # set uniforms on shader
   shaderDepth.setMat4("view", view)
   shaderDepth.setMat4("projection", projection)

   # clear colors and buffers
   glClearColor(0.1, 0.1, 0.1, 1.0)
   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT)  

   # Setting stencil buffer
   # --------------------------------------------------------------------
   glStencilMask(0xFF) # where am I writing
   glStencilFunc(GL_ALWAYS, 1, 0xFF) # All fragments will pass
   glStencilOp(GL_KEEP,    # stencil fails
               GL_REPLACE,    # stencil passes, depth fails
               GL_KEEP)    # stencil and depth passes

   # cube 1
   glBindVertexArray(cubeVAO)
   model = glm.mat4(1.0)
   model = glm.translate(model, glm.vec3(0.0, 0.0, -0.75))
   shaderDepth.setMat4("model", model)
   glDrawArrays(GL_TRIANGLES, 0, 36)
   
   # set it so that we do not draw the next to the stencil
   glStencilMask(0x00)
   # cube 2
   scale = 0.25
   model = glm.mat4(1.0)
   model = glm.scale(model, glm.vec3(scale, scale, scale))
   model = glm.translate(model, glm.vec3(0.0, 0, 2.0))
   shaderDepth.setMat4("model", model)
   glDrawArrays(GL_TRIANGLES, 0, 36)
   glBindVertexArray(0)
           
   # OUTPUTS                  
   #depth
   depth = glReadPixelsf(0,0, SCR_WIDTH, SCR_HEIGHT, GL_RED, GL_FLOAT)
   np.save('depth', np.flipud(depth))
   
   # stencil
   stencil = glReadPixels(0,0, SCR_WIDTH, SCR_HEIGHT, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE)
   stencil = np.frombuffer(stencil, dtype=np.uint8).reshape((800,-1))
   np.save('stencil', np.flipud(stencil))

   # unbind framebuffer back to default
   glBindFramebuffer(GL_FRAMEBUFFER,0)
   
   glfwSwapBuffers(window)
   glfwPollEvents()

   glDeleteVertexArrays(1, (cubeVAO,))
   glDeleteBuffers(1, (cubeVBO,))

   glfwTerminate()
   return 0

The framebuffer is generated using the following code,

def create_framebuffer():

   # create depth buffer
   fbo= glGenFramebuffers(1)
   glBindFramebuffer(GL_FRAMEBUFFER, fbo)

   # COLOR DEPTH  
   color_depth_tex = glGenTextures(1)
   glBindTexture(GL_TEXTURE_2D, color_depth_tex)
   glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, SCR_WIDTH, SCR_HEIGHT, 0, GL_RED, GL_FLOAT, None)
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
   glBindTexture(GL_TEXTURE_2D, 0)
 
   # Attach depth to framebuffer
   glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, color_depth_tex, 0)

   # DEPTH  
   depth_tex = glGenTextures(1)
   glBindTexture(GL_TEXTURE_2D, depth_tex)
   glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT32F, SCR_WIDTH, SCR_HEIGHT, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
   glBindTexture(GL_TEXTURE_2D, 0)

   # Attach depth to framebuffer
   glFramebufferTexture(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, depth_tex, 0)

   # STENCIL
   stencil_tex = glGenTextures(1)
   glBindTexture(GL_TEXTURE_2D, stencil_tex)
   glTexImage2D(GL_TEXTURE_2D, 0, GL_STENCIL_INDEX8, SCR_WIDTH, SCR_HEIGHT, 0, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, None)
   glBindTexture(GL_TEXTURE_2D, 0)

   # Attach stencil to framebuffer
   glFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, stencil_tex, 0)

   # check for errors (after attaching a texture)
   if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE):
       print(glCheckFramebufferStatus(GL_FRAMEBUFFER))
       raise RuntimeError("ERROR.FRAMEBUFFER. Framebuffer is not complete!")
   
   # unbind framebuffer
   glBindFramebuffer(GL_FRAMEBUFFER, 0)

   return fbo

The shader I use is very simple,

*shaderDepth*

#version 330 core
layout (location = 0) in vec3 aPos;
uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;
out vec4 pos;
void main(){
   pos = view * model * vec4(aPos, 1.0);    
   gl_Position = projection * view * model * vec4(aPos, 1.0);
}

#version 330 core
in vec4 pos;
out float depth;
void main(){
   depth = length(pos.xyz);
}

Here are two renderings of what gets saved. As mentioned, the one on the left shows distance to the two cubes. The one to the right shows what is the result of the stencil operations above.

The small cube that we can see on the left is clearly in front of the large cube yet the output I get on the right does not seem to correspond at all with what I would expect based on the left image. There is definitely a disconnect between what I am rendering and the result I am getting from the stencil buffer. My expectation is that all fragments from the large cube pass the stencil test but those blocked by the small cube fail the depth test, however, this is not what I am getting.

I am trying to figure out what I am doing wrong?

This is not that easy. If you want to test for any objects you rendered (as you seemed to mention), then you'll have to do the stencil test for each object you want to test its visibility.

Apart from this, for start, clear your stencil buffer with 0, and replace whenever depth test succeed with 1. You should start to see something more relevant.

And as a side note, occlusion queries (see the extensions) are meant for what you're trying to do.

Honestly, if this is just for a 2D simple thing, you can just make your objects the same color as the background and render them on top. This will clear them all to the background color (purple) and look like it's cutting out portions.

To do this I would render the smaller cube first, which fills in some stencil information and then when rendering the bigger cube put the stencil operation to not always pass, but to fail if less than.

Thanks @_Silence_

Thank you for the tip on the ‘occlusion query’.

My idea was to have the stencil buffer on but only allow certain objects to be drawn to it by changing the value of the stencil mask. The logic is as follows,

1. Set the stencil mask to allow everything to be drawn
2. Set the stencil test to always pass
3. Set the stencil op to draw in stencil buffer ONLY those fragments that pass the stencil test but fail the depth test
4. Draw my large cube
5. Set the stencil mask to 0 (so no drawing to the stencil buffer)
6. Draw small cube in front of large cube

What I am confused about is the StencilOp() function. I have managed to get the entire large cube drawn to the stencil buffer, no problem. But what I am after are the parts of the large cube that are not blocked by the small cube in front. I thought this would be achieve by only drawing to the stencil buffer whenever the stencil test passes BUT the depth test fails. But the result I get does not appear to fit well with my expectation (which in this case would be a large cube with a square hole in the middle due to the small cube being in front).

Hope this helps explain a bit better.

Thanks @undefined !

Interesting thought! So you are recommending to reverse the order of how the stencil buffer is updated?

@mllobera

If I understood you correctly:

1. Render your scene normally, without the object you want to test
2. Enable stencil, clear stencil with 0, set stencil op to keep, keep, replace, and stencil func to (always,1,1)
3. Draw the object you need its visible pixels

You should then have the stencil buffer filled with 1 where your object is visible, and 0 everywhere else.

Thanks @_Silence_ I will give it a try!

@_Silence_ I tried your suggestion. Basically I rearranged the code above to do what you have suggested. Here is what I changed,

    # cube 2 small cube
    glBindVertexArray(cubeVAO)
    scale = 0.25
    model = glm.mat4(1.0)
    model = glm.scale(model, glm.vec3(scale, scale, scale))
    model = glm.translate(model, glm.vec3(0.0, 0, 3.5))
    shaderDepth.setMat4("model", model)
    glDrawArrays(GL_TRIANGLES, 0, 36)
   
    # clear colors and buffers
    glEnable(GL_STENCIL_TEST)
    glClearColor(0.1, 0.1, 0.1, 1.0)
    glClearStencil(0) # stencil value used when it is cleared
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT)


    # Setting stencil buffer
    # --------------------------------------------------------------------
    glStencilFunc(GL_ALWAYS, 1, 1) # All fragments will pass
    glStencilOp(GL_KEEP,    # stencil fails
                GL_KEEP,    # stencil passes, depth fails
                GL_REPLACE)    # stencil and depth passes
   
    # cube 1 large cube
    # glBindVertexArray(cubeVAO)
    model = glm.mat4(1.0)
    model = glm.translate(model, glm.vec3(0.0, 0.0, -1.75))
    shaderDepth.setMat4("model", model)
    glDrawArrays(GL_TRIANGLES, 0, 36)
    glBindVertexArray(0)

Unfortunately, the result is not as expected again,

Why clearing color and depth after rendering your first cube ? Basically, you're just forgetting everything you did prior to this point.

Clear color and depth before rendering the first cube, then only clear stencil before drawing the second cube.

Thanks @_Silence_. You are right I missed that part. Your suggestion worked,

I am still uncertain what was wrong with my previous thinking. Why that was not working!?? I also have to check to see how this would extend if there were several objects that I was interested in.