How to add 3D AUDIO to Games & Graphics

We need 3D audio in our games, but what is it exactly? You’re probably familiar with stereophonic (stereo) systems which produce different sounds in each ear.

Those different sounds typically consist of different volumes of the same single source/environment, which is then sent to each ear. But we can of course play a completely separate track to each ear, e.g. a two-channel stereo system could have one person's voice in the left ear and another person's voice in the right ear.

You’ve probably also heard of surround sound (AKA audio spatialisation). Well, 3D audio is just another term for audio spatialisation, i.e. each term is referring to the same thing, it’s just that people typically use different terms. Surround sound systems are better than two-channel stereo systems – they have up to 5, or even 7 audio channels. We’ve also got monophonic (mono), which simply gives us the exact same sound in each ear, for example there are lots of mono microphones available that record your voice to a single track.

Perhaps surprisingly, any audio input source played by the SFML audio module needs to be in mono – sounds and music when in stereo (or indeed more than two channels) already have a deliberately designed unique audio and volume for each speaker. Therefore, if we try to create our own 3D audio from a stereo source, we’ll basically be ruining the intended sound of the stereo source (even though we might get some desired results). The developers therefore decided to not implement the option of using stereophonic sources with audio spatialisation.

Finally, the SFML audio module is based on OpenAL Soft, and so what awesomeness of surround can we get from that… well here’s a quote from Wikipedia… “OpenAL Soft supports mono, stereo (including HRTF and UHJ), 4-channel, 5.1, 6.1, 7.1, and B-Format output. Ambisonic assets are supported.”


Source code: C++... main.cpp

File: main.cpp
  1. #include <glad/glad.h> // GLAD: https://github.com/Dav1dde/glad
  2. #include <GLFW/glfw3.h>
  3.  
  4. #define STB_IMAGE_IMPLEMENTATION
  5. #include "stb_image.h"
  6.  
  7. // OpenGL Mathematics(GLM) https://github.com/g-truc/glm/blob/master/manual.md
  8. // -----------------------
  9. // GLM Headers
  10. // -----------
  11. #include <glm/glm.hpp>   
  12. #include <glm/gtc/matrix_transform.hpp>
  13. #include <glm/gtc/type_ptr.hpp>
  14.  
  15. #include <assimp/Importer.hpp>
  16. #include <assimp/scene.h>
  17. #include <assimp/postprocess.h>
  18.  
  19. #include <vector>
  20. #include <iostream>
  21. #include <fstream> // Used in "shader_configure.h" to read the shader text files
  22.  
  23. #include "load_model_meshes.h"
  24. #include "shader_configure.h" // Used to create the shaders
  25.  
  26. #include <SFML\Audio.hpp>
  27.  
  28. int main()
  29. {
  30.     // (1) GLFW: Initialise & Configure
  31.     // --------------------------------
  32.     if (!glfwInit())
  33.         exit(EXIT_FAILURE);
  34.  
  35.     glfwWindowHint(GLFW_SAMPLES, 4); // Anti-aliasing
  36.     glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
  37.     glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
  38.     glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
  39.    
  40.     const GLFWvidmode* mode = glfwGetVideoMode(glfwGetPrimaryMonitor());
  41.  
  42.     int monitor_width = mode->width; // Monitor's width
  43.     int monitor_height = mode->height;
  44.  
  45.     int window_width = (int)(monitor_width * 0.90f); // Window size will be 50% the monitor's size...
  46.     int window_height = (int)(monitor_height * 0.90f); // ... Cast is simply to silence the compiler warning
  47.  
  48.     GLFWwindow* window = glfwCreateWindow(window_width, window_height, "OpenAL 3D Spatial Audio - Surround Sound", NULL, NULL);
  49.    
  50.     if (!window)
  51.     {
  52.         glfwTerminate();
  53.         exit(EXIT_FAILURE);
  54.     }
  55.     glfwMakeContextCurrent(window); // Set the window to be used and then centre that window on the monitor
  56.     glfwSetWindowPos(window, (monitor_width - window_width) / 2, (monitor_height - window_height) / 2);
  57.  
  58.     glfwSwapInterval(1); // Set VSync rate 1:1 with monitor's refresh rate
  59.    
  60.     // (2) GLAD: Load OpenGL Function Pointers
  61.     // ---------------------------------------
  62.     if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress)) // For GLAD 2 use the following instead: gladLoadGL(glfwGetProcAddress)
  63.     {
  64.         glfwTerminate();
  65.         exit(EXIT_FAILURE);
  66.     }
  67.     glEnable(GL_DEPTH_TEST); // Enabling depth testing allows rear faces of 3D objects to be hidden behind front faces
  68.     glEnable(GL_MULTISAMPLE); // Anti-aliasing
  69.     glEnable(GL_BLEND); // GL_BLEND for OpenGL transparency which is further set within the fragment shader
  70.     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);   
  71.  
  72.     // (3) Compile Shaders Read from Text Files
  73.     // ----------------------------------------
  74.     const char* vert_shader = "../../Shaders/shader_glsl.vert";
  75.     const char* frag_shader = "../../Shaders/shader_glsl.frag";
  76.  
  77.     Shader main_shader(vert_shader, frag_shader);
  78.     main_shader.use();   
  79.    
  80.     unsigned int view_matrix_loc = glGetUniformLocation(main_shader.ID, "view");
  81.     unsigned int projection_matrix_loc = glGetUniformLocation(main_shader.ID, "projection");
  82.     unsigned int camera_position_loc = glGetUniformLocation(main_shader.ID, "camera_position");
  83.  
  84.     unsigned int heli_mat_loc = glGetUniformLocation(main_shader.ID, "heli_mat");
  85.     unsigned int plane_mat_loc = glGetUniformLocation(main_shader.ID, "plane_mat");
  86.  
  87.     unsigned int model_num_loc = glGetUniformLocation(main_shader.ID, "model_num");
  88.     unsigned int image_sampler_loc = glGetUniformLocation(main_shader.ID, "image");
  89.  
  90.     // (4) Load Models & Set Camera
  91.     // ----------------------------
  92.     Model heli("The_Beast_Helicopter.obj");
  93.     Model plane("Plane_CAP_232.obj");
  94.  
  95.     glm::vec3 camera_position(0.0f, 0.0f, 15.0f); // -Z is into the screen   
  96.     glm::vec3 camera_target(0.0f, 0.0f, 0.0f);
  97.     glm::vec3 camera_up(0.0f, 1.0f, 0.0f);
  98.  
  99.     glActiveTexture(GL_TEXTURE0); // Reusing the same texture unit for each plane mesh
  100.     glUniform1i(image_sampler_loc, 0);
  101.  
  102.     glUniform3f(camera_position_loc, camera_position.x, camera_position.y, camera_position.z);
  103.  
  104.     glm::mat4 view = glm::lookAt(camera_position, camera_target, camera_up);
  105.     glUniformMatrix4fv(view_matrix_loc, 1, GL_FALSE, glm::value_ptr(view)); // Transfer view matrix to vertex shader uniform
  106.  
  107.     glm::mat4 projection = glm::perspective(glm::radians(55.0f), (float)window_width / (float)window_height, 0.1f, 100.0f);
  108.     glUniformMatrix4fv(projection_matrix_loc, 1, GL_FALSE, glm::value_ptr(projection));
  109.  
  110.     // (5) Model Initial Positions
  111.     // ---------------------------
  112.     glm::vec3 heli_init_pos(0.0f, 0.0f, 0.0f);
  113.     glm::mat4 heli_mat(1.0f);
  114.     heli_mat = glm::translate(heli_mat, heli_init_pos); // Translate to initial position   
  115.     heli_mat = glm::rotate(heli_mat, glm::radians(-180.0f), glm::normalize(glm::vec3(0, 1, 0)));
  116.     heli_mat = glm::rotate(heli_mat, glm::radians(-90.0f), glm::normalize(glm::vec3(1, 0, 0)));
  117.  
  118.     glm::vec3 heli_dir = glm::mat3(heli_mat) * glm::vec3(0, 0, -1); 
  119.  
  120.     glm::vec3 plane_init_pos(0.0f, 0.0f, 0.0f);
  121.     glm::mat4 plane_mat(1.0f);   
  122.     plane_mat = glm::translate(plane_mat, plane_init_pos);
  123.     //plane_mat = glm::rotate(plane_mat, glm::radians(90.0f), glm::normalize(glm::vec3(0, 1, 0)));
  124.  
  125.     // (6) Configure 3D Spatial Audio
  126.     // ------------------------------   
  127.     sf::Music music;
  128.     music.openFromFile("The Trapezist - Quincas Moreira (Mono).ogg"); // Specialization works for Mono audio but not Stereo audio
  129.    
  130.     sf::Listener::setPosition(heli_init_pos.x, heli_init_pos.y, heli_init_pos.z);
  131.     sf::Listener::setDirection(heli_dir.x, heli_dir.y, heli_dir.z); // facing into the screen
  132.     //sf::Listener::setDirection(0.0f, 0.0f, 1.0f); // facing out of the screen
  133.     //sf::Listener::setDirection(1.0f, 0.0f, 0.0f); // facing to the right
  134.     //sf::Listener::setDirection(-1.0f, 0.0f, 0.0f); // facing to the left
  135.  
  136.     //music.setPosition(0.0f, 0.0f, -5.0f);
  137.     //music.setRelativeToListener(false);
  138.     music.setMinDistance(1.0f);
  139.     music.setAttenuation(0.75f);
  140.  
  141.     music.setVolume(100.0f);
  142.     music.setPitch(1.0f);
  143.     music.setLoop(true);
  144.  
  145.     music.play();
  146.    
  147.     float movement = -0.2f;
  148.  
  149.     while (!glfwWindowShouldClose(window)) // Main-Loop
  150.     {   
  151.         // (7) Transform Aeroplane in a Straight Line
  152.         // ------------------------------------------
  153.         plane_mat = glm::translate(plane_mat, glm::vec3(0.0f, movement, 0.0f));
  154.        
  155.         // (8) Calculate the Transformation Matrix for the Orbiting Aeroplane
  156.         // ------------------------------------------------------------------
  157.         // (Temporarily moving back to centre (0, 0, 0) before rotating, and then returning along a different path)
  158.         //plane_mat = glm::translate(plane_mat, -plane_init_pos);
  159.         //plane_mat = glm::rotate(plane_mat, glm::radians(3.5f), glm::vec3(0, 0, 1)); // Rotate around the plane's local axis
  160.         //plane_mat = glm::rotate(plane_mat, glm::radians(-3.5f), glm::vec3(1, 0, 0));
  161.         //plane_mat = glm::translate(plane_mat, plane_init_pos);
  162.  
  163.         glm::vec3 plane_new_pos = plane_mat * glm::vec4(0.0f, 0.0f, 0.0f, 1.0f);
  164.         std::cout << "\norbit_1_pos.x: " << plane_new_pos.x << " --- plane_new_pos.y: " << plane_new_pos.y << " --- plane_new_pos.z: " << plane_new_pos.z;
  165.            
  166.         music.setPosition(plane_new_pos.x, plane_new_pos.y, plane_new_pos.z);
  167.  
  168.         if (plane_new_pos.y < -10 || plane_new_pos.y > 10)
  169.             movement = -movement;
  170.  
  171.         glUniformMatrix4fv(heli_mat_loc, 1, GL_FALSE, glm::value_ptr(heli_mat)); // Pass model matrix to vertex shader
  172.         glUniformMatrix4fv(plane_mat_loc, 1, GL_FALSE, glm::value_ptr(plane_mat));
  173.        
  174.         // (9) Clear the Screen & Draw Model Meshes
  175.         // ----------------------------------------
  176.         glClearColor(0.30f, 0.55f, 0.65f, 1.0f);
  177.         glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
  178.  
  179.         glUniform1i(model_num_loc, 1);
  180.         for (unsigned int i = 0; i < plane.num_meshes; ++i)
  181.         {           
  182.             glBindTexture(GL_TEXTURE_2D, plane.mesh_list[i].tex_handle); // Bind texture for the current mesh
  183.  
  184.             glBindVertexArray(plane.mesh_list[i].VAO);
  185.             glDrawElements(GL_TRIANGLES, (GLsizei)plane.mesh_list[i].vert_indices.size(), GL_UNSIGNED_INT, 0);
  186.             glBindVertexArray(0);
  187.         }
  188.         glUniform1i(model_num_loc, 2);
  189.         for (unsigned int i = 0; i < heli.num_meshes; ++i)
  190.         {
  191.             glBindTexture(GL_TEXTURE_2D, heli.mesh_list[i].tex_handle); // Bind texture for the current mesh
  192.  
  193.             glBindVertexArray(heli.mesh_list[i].VAO);
  194.             glDrawElements(GL_TRIANGLES, (GLsizei)heli.mesh_list[i].vert_indices.size(), GL_UNSIGNED_INT, 0);
  195.             glBindVertexArray(0);
  196.         }
  197.         glfwSwapBuffers(window);
  198.         glfwPollEvents();
  199.     }
  200.  
  201.     // (10) Exit the Application
  202.     // -------------------------
  203.     glDeleteProgram(main_shader.ID); // This OpenGL function call is talked about in: shader_configure.h
  204.  
  205.     /* glfwDestroyWindow(window) // Call this function to destroy a specific window */   
  206.     glfwTerminate(); // Destroys all remaining windows and cursors, restores modified gamma ramps, and frees resources
  207.  
  208.     exit(EXIT_SUCCESS); // Function call: exit() is a C/C++ function that performs various tasks to help clean up resources
  209. }

Source code: C++... shader_configure.h & load_model_meshes.h

File shader_configure.h and file load_model_meshes.h can be found here: OpenGL Tutorial 5 (Quick Start) – Model Loading – Assimp Blender & Lighting


Source code: GLSL... shader_glsl.vert (vertex shader)

File: shader_glsl.vert
  1. #version 420 core
  2.  
  3. layout (location = 0) in vec3 aPos; // Attribute data: vertex(s) X, Y, Z position via VBO set up on the CPU side
  4. layout (location = 1) in vec3 aNormal;
  5. layout (location = 2) in vec2 aTexCoord;
  6.  
  7. out vec3 vertex_normal;
  8. out vec3 vert_pos_transformed; // Transformed model vertex position values passed as interpolated
  9. out vec2 texture_coordinates;
  10.  
  11. uniform mat4 view;
  12. uniform mat4 projection;
  13.  
  14. uniform mat4 heli_mat;
  15. uniform mat4 plane_mat;
  16.  
  17. uniform int model_num;
  18.  
  19. void main()
  20. {
  21.     mat4 mat_trans = mat4(1);
  22.  
  23.     if (model_num == 1)
  24.         mat_trans = plane_mat;
  25.  
  26.     if (model_num == 2)
  27.         mat_trans = heli_mat;   
  28.    
  29.     vert_pos_transformed = vec3(mat_trans * vec4(aPos, 1.0)); // Transformed position values used for the lighting effects
  30.     texture_coordinates = aTexCoord;
  31.  
  32.     mat3 normal_matrix = transpose(inverse(mat3(mat_trans)));
  33.     vertex_normal = normal_matrix * aNormal;
  34.    
  35.     if (length(vertex_normal) > 0)
  36.         vertex_normal = normalize(vertex_normal); // Never try to normalise zero vectors (0,0,0)   
  37.  
  38.     gl_Position = projection * view * mat_trans * vec4(aPos, 1.0);
  39. }

Source code: GLSL... shader_glsl.frag (fragment shader)

File: shader_glsl.frag
  1. #version 420 core
  2.  
  3. out vec4 fragment_colour;
  4.  
  5. // Must be the exact same name as declared in the vertex shader
  6. // ------------------------------------------------------------
  7. in vec3 vert_pos_transformed; // Transformed model position coordinates received as interpolated
  8. in vec3 vertex_normal;
  9. in vec2 texture_coordinates;
  10.  
  11. uniform sampler2D image;
  12. uniform vec3 camera_position; // -Z is into the screen... camera_position is set in main() on CPU side
  13.  
  14. void main()
  15. {   
  16.     vec3 view_direction = normalize(camera_position - vert_pos_transformed);
  17.  
  18.     vec3 light_position = vec3(0.0, 20.0, 0.0); // A position used as a light source acts as a point light (Not a directional light)
  19.     vec3 light_direction = normalize(vec3(light_position - vert_pos_transformed));   
  20.    
  21.     vec4 image_colour = texture(image, texture_coordinates);
  22.  
  23.     float ambient_factor = 0.65; // Intensity multiplier
  24.     vec4 ambient_result = vec4(ambient_factor * image_colour.rgb, 1.0);
  25.  
  26.     // Perpendicular vectors dot product = 0
  27.     // Parallel vectors in same direction dot product = 1
  28.     // Parallel vectors in opposite direction dot product = -1
  29.     // -------------------------------------------------------
  30.     float diffuse_factor = 0.85;
  31.     float diffuse_angle = max(dot(light_direction, vertex_normal), -0.45); // [-1.0 to 0] down to -1 results in darker lighting past 90 degrees
  32.     vec4 diffuse_result =  vec4(diffuse_factor * diffuse_angle * image_colour.rgb, 1.0);   
  33.        
  34.     vec3 specular_colour = vec3(0.5, 0.5, 0.5);
  35.     vec3 reflect_direction = normalize(reflect(-light_direction, vertex_normal)); // Light direction is negated here
  36.     float specular_strength = pow(max(dot(view_direction, reflect_direction), 0), 32);
  37.     vec4 specular_result = vec4(specular_colour * specular_strength, 1.0);
  38.  
  39.     fragment_colour = ambient_result + diffuse_result + specular_result;       
  40. }