diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc
index 901421c86cb3..8ac493e1d1d2 100644
--- a/cpp/src/gandiva/engine.cc
+++ b/cpp/src/gandiva/engine.cc
@@ -140,6 +140,9 @@ Result<llvm::orc::JITTargetMachineBuilder> MakeTargetMachineBuilder(
     const Configuration& conf) {
   llvm::orc::JITTargetMachineBuilder jtmb(
       (llvm::Triple(llvm::sys::getDefaultTargetTriple())));
+#if defined(__riscv)
+  jtmb.setRelocationModel(llvm::Reloc::PIC_);
+#endif
   if (conf.target_host_cpu()) {
     jtmb.setCPU(cpu_name.str());
     jtmb.addFeatures(cpu_attrs);