diff --git a/config/htcondor/execute/Dockerfile.local b/config/htcondor/execute/Dockerfile.local
index 4a2258c616ce96deb9081183fab0272cb8eea662..3259f5a5e0eec100b1c13e94dfeb81692e49d869 100644
--- a/config/htcondor/execute/Dockerfile.local
+++ b/config/htcondor/execute/Dockerfile.local
@@ -20,9 +20,20 @@ COPY docker.properties /home/casa/capo/docker.properties
 
 ENV CAPO_PROFILE "docker"
 
-RUN git clone --depth=1 https://gitlab.nrao.edu/krowe/condor_transfer_plugin.git \
+# Add gitlab as a known SSH host
+RUN mkdir -p ~/.ssh && \
+    chmod 0700 ~/.ssh && \
+    ssh-keyscan gitlab.nrao.edu > ~/.ssh/known_hosts
+
+# Link to the host's SSH key and clone over SSH (non-public repo)
+RUN --mount=type=secret,id=host_ssh_key,required \
+    ln -s /run/secrets/host_ssh_key ~/.ssh/id_rsa \
+    && git clone --depth=1 gitlab@gitlab.nrao.edu:scg/condor_transfer_plugin.git \
     && cp condor_transfer_plugin/nraorsync_plugin.py /usr/libexec/condor/
 
+# Remove SSH stuff
+RUN rm -rf ~/.ssh/
+
 # Apply config within the container
 RUN ./update-config
 RUN /root/bin/boot-execute.sh
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 4cb6492a29bdb66faa84be5bb9e20ad639c5f6d6..fdc34ec608b79651f230937b0fb3fc9831990df9 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -53,6 +53,8 @@ services:
     build:
       context: .
       dockerfile: ./config/htcondor/execute/Dockerfile.local
+      secrets:
+        - host_ssh_key # Needed to clone a repo from gitlab, will be mounted during the build
     volumes:
       - ./delivery_root:/tmp/delivery_root
       - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
@@ -267,3 +269,8 @@ services:
 
 volumes:
   condor:
+
+# Store the host's SSH key as a secret to have it removed after building the container, note only RSA for now
+secrets:
+  host_ssh_key:
+    file: ~/.ssh/id_rsa